Decouple ServerStorageCheckJob from Sentinel sync status
Server disk usage checks now run on their configured schedule regardless of Sentinel status, eliminating monitoring blind spots when Sentinel is offline, out of sync, or disabled. Storage checks now respect server timezone settings, consistent with patch checks. Changes: - Moved server timezone calculation to top of processServerTasks() - Extracted ServerStorageCheckJob dispatch from Sentinel conditional - Fixed default frequency to '0 23 * * *' (11 PM daily) - Added timezone parameter to storage check scheduling 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
2302a70a44
commit
b47181c790
2 changed files with 204 additions and 15 deletions
|
|
@ -111,32 +111,33 @@ private function processScheduledTasks(Collection $servers): void
|
|||
|
||||
private function processServerTasks(Server $server): void
|
||||
{
|
||||
// Get server timezone (used for all scheduled tasks)
|
||||
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
||||
if (validate_timezone($serverTimezone) === false) {
|
||||
$serverTimezone = config('app.timezone');
|
||||
}
|
||||
|
||||
// Check if we should run sentinel-based checks
|
||||
$lastSentinelUpdate = $server->sentinel_updated_at;
|
||||
$waitTime = $server->waitBeforeDoingSshCheck();
|
||||
$sentinelOutOfSync = Carbon::parse($lastSentinelUpdate)->isBefore($this->executionTime->subSeconds($waitTime));
|
||||
|
||||
if ($sentinelOutOfSync) {
|
||||
// Dispatch jobs if Sentinel is out of sync
|
||||
// Dispatch ServerCheckJob if Sentinel is out of sync
|
||||
if ($this->shouldRunNow($this->checkFrequency)) {
|
||||
ServerCheckJob::dispatch($server);
|
||||
}
|
||||
|
||||
// Dispatch ServerStorageCheckJob if due
|
||||
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 * * * *');
|
||||
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
|
||||
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
|
||||
}
|
||||
$shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency);
|
||||
|
||||
if ($shouldRunStorageCheck) {
|
||||
ServerStorageCheckJob::dispatch($server);
|
||||
}
|
||||
}
|
||||
|
||||
$serverTimezone = data_get($server->settings, 'server_timezone', $this->instanceTimezone);
|
||||
if (validate_timezone($serverTimezone) === false) {
|
||||
$serverTimezone = config('app.timezone');
|
||||
// Dispatch ServerStorageCheckJob if due (independent of Sentinel status)
|
||||
$serverDiskUsageCheckFrequency = data_get($server->settings, 'server_disk_usage_check_frequency', '0 23 * * *');
|
||||
if (isset(VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency])) {
|
||||
$serverDiskUsageCheckFrequency = VALID_CRON_STRINGS[$serverDiskUsageCheckFrequency];
|
||||
}
|
||||
$shouldRunStorageCheck = $this->shouldRunNow($serverDiskUsageCheckFrequency, $serverTimezone);
|
||||
|
||||
if ($shouldRunStorageCheck) {
|
||||
ServerStorageCheckJob::dispatch($server);
|
||||
}
|
||||
|
||||
// Dispatch ServerPatchCheckJob if due (weekly)
|
||||
|
|
|
|||
188
tests/Feature/ServerStorageCheckIndependenceTest.php
Normal file
188
tests/Feature/ServerStorageCheckIndependenceTest.php
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
<?php
|
||||
|
||||
use App\Jobs\ServerCheckJob;
|
||||
use App\Jobs\ServerManagerJob;
|
||||
use App\Jobs\ServerStorageCheckJob;
|
||||
use App\Models\Server;
|
||||
use App\Models\Team;
|
||||
use Carbon\Carbon;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Queue;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Queue::fake();
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
Carbon::setTestNow();
|
||||
});
|
||||
|
||||
it('dispatches storage check when sentinel is in sync', function () {
|
||||
// Given: A server with Sentinel recently updated (in sync)
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'sentinel_updated_at' => now(),
|
||||
]);
|
||||
|
||||
$server->settings->update([
|
||||
'server_disk_usage_check_frequency' => '0 23 * * *',
|
||||
'server_timezone' => 'UTC',
|
||||
]);
|
||||
|
||||
// When: ServerManagerJob runs at 11 PM
|
||||
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Then: ServerStorageCheckJob should be dispatched
|
||||
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('dispatches storage check when sentinel is out of sync', function () {
|
||||
// Given: A server with Sentinel out of sync (last update 10 minutes ago)
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'sentinel_updated_at' => now()->subMinutes(10),
|
||||
]);
|
||||
|
||||
$server->settings->update([
|
||||
'server_disk_usage_check_frequency' => '0 23 * * *',
|
||||
'server_timezone' => 'UTC',
|
||||
]);
|
||||
|
||||
// When: ServerManagerJob runs at 11 PM
|
||||
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Then: Both ServerCheckJob and ServerStorageCheckJob should be dispatched
|
||||
Queue::assertPushed(ServerCheckJob::class);
|
||||
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('dispatches storage check when sentinel is disabled', function () {
|
||||
// Given: A server with Sentinel disabled
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'sentinel_updated_at' => now()->subHours(24),
|
||||
]);
|
||||
|
||||
$server->settings->update([
|
||||
'server_disk_usage_check_frequency' => '0 23 * * *',
|
||||
'server_timezone' => 'UTC',
|
||||
'is_metrics_enabled' => false,
|
||||
]);
|
||||
|
||||
// When: ServerManagerJob runs at 11 PM
|
||||
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Then: ServerStorageCheckJob should be dispatched
|
||||
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('respects custom hourly storage check frequency', function () {
|
||||
// Given: A server with hourly storage check frequency
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'sentinel_updated_at' => now(),
|
||||
]);
|
||||
|
||||
$server->settings->update([
|
||||
'server_disk_usage_check_frequency' => '0 * * * *',
|
||||
'server_timezone' => 'UTC',
|
||||
]);
|
||||
|
||||
// When: ServerManagerJob runs at the top of the hour (23:00)
|
||||
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Then: ServerStorageCheckJob should be dispatched
|
||||
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('handles VALID_CRON_STRINGS mapping correctly', function () {
|
||||
// Given: A server with 'hourly' string (should be converted to '0 * * * *')
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'sentinel_updated_at' => now(),
|
||||
]);
|
||||
|
||||
$server->settings->update([
|
||||
'server_disk_usage_check_frequency' => 'hourly',
|
||||
'server_timezone' => 'UTC',
|
||||
]);
|
||||
|
||||
// When: ServerManagerJob runs at the top of the hour
|
||||
Carbon::setTestNow(Carbon::parse('2025-01-15 23:00:00', 'UTC'));
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Then: ServerStorageCheckJob should be dispatched (hourly was converted to cron)
|
||||
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('respects server timezone for storage checks', function () {
|
||||
// Given: A server in America/New_York timezone (UTC-5) configured for 11 PM local time
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'sentinel_updated_at' => now(),
|
||||
]);
|
||||
|
||||
$server->settings->update([
|
||||
'server_disk_usage_check_frequency' => '0 23 * * *',
|
||||
'server_timezone' => 'America/New_York',
|
||||
]);
|
||||
|
||||
// When: ServerManagerJob runs at 11 PM New York time (4 AM UTC next day)
|
||||
Carbon::setTestNow(Carbon::parse('2025-01-16 04:00:00', 'UTC'));
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Then: ServerStorageCheckJob should be dispatched
|
||||
Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('does not dispatch storage check outside schedule', function () {
|
||||
// Given: A server with daily storage check at 11 PM
|
||||
$team = Team::factory()->create();
|
||||
$server = Server::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'sentinel_updated_at' => now(),
|
||||
]);
|
||||
|
||||
$server->settings->update([
|
||||
'server_disk_usage_check_frequency' => '0 23 * * *',
|
||||
'server_timezone' => 'UTC',
|
||||
]);
|
||||
|
||||
// When: ServerManagerJob runs at 10 PM (not 11 PM)
|
||||
Carbon::setTestNow(Carbon::parse('2025-01-15 22:00:00', 'UTC'));
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Then: ServerStorageCheckJob should NOT be dispatched
|
||||
Queue::assertNotPushed(ServerStorageCheckJob::class);
|
||||
});
|
||||
Loading…
Reference in a new issue