feat(jobs): add queue delay resilience to scheduled job execution
Implement dedup key-based cron tracking to make scheduled jobs resilient to queue delays. Even if a job is delayed by minutes, it will catch the missed cron window by tracking previousRunDate in cache instead of relying on isDue() alone. - Add dedupKey parameter to shouldRunNow() in ScheduledJobManager - When provided, uses getPreviousRunDate() + cache tracking for resilience - Falls back to isDue() for docker cleanups without dedup key - Prevents double-dispatch within same cron window - Optimize ServerConnectionCheckJob dispatch - Skip SSH checks if Sentinel is healthy (enabled and live) - Reduces redundant checks when Sentinel heartbeat proves connectivity - Remove hourly Sentinel update checks - Consolidate to daily CheckAndStartSentinelJob dispatch - Crash recovery handled by sentinelOutOfSync → ServerCheckJob flow - Add logging for skipped database backups with context (backup_id, database_id, status) - Refactor skip reason methods to accept server parameter, avoiding redundant queries - Add comprehensive test suite for scheduling with various delay scenarios and timezones
This commit is contained in:
parent
f68793ed69
commit
a0c177f6f2
5 changed files with 322 additions and 106 deletions
|
|
@ -111,6 +111,12 @@ public function handle(): void
|
|||
|
||||
$status = str(data_get($this->database, 'status'));
|
||||
if (! $status->startsWith('running') && $this->database->id !== 0) {
|
||||
Log::info('DatabaseBackupJob skipped: database not running', [
|
||||
'backup_id' => $this->backup->id,
|
||||
'database_id' => $this->database->id,
|
||||
'status' => (string) $status,
|
||||
]);
|
||||
|
||||
return;
|
||||
}
|
||||
if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
|
||||
|
|
|
|||
|
|
@ -160,7 +160,8 @@ private function processScheduledBackups(): void
|
|||
|
||||
foreach ($backups as $backup) {
|
||||
try {
|
||||
$skipReason = $this->getBackupSkipReason($backup);
|
||||
$server = $backup->server();
|
||||
$skipReason = $this->getBackupSkipReason($backup, $server);
|
||||
if ($skipReason !== null) {
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('backup', $skipReason, [
|
||||
|
|
@ -173,7 +174,6 @@ private function processScheduledBackups(): void
|
|||
continue;
|
||||
}
|
||||
|
||||
$server = $backup->server();
|
||||
$serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
|
||||
|
||||
if (validate_timezone($serverTimezone) === false) {
|
||||
|
|
@ -185,7 +185,7 @@ private function processScheduledBackups(): void
|
|||
$frequency = VALID_CRON_STRINGS[$frequency];
|
||||
}
|
||||
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone)) {
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}")) {
|
||||
DatabaseBackupJob::dispatch($backup);
|
||||
$this->dispatchedCount++;
|
||||
Log::channel('scheduled')->info('Backup dispatched', [
|
||||
|
|
@ -213,19 +213,19 @@ private function processScheduledTasks(): void
|
|||
|
||||
foreach ($tasks as $task) {
|
||||
try {
|
||||
$skipReason = $this->getTaskSkipReason($task);
|
||||
$server = $task->server();
|
||||
$skipReason = $this->getTaskSkipReason($task, $server);
|
||||
if ($skipReason !== null) {
|
||||
$this->skippedCount++;
|
||||
$this->logSkip('task', $skipReason, [
|
||||
'task_id' => $task->id,
|
||||
'task_name' => $task->name,
|
||||
'team_id' => $task->server()?->team_id,
|
||||
'team_id' => $server?->team_id,
|
||||
]);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$server = $task->server();
|
||||
$serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
|
||||
|
||||
if (validate_timezone($serverTimezone) === false) {
|
||||
|
|
@ -237,7 +237,7 @@ private function processScheduledTasks(): void
|
|||
$frequency = VALID_CRON_STRINGS[$frequency];
|
||||
}
|
||||
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone)) {
|
||||
if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-task:{$task->id}")) {
|
||||
ScheduledTaskJob::dispatch($task);
|
||||
$this->dispatchedCount++;
|
||||
Log::channel('scheduled')->info('Task dispatched', [
|
||||
|
|
@ -256,7 +256,7 @@ private function processScheduledTasks(): void
|
|||
}
|
||||
}
|
||||
|
||||
private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
|
||||
private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
|
||||
{
|
||||
if (blank(data_get($backup, 'database'))) {
|
||||
$backup->delete();
|
||||
|
|
@ -264,7 +264,6 @@ private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
|
|||
return 'database_deleted';
|
||||
}
|
||||
|
||||
$server = $backup->server();
|
||||
if (blank($server)) {
|
||||
$backup->delete();
|
||||
|
||||
|
|
@ -282,12 +281,11 @@ private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
|
|||
return null;
|
||||
}
|
||||
|
||||
private function getTaskSkipReason(ScheduledTask $task): ?string
|
||||
private function getTaskSkipReason(ScheduledTask $task, ?Server $server): ?string
|
||||
{
|
||||
$service = $task->service;
|
||||
$application = $task->application;
|
||||
|
||||
$server = $task->server();
|
||||
if (blank($server)) {
|
||||
$task->delete();
|
||||
|
||||
|
|
@ -319,16 +317,38 @@ private function getTaskSkipReason(ScheduledTask $task): ?string
|
|||
return null;
|
||||
}
|
||||
|
||||
private function shouldRunNow(string $frequency, string $timezone): bool
|
||||
/**
|
||||
* Determine if a cron schedule should run now.
|
||||
*
|
||||
* When a dedupKey is provided, uses getPreviousRunDate() + last-dispatch tracking
|
||||
* instead of isDue(). This is resilient to queue delays — even if the job is delayed
|
||||
* by minutes, it still catches the missed cron window. Without dedupKey, falls back
|
||||
* to simple isDue() check.
|
||||
*/
|
||||
private function shouldRunNow(string $frequency, string $timezone, ?string $dedupKey = null): bool
|
||||
{
|
||||
$cron = new CronExpression($frequency);
|
||||
|
||||
// Use the frozen execution time, not the current time
|
||||
// Fallback to current time if execution time is not set (shouldn't happen)
|
||||
$baseTime = $this->executionTime ?? Carbon::now();
|
||||
$executionTime = $baseTime->copy()->setTimezone($timezone);
|
||||
|
||||
return $cron->isDue($executionTime);
|
||||
// No dedup key → simple isDue check (used by docker cleanups)
|
||||
if ($dedupKey === null) {
|
||||
return $cron->isDue($executionTime);
|
||||
}
|
||||
|
||||
// Get the most recent time this cron was due (including current minute)
|
||||
$previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
|
||||
|
||||
$lastDispatched = Cache::get($dedupKey);
|
||||
|
||||
// Run if: never dispatched before, OR there's been a due time since last dispatch
|
||||
if ($lastDispatched === null || $previousDue->gt(Carbon::parse($lastDispatched))) {
|
||||
Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private function processDockerCleanups(): void
|
||||
|
|
|
|||
|
|
@ -64,11 +64,11 @@ public function handle(): void
|
|||
|
||||
private function getServers(): Collection
|
||||
{
|
||||
$allServers = Server::where('ip', '!=', '1.2.3.4');
|
||||
$allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
|
||||
|
||||
if (isCloud()) {
|
||||
$servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
|
||||
$own = Team::find(0)->servers;
|
||||
$own = Team::find(0)->servers()->with('settings')->get();
|
||||
|
||||
return $servers->merge($own);
|
||||
} else {
|
||||
|
|
@ -82,6 +82,10 @@ private function dispatchConnectionChecks(Collection $servers): void
|
|||
if ($this->shouldRunNow($this->checkFrequency)) {
|
||||
$servers->each(function (Server $server) {
|
||||
try {
|
||||
// Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
|
||||
if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
|
||||
return;
|
||||
}
|
||||
ServerConnectionCheckJob::dispatch($server);
|
||||
} catch (\Exception $e) {
|
||||
Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
|
||||
|
|
@ -134,9 +138,7 @@ private function processServerTasks(Server $server): void
|
|||
// Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
|
||||
|
||||
if ($shouldRestartSentinel) {
|
||||
dispatch(function () use ($server) {
|
||||
$server->restartContainer('coolify-sentinel');
|
||||
});
|
||||
CheckAndStartSentinelJob::dispatch($server);
|
||||
}
|
||||
|
||||
// Dispatch ServerStorageCheckJob if due (only when Sentinel is out of sync or disabled)
|
||||
|
|
@ -160,11 +162,8 @@ private function processServerTasks(Server $server): void
|
|||
ServerPatchCheckJob::dispatch($server);
|
||||
}
|
||||
|
||||
// Sentinel update checks (hourly) - check for updates to Sentinel version
|
||||
// No timezone needed for hourly - runs at top of every hour
|
||||
if ($isSentinelEnabled && $this->shouldRunNow('0 * * * *')) {
|
||||
CheckAndStartSentinelJob::dispatch($server);
|
||||
}
|
||||
// Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
|
||||
// Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
|
||||
}
|
||||
|
||||
private function shouldRunNow(string $frequency, ?string $timezone = null): bool
|
||||
|
|
|
|||
208
tests/Feature/ScheduledJobManagerShouldRunNowTest.php
Normal file
208
tests/Feature/ScheduledJobManagerShouldRunNowTest.php
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
<?php
|
||||
|
||||
use App\Jobs\ScheduledJobManager;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Illuminate\Support\Facades\Cache;
|
||||
|
||||
beforeEach(function () {
|
||||
// Clear any dedup keys
|
||||
Cache::flush();
|
||||
});
|
||||
|
||||
it('dispatches backup when job runs on time at the cron minute', function () {
|
||||
// Freeze time at exactly 02:00 — daily cron "0 2 * * *" is due
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
|
||||
|
||||
$job = new ScheduledJobManager;
|
||||
|
||||
// Use reflection to test shouldRunNow
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1');
|
||||
|
||||
expect($result)->toBeTrue();
|
||||
});
|
||||
|
||||
it('dispatches backup when job is delayed past the cron minute', function () {
|
||||
// Freeze time at 02:07 — job was delayed 7 minutes past the 02:00 cron
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 7, 0, 'UTC'));
|
||||
|
||||
$job = new ScheduledJobManager;
|
||||
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
// isDue() would return false at 02:07, but getPreviousRunDate() = 02:00
|
||||
// No lastDispatched in cache → should run
|
||||
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1');
|
||||
|
||||
expect($result)->toBeTrue();
|
||||
});
|
||||
|
||||
it('does not double-dispatch on subsequent runs within same cron window', function () {
|
||||
// First run at 02:00 — dispatches and sets cache
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
|
||||
|
||||
$job = new ScheduledJobManager;
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
$first = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2');
|
||||
expect($first)->toBeTrue();
|
||||
|
||||
// Second run at 02:01 — should NOT dispatch (previousDue=02:00, lastDispatched=02:00)
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$second = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2');
|
||||
expect($second)->toBeFalse();
|
||||
});
|
||||
|
||||
it('fires every_minute cron correctly on consecutive minutes', function () {
|
||||
$job = new ScheduledJobManager;
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
// Minute 1
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
$result1 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
|
||||
expect($result1)->toBeTrue();
|
||||
|
||||
// Minute 2
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 1, 0, 'UTC'));
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
$result2 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
|
||||
expect($result2)->toBeTrue();
|
||||
|
||||
// Minute 3
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 2, 0, 'UTC'));
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
$result3 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
|
||||
expect($result3)->toBeTrue();
|
||||
});
|
||||
|
||||
it('re-dispatches after cache loss instead of missing', function () {
|
||||
// First run at 02:00 — dispatches
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
|
||||
|
||||
$job = new ScheduledJobManager;
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
$method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
|
||||
|
||||
// Simulate Redis restart — cache lost
|
||||
Cache::forget('test-backup:4');
|
||||
|
||||
// Run again at 02:01 — should re-dispatch (lastDispatched = null after cache loss)
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
|
||||
expect($result)->toBeTrue();
|
||||
});
|
||||
|
||||
it('does not dispatch when cron is not due and was not recently due', function () {
|
||||
// Time is 10:00, cron is daily at 02:00 — last due was 8 hours ago
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
|
||||
|
||||
$job = new ScheduledJobManager;
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
// previousDue = 02:00, but lastDispatched was set at 02:00 (simulate)
|
||||
Cache::put('test-backup:5', Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC')->toIso8601String(), 86400);
|
||||
|
||||
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:5');
|
||||
expect($result)->toBeFalse();
|
||||
});
|
||||
|
||||
it('falls back to isDue when no dedup key is provided', function () {
|
||||
// Time is exactly 02:00, cron is "0 2 * * *" — should be due
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
|
||||
|
||||
$job = new ScheduledJobManager;
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
// No dedup key → simple isDue check
|
||||
$result = $method->invoke($job, '0 2 * * *', 'UTC');
|
||||
expect($result)->toBeTrue();
|
||||
|
||||
// At 02:01 without dedup key → isDue returns false
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$result2 = $method->invoke($job, '0 2 * * *', 'UTC');
|
||||
expect($result2)->toBeFalse();
|
||||
});
|
||||
|
||||
it('respects server timezone for cron evaluation', function () {
|
||||
// UTC time is 22:00 Feb 28, which is 06:00 Mar 1 in Asia/Singapore (+8)
|
||||
Carbon::setTestNow(Carbon::create(2026, 2, 28, 22, 0, 0, 'UTC'));
|
||||
|
||||
$job = new ScheduledJobManager;
|
||||
$reflection = new ReflectionClass($job);
|
||||
|
||||
$executionTimeProp = $reflection->getProperty('executionTime');
|
||||
$executionTimeProp->setAccessible(true);
|
||||
$executionTimeProp->setValue($job, Carbon::now());
|
||||
|
||||
$method = $reflection->getMethod('shouldRunNow');
|
||||
$method->setAccessible(true);
|
||||
|
||||
// Simulate that today's 06:00 UTC run was already dispatched (at 06:00 UTC)
|
||||
Cache::put('test-backup:7', Carbon::create(2026, 2, 28, 6, 0, 0, 'UTC')->toIso8601String(), 86400);
|
||||
|
||||
// Cron "0 6 * * *" in Asia/Singapore: local time is 06:00 Mar 1 → previousDue = 06:00 Mar 1 SGT
|
||||
// That's a NEW cron window (Mar 1) that hasn't been dispatched → should fire
|
||||
$resultSingapore = $method->invoke($job, '0 6 * * *', 'Asia/Singapore', 'test-backup:6');
|
||||
expect($resultSingapore)->toBeTrue();
|
||||
|
||||
// Cron "0 6 * * *" in UTC: previousDue = 06:00 Feb 28 UTC, already dispatched at 06:00 → should NOT fire
|
||||
$resultUtc = $method->invoke($job, '0 6 * * *', 'UTC', 'test-backup:7');
|
||||
expect($resultUtc)->toBeFalse();
|
||||
});
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
<?php
|
||||
|
||||
use App\Jobs\CheckAndStartSentinelJob;
|
||||
use App\Jobs\ServerConnectionCheckJob;
|
||||
use App\Jobs\ServerManagerJob;
|
||||
use App\Models\InstanceSettings;
|
||||
use App\Models\Server;
|
||||
|
|
@ -10,23 +11,22 @@
|
|||
|
||||
beforeEach(function () {
|
||||
Queue::fake();
|
||||
Carbon::setTestNow('2025-01-15 12:00:00'); // Set to top of the hour for cron matching
|
||||
Carbon::setTestNow('2025-01-15 12:00:00');
|
||||
});
|
||||
|
||||
afterEach(function () {
|
||||
Mockery::close();
|
||||
Carbon::setTestNow(); // Reset frozen time
|
||||
Carbon::setTestNow();
|
||||
});
|
||||
|
||||
it('dispatches CheckAndStartSentinelJob hourly for sentinel-enabled servers', function () {
|
||||
// Mock InstanceSettings
|
||||
it('does not dispatch CheckAndStartSentinelJob hourly anymore', function () {
|
||||
$settings = Mockery::mock(InstanceSettings::class);
|
||||
$settings->instance_timezone = 'UTC';
|
||||
$this->app->instance(InstanceSettings::class, $settings);
|
||||
|
||||
// Create a mock server with sentinel enabled
|
||||
$server = Mockery::mock(Server::class)->makePartial();
|
||||
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
|
||||
$server->shouldReceive('isSentinelLive')->andReturn(true);
|
||||
$server->id = 1;
|
||||
$server->name = 'test-server';
|
||||
$server->ip = '192.168.1.100';
|
||||
|
|
@ -34,29 +34,76 @@
|
|||
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
|
||||
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
|
||||
|
||||
// Mock the Server query
|
||||
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
|
||||
Server::shouldReceive('get')->andReturn(collect([$server]));
|
||||
|
||||
// Execute the job
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Assert CheckAndStartSentinelJob was dispatched for the sentinel-enabled server
|
||||
Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
// Hourly CheckAndStartSentinelJob dispatch was removed — ServerCheckJob handles it when Sentinel is out of sync
|
||||
Queue::assertNotPushed(CheckAndStartSentinelJob::class);
|
||||
});
|
||||
|
||||
it('does not dispatch CheckAndStartSentinelJob for servers without sentinel enabled', function () {
|
||||
// Mock InstanceSettings
|
||||
it('skips ServerConnectionCheckJob when sentinel is live', function () {
|
||||
$settings = Mockery::mock(InstanceSettings::class);
|
||||
$settings->instance_timezone = 'UTC';
|
||||
$this->app->instance(InstanceSettings::class, $settings);
|
||||
|
||||
$server = Mockery::mock(Server::class)->makePartial();
|
||||
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
|
||||
$server->shouldReceive('isSentinelLive')->andReturn(true);
|
||||
$server->id = 1;
|
||||
$server->name = 'test-server';
|
||||
$server->ip = '192.168.1.100';
|
||||
$server->sentinel_updated_at = Carbon::now();
|
||||
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
|
||||
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
|
||||
|
||||
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
|
||||
Server::shouldReceive('get')->andReturn(collect([$server]));
|
||||
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Sentinel is healthy so SSH connection check is skipped
|
||||
Queue::assertNotPushed(ServerConnectionCheckJob::class);
|
||||
});
|
||||
|
||||
it('dispatches ServerConnectionCheckJob when sentinel is not live', function () {
|
||||
$settings = Mockery::mock(InstanceSettings::class);
|
||||
$settings->instance_timezone = 'UTC';
|
||||
$this->app->instance(InstanceSettings::class, $settings);
|
||||
|
||||
$server = Mockery::mock(Server::class)->makePartial();
|
||||
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
|
||||
$server->shouldReceive('isSentinelLive')->andReturn(false);
|
||||
$server->id = 1;
|
||||
$server->name = 'test-server';
|
||||
$server->ip = '192.168.1.100';
|
||||
$server->sentinel_updated_at = Carbon::now()->subMinutes(10);
|
||||
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
|
||||
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
|
||||
|
||||
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
|
||||
Server::shouldReceive('get')->andReturn(collect([$server]));
|
||||
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Sentinel is out of sync so SSH connection check is needed
|
||||
Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('dispatches ServerConnectionCheckJob when sentinel is not enabled', function () {
|
||||
$settings = Mockery::mock(InstanceSettings::class);
|
||||
$settings->instance_timezone = 'UTC';
|
||||
$this->app->instance(InstanceSettings::class, $settings);
|
||||
|
||||
// Create a mock server with sentinel disabled
|
||||
$server = Mockery::mock(Server::class)->makePartial();
|
||||
$server->shouldReceive('isSentinelEnabled')->andReturn(false);
|
||||
$server->shouldReceive('isSentinelLive')->never();
|
||||
$server->id = 2;
|
||||
$server->name = 'test-server-no-sentinel';
|
||||
$server->ip = '192.168.1.101';
|
||||
|
|
@ -64,78 +111,14 @@
|
|||
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
|
||||
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
|
||||
|
||||
// Mock the Server query
|
||||
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
|
||||
Server::shouldReceive('get')->andReturn(collect([$server]));
|
||||
|
||||
// Execute the job
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Assert CheckAndStartSentinelJob was NOT dispatched
|
||||
Queue::assertNotPushed(CheckAndStartSentinelJob::class);
|
||||
});
|
||||
|
||||
it('respects server timezone when scheduling sentinel checks', function () {
|
||||
// Mock InstanceSettings
|
||||
$settings = Mockery::mock(InstanceSettings::class);
|
||||
$settings->instance_timezone = 'UTC';
|
||||
$this->app->instance(InstanceSettings::class, $settings);
|
||||
|
||||
// Set test time to top of hour in America/New_York (which is 17:00 UTC)
|
||||
Carbon::setTestNow('2025-01-15 17:00:00'); // 12:00 PM EST (top of hour in EST)
|
||||
|
||||
// Create a mock server with sentinel enabled and America/New_York timezone
|
||||
$server = Mockery::mock(Server::class)->makePartial();
|
||||
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
|
||||
$server->id = 3;
|
||||
$server->name = 'test-server-est';
|
||||
$server->ip = '192.168.1.102';
|
||||
$server->sentinel_updated_at = Carbon::now();
|
||||
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'America/New_York']);
|
||||
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
|
||||
|
||||
// Mock the Server query
|
||||
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
|
||||
Server::shouldReceive('get')->andReturn(collect([$server]));
|
||||
|
||||
// Execute the job
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Assert CheckAndStartSentinelJob was dispatched (should run at top of hour in server's timezone)
|
||||
Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
|
||||
// Sentinel is not enabled so SSH connection check must run
|
||||
Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
|
||||
return $job->server->id === $server->id;
|
||||
});
|
||||
});
|
||||
|
||||
it('does not dispatch sentinel check when not at top of hour', function () {
|
||||
// Mock InstanceSettings
|
||||
$settings = Mockery::mock(InstanceSettings::class);
|
||||
$settings->instance_timezone = 'UTC';
|
||||
$this->app->instance(InstanceSettings::class, $settings);
|
||||
|
||||
// Set test time to middle of the hour (not top of hour)
|
||||
Carbon::setTestNow('2025-01-15 12:30:00');
|
||||
|
||||
// Create a mock server with sentinel enabled
|
||||
$server = Mockery::mock(Server::class)->makePartial();
|
||||
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
|
||||
$server->id = 4;
|
||||
$server->name = 'test-server-mid-hour';
|
||||
$server->ip = '192.168.1.103';
|
||||
$server->sentinel_updated_at = Carbon::now();
|
||||
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
|
||||
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
|
||||
|
||||
// Mock the Server query
|
||||
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
|
||||
Server::shouldReceive('get')->andReturn(collect([$server]));
|
||||
|
||||
// Execute the job
|
||||
$job = new ServerManagerJob;
|
||||
$job->handle();
|
||||
|
||||
// Assert CheckAndStartSentinelJob was NOT dispatched (not top of hour)
|
||||
Queue::assertNotPushed(CheckAndStartSentinelJob::class);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in a new issue