From a0c177f6f24f1f75bfaa84d894b020f9cd60f774 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 28 Feb 2026 15:06:25 +0100 Subject: [PATCH] feat(jobs): add queue delay resilience to scheduled job execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement dedup key-based cron tracking to make scheduled jobs resilient to queue delays. Even if a job is delayed by minutes, it will catch the missed cron window by tracking previousRunDate in cache instead of relying on isDue() alone. - Add dedupKey parameter to shouldRunNow() in ScheduledJobManager - When provided, uses getPreviousRunDate() + cache tracking for resilience - Falls back to isDue() for docker cleanups without dedup key - Prevents double-dispatch within same cron window - Optimize ServerConnectionCheckJob dispatch - Skip SSH checks if Sentinel is healthy (enabled and live) - Reduces redundant checks when Sentinel heartbeat proves connectivity - Remove hourly Sentinel update checks - Consolidate to daily CheckAndStartSentinelJob dispatch - Crash recovery handled by sentinelOutOfSync → ServerCheckJob flow - Add logging for skipped database backups with context (backup_id, database_id, status) - Refactor skip reason methods to accept server parameter, avoiding redundant queries - Add comprehensive test suite for scheduling with various delay scenarios and timezones --- app/Jobs/DatabaseBackupJob.php | 6 + app/Jobs/ScheduledJobManager.php | 52 +++-- app/Jobs/ServerManagerJob.php | 19 +- .../ScheduledJobManagerShouldRunNowTest.php | 208 ++++++++++++++++++ .../ServerManagerJobSentinelCheckTest.php | 143 ++++++------ 5 files changed, 322 insertions(+), 106 deletions(-) create mode 100644 tests/Feature/ScheduledJobManagerShouldRunNowTest.php diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index a585baa69..f2f454f87 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -111,6 +111,12 @@ public function handle(): void $status = str(data_get($this->database, 'status')); if (! $status->startsWith('running') && $this->database->id !== 0) { + Log::info('DatabaseBackupJob skipped: database not running', [ + 'backup_id' => $this->backup->id, + 'database_id' => $this->database->id, + 'status' => (string) $status, + ]); + return; } if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) { diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php index de782b96d..fd641abb0 100644 --- a/app/Jobs/ScheduledJobManager.php +++ b/app/Jobs/ScheduledJobManager.php @@ -160,7 +160,8 @@ private function processScheduledBackups(): void foreach ($backups as $backup) { try { - $skipReason = $this->getBackupSkipReason($backup); + $server = $backup->server(); + $skipReason = $this->getBackupSkipReason($backup, $server); if ($skipReason !== null) { $this->skippedCount++; $this->logSkip('backup', $skipReason, [ @@ -173,7 +174,6 @@ private function processScheduledBackups(): void continue; } - $server = $backup->server(); $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone')); if (validate_timezone($serverTimezone) === false) { @@ -185,7 +185,7 @@ private function processScheduledBackups(): void $frequency = VALID_CRON_STRINGS[$frequency]; } - if ($this->shouldRunNow($frequency, $serverTimezone)) { + if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}")) { DatabaseBackupJob::dispatch($backup); $this->dispatchedCount++; Log::channel('scheduled')->info('Backup dispatched', [ @@ -213,19 +213,19 @@ private function processScheduledTasks(): void foreach ($tasks as $task) { try { - $skipReason = $this->getTaskSkipReason($task); + $server = $task->server(); + $skipReason = $this->getTaskSkipReason($task, $server); if ($skipReason !== null) { $this->skippedCount++; $this->logSkip('task', $skipReason, [ 'task_id' => $task->id, 'task_name' => $task->name, - 'team_id' => $task->server()?->team_id, + 'team_id' => $server?->team_id, ]); continue; } - $server = $task->server(); $serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone')); if (validate_timezone($serverTimezone) === false) { @@ -237,7 +237,7 @@ private function processScheduledTasks(): void $frequency = VALID_CRON_STRINGS[$frequency]; } - if ($this->shouldRunNow($frequency, $serverTimezone)) { + if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-task:{$task->id}")) { ScheduledTaskJob::dispatch($task); $this->dispatchedCount++; Log::channel('scheduled')->info('Task dispatched', [ @@ -256,7 +256,7 @@ private function processScheduledTasks(): void } } - private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string + private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string { if (blank(data_get($backup, 'database'))) { $backup->delete(); @@ -264,7 +264,6 @@ private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string return 'database_deleted'; } - $server = $backup->server(); if (blank($server)) { $backup->delete(); @@ -282,12 +281,11 @@ private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string return null; } - private function getTaskSkipReason(ScheduledTask $task): ?string + private function getTaskSkipReason(ScheduledTask $task, ?Server $server): ?string { $service = $task->service; $application = $task->application; - $server = $task->server(); if (blank($server)) { $task->delete(); @@ -319,16 +317,38 @@ private function getTaskSkipReason(ScheduledTask $task): ?string return null; } - private function shouldRunNow(string $frequency, string $timezone): bool + /** + * Determine if a cron schedule should run now. + * + * When a dedupKey is provided, uses getPreviousRunDate() + last-dispatch tracking + * instead of isDue(). This is resilient to queue delays — even if the job is delayed + * by minutes, it still catches the missed cron window. Without dedupKey, falls back + * to simple isDue() check. + */ + private function shouldRunNow(string $frequency, string $timezone, ?string $dedupKey = null): bool { $cron = new CronExpression($frequency); - - // Use the frozen execution time, not the current time - // Fallback to current time if execution time is not set (shouldn't happen) $baseTime = $this->executionTime ?? Carbon::now(); $executionTime = $baseTime->copy()->setTimezone($timezone); - return $cron->isDue($executionTime); + // No dedup key → simple isDue check (used by docker cleanups) + if ($dedupKey === null) { + return $cron->isDue($executionTime); + } + + // Get the most recent time this cron was due (including current minute) + $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true)); + + $lastDispatched = Cache::get($dedupKey); + + // Run if: never dispatched before, OR there's been a due time since last dispatch + if ($lastDispatched === null || $previousDue->gt(Carbon::parse($lastDispatched))) { + Cache::put($dedupKey, $executionTime->toIso8601String(), 86400); + + return true; + } + + return false; } private function processDockerCleanups(): void diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php index a4619354d..c8219a2ea 100644 --- a/app/Jobs/ServerManagerJob.php +++ b/app/Jobs/ServerManagerJob.php @@ -64,11 +64,11 @@ public function handle(): void private function getServers(): Collection { - $allServers = Server::where('ip', '!=', '1.2.3.4'); + $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4'); if (isCloud()) { $servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get(); - $own = Team::find(0)->servers; + $own = Team::find(0)->servers()->with('settings')->get(); return $servers->merge($own); } else { @@ -82,6 +82,10 @@ private function dispatchConnectionChecks(Collection $servers): void if ($this->shouldRunNow($this->checkFrequency)) { $servers->each(function (Server $server) { try { + // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity + if ($server->isSentinelEnabled() && $server->isSentinelLive()) { + return; + } ServerConnectionCheckJob::dispatch($server); } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [ @@ -134,9 +138,7 @@ private function processServerTasks(Server $server): void // Dispatch Sentinel restart if due (daily for Sentinel-enabled servers) if ($shouldRestartSentinel) { - dispatch(function () use ($server) { - $server->restartContainer('coolify-sentinel'); - }); + CheckAndStartSentinelJob::dispatch($server); } // Dispatch ServerStorageCheckJob if due (only when Sentinel is out of sync or disabled) @@ -160,11 +162,8 @@ private function processServerTasks(Server $server): void ServerPatchCheckJob::dispatch($server); } - // Sentinel update checks (hourly) - check for updates to Sentinel version - // No timezone needed for hourly - runs at top of every hour - if ($isSentinelEnabled && $this->shouldRunNow('0 * * * *')) { - CheckAndStartSentinelJob::dispatch($server); - } + // Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates. + // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob. } private function shouldRunNow(string $frequency, ?string $timezone = null): bool diff --git a/tests/Feature/ScheduledJobManagerShouldRunNowTest.php b/tests/Feature/ScheduledJobManagerShouldRunNowTest.php new file mode 100644 index 000000000..8862cc71e --- /dev/null +++ b/tests/Feature/ScheduledJobManagerShouldRunNowTest.php @@ -0,0 +1,208 @@ +getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + $executionTimeProp->setValue($job, Carbon::now()); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1'); + + expect($result)->toBeTrue(); +}); + +it('dispatches backup when job is delayed past the cron minute', function () { + // Freeze time at 02:07 — job was delayed 7 minutes past the 02:00 cron + Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 7, 0, 'UTC')); + + $job = new ScheduledJobManager; + + $reflection = new ReflectionClass($job); + + $executionTimeProp = $reflection->getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + $executionTimeProp->setValue($job, Carbon::now()); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + // isDue() would return false at 02:07, but getPreviousRunDate() = 02:00 + // No lastDispatched in cache → should run + $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1'); + + expect($result)->toBeTrue(); +}); + +it('does not double-dispatch on subsequent runs within same cron window', function () { + // First run at 02:00 — dispatches and sets cache + Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC')); + + $job = new ScheduledJobManager; + $reflection = new ReflectionClass($job); + + $executionTimeProp = $reflection->getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + $executionTimeProp->setValue($job, Carbon::now()); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + $first = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2'); + expect($first)->toBeTrue(); + + // Second run at 02:01 — should NOT dispatch (previousDue=02:00, lastDispatched=02:00) + Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC')); + $executionTimeProp->setValue($job, Carbon::now()); + + $second = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2'); + expect($second)->toBeFalse(); +}); + +it('fires every_minute cron correctly on consecutive minutes', function () { + $job = new ScheduledJobManager; + $reflection = new ReflectionClass($job); + + $executionTimeProp = $reflection->getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + // Minute 1 + Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC')); + $executionTimeProp->setValue($job, Carbon::now()); + $result1 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3'); + expect($result1)->toBeTrue(); + + // Minute 2 + Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 1, 0, 'UTC')); + $executionTimeProp->setValue($job, Carbon::now()); + $result2 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3'); + expect($result2)->toBeTrue(); + + // Minute 3 + Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 2, 0, 'UTC')); + $executionTimeProp->setValue($job, Carbon::now()); + $result3 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3'); + expect($result3)->toBeTrue(); +}); + +it('re-dispatches after cache loss instead of missing', function () { + // First run at 02:00 — dispatches + Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC')); + + $job = new ScheduledJobManager; + $reflection = new ReflectionClass($job); + + $executionTimeProp = $reflection->getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + $executionTimeProp->setValue($job, Carbon::now()); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4'); + + // Simulate Redis restart — cache lost + Cache::forget('test-backup:4'); + + // Run again at 02:01 — should re-dispatch (lastDispatched = null after cache loss) + Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC')); + $executionTimeProp->setValue($job, Carbon::now()); + + $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4'); + expect($result)->toBeTrue(); +}); + +it('does not dispatch when cron is not due and was not recently due', function () { + // Time is 10:00, cron is daily at 02:00 — last due was 8 hours ago + Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC')); + + $job = new ScheduledJobManager; + $reflection = new ReflectionClass($job); + + $executionTimeProp = $reflection->getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + $executionTimeProp->setValue($job, Carbon::now()); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + // previousDue = 02:00, but lastDispatched was set at 02:00 (simulate) + Cache::put('test-backup:5', Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC')->toIso8601String(), 86400); + + $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:5'); + expect($result)->toBeFalse(); +}); + +it('falls back to isDue when no dedup key is provided', function () { + // Time is exactly 02:00, cron is "0 2 * * *" — should be due + Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC')); + + $job = new ScheduledJobManager; + $reflection = new ReflectionClass($job); + + $executionTimeProp = $reflection->getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + $executionTimeProp->setValue($job, Carbon::now()); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + // No dedup key → simple isDue check + $result = $method->invoke($job, '0 2 * * *', 'UTC'); + expect($result)->toBeTrue(); + + // At 02:01 without dedup key → isDue returns false + Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC')); + $executionTimeProp->setValue($job, Carbon::now()); + + $result2 = $method->invoke($job, '0 2 * * *', 'UTC'); + expect($result2)->toBeFalse(); +}); + +it('respects server timezone for cron evaluation', function () { + // UTC time is 22:00 Feb 28, which is 06:00 Mar 1 in Asia/Singapore (+8) + Carbon::setTestNow(Carbon::create(2026, 2, 28, 22, 0, 0, 'UTC')); + + $job = new ScheduledJobManager; + $reflection = new ReflectionClass($job); + + $executionTimeProp = $reflection->getProperty('executionTime'); + $executionTimeProp->setAccessible(true); + $executionTimeProp->setValue($job, Carbon::now()); + + $method = $reflection->getMethod('shouldRunNow'); + $method->setAccessible(true); + + // Simulate that today's 06:00 UTC run was already dispatched (at 06:00 UTC) + Cache::put('test-backup:7', Carbon::create(2026, 2, 28, 6, 0, 0, 'UTC')->toIso8601String(), 86400); + + // Cron "0 6 * * *" in Asia/Singapore: local time is 06:00 Mar 1 → previousDue = 06:00 Mar 1 SGT + // That's a NEW cron window (Mar 1) that hasn't been dispatched → should fire + $resultSingapore = $method->invoke($job, '0 6 * * *', 'Asia/Singapore', 'test-backup:6'); + expect($resultSingapore)->toBeTrue(); + + // Cron "0 6 * * *" in UTC: previousDue = 06:00 Feb 28 UTC, already dispatched at 06:00 → should NOT fire + $resultUtc = $method->invoke($job, '0 6 * * *', 'UTC', 'test-backup:7'); + expect($resultUtc)->toBeFalse(); +}); diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.php index 0f2613f11..d8449adc3 100644 --- a/tests/Unit/ServerManagerJobSentinelCheckTest.php +++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php @@ -1,6 +1,7 @@ instance_timezone = 'UTC'; $this->app->instance(InstanceSettings::class, $settings); - // Create a mock server with sentinel enabled $server = Mockery::mock(Server::class)->makePartial(); $server->shouldReceive('isSentinelEnabled')->andReturn(true); + $server->shouldReceive('isSentinelLive')->andReturn(true); $server->id = 1; $server->name = 'test-server'; $server->ip = '192.168.1.100'; @@ -34,29 +34,76 @@ $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']); $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120); - // Mock the Server query Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf(); Server::shouldReceive('get')->andReturn(collect([$server])); - // Execute the job $job = new ServerManagerJob; $job->handle(); - // Assert CheckAndStartSentinelJob was dispatched for the sentinel-enabled server - Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) { - return $job->server->id === $server->id; - }); + // Hourly CheckAndStartSentinelJob dispatch was removed — ServerCheckJob handles it when Sentinel is out of sync + Queue::assertNotPushed(CheckAndStartSentinelJob::class); }); -it('does not dispatch CheckAndStartSentinelJob for servers without sentinel enabled', function () { - // Mock InstanceSettings +it('skips ServerConnectionCheckJob when sentinel is live', function () { + $settings = Mockery::mock(InstanceSettings::class); + $settings->instance_timezone = 'UTC'; + $this->app->instance(InstanceSettings::class, $settings); + + $server = Mockery::mock(Server::class)->makePartial(); + $server->shouldReceive('isSentinelEnabled')->andReturn(true); + $server->shouldReceive('isSentinelLive')->andReturn(true); + $server->id = 1; + $server->name = 'test-server'; + $server->ip = '192.168.1.100'; + $server->sentinel_updated_at = Carbon::now(); + $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']); + $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120); + + Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf(); + Server::shouldReceive('get')->andReturn(collect([$server])); + + $job = new ServerManagerJob; + $job->handle(); + + // Sentinel is healthy so SSH connection check is skipped + Queue::assertNotPushed(ServerConnectionCheckJob::class); +}); + +it('dispatches ServerConnectionCheckJob when sentinel is not live', function () { + $settings = Mockery::mock(InstanceSettings::class); + $settings->instance_timezone = 'UTC'; + $this->app->instance(InstanceSettings::class, $settings); + + $server = Mockery::mock(Server::class)->makePartial(); + $server->shouldReceive('isSentinelEnabled')->andReturn(true); + $server->shouldReceive('isSentinelLive')->andReturn(false); + $server->id = 1; + $server->name = 'test-server'; + $server->ip = '192.168.1.100'; + $server->sentinel_updated_at = Carbon::now()->subMinutes(10); + $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']); + $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120); + + Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf(); + Server::shouldReceive('get')->andReturn(collect([$server])); + + $job = new ServerManagerJob; + $job->handle(); + + // Sentinel is out of sync so SSH connection check is needed + Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) { + return $job->server->id === $server->id; + }); +}); + +it('dispatches ServerConnectionCheckJob when sentinel is not enabled', function () { $settings = Mockery::mock(InstanceSettings::class); $settings->instance_timezone = 'UTC'; $this->app->instance(InstanceSettings::class, $settings); - // Create a mock server with sentinel disabled $server = Mockery::mock(Server::class)->makePartial(); $server->shouldReceive('isSentinelEnabled')->andReturn(false); + $server->shouldReceive('isSentinelLive')->never(); $server->id = 2; $server->name = 'test-server-no-sentinel'; $server->ip = '192.168.1.101'; @@ -64,78 +111,14 @@ $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']); $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120); - // Mock the Server query Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf(); Server::shouldReceive('get')->andReturn(collect([$server])); - // Execute the job $job = new ServerManagerJob; $job->handle(); - // Assert CheckAndStartSentinelJob was NOT dispatched - Queue::assertNotPushed(CheckAndStartSentinelJob::class); -}); - -it('respects server timezone when scheduling sentinel checks', function () { - // Mock InstanceSettings - $settings = Mockery::mock(InstanceSettings::class); - $settings->instance_timezone = 'UTC'; - $this->app->instance(InstanceSettings::class, $settings); - - // Set test time to top of hour in America/New_York (which is 17:00 UTC) - Carbon::setTestNow('2025-01-15 17:00:00'); // 12:00 PM EST (top of hour in EST) - - // Create a mock server with sentinel enabled and America/New_York timezone - $server = Mockery::mock(Server::class)->makePartial(); - $server->shouldReceive('isSentinelEnabled')->andReturn(true); - $server->id = 3; - $server->name = 'test-server-est'; - $server->ip = '192.168.1.102'; - $server->sentinel_updated_at = Carbon::now(); - $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'America/New_York']); - $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120); - - // Mock the Server query - Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf(); - Server::shouldReceive('get')->andReturn(collect([$server])); - - // Execute the job - $job = new ServerManagerJob; - $job->handle(); - - // Assert CheckAndStartSentinelJob was dispatched (should run at top of hour in server's timezone) - Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) { + // Sentinel is not enabled so SSH connection check must run + Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) { return $job->server->id === $server->id; }); }); - -it('does not dispatch sentinel check when not at top of hour', function () { - // Mock InstanceSettings - $settings = Mockery::mock(InstanceSettings::class); - $settings->instance_timezone = 'UTC'; - $this->app->instance(InstanceSettings::class, $settings); - - // Set test time to middle of the hour (not top of hour) - Carbon::setTestNow('2025-01-15 12:30:00'); - - // Create a mock server with sentinel enabled - $server = Mockery::mock(Server::class)->makePartial(); - $server->shouldReceive('isSentinelEnabled')->andReturn(true); - $server->id = 4; - $server->name = 'test-server-mid-hour'; - $server->ip = '192.168.1.103'; - $server->sentinel_updated_at = Carbon::now(); - $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']); - $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120); - - // Mock the Server query - Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf(); - Server::shouldReceive('get')->andReturn(collect([$server])); - - // Execute the job - $job = new ServerManagerJob; - $job->handle(); - - // Assert CheckAndStartSentinelJob was NOT dispatched (not top of hour) - Queue::assertNotPushed(CheckAndStartSentinelJob::class); -});