fix(jobs): prevent non-due jobs firing on restart and enrich skip logs with resource links

- Refactor shouldRunNow() to only fire on first run (empty cache) if actually due by cron schedule, preventing spurious executions after cache loss or service restart
- Add enrichSkipLogsWithLinks() method to fetch and populate resource names and links for tasks, backups, and docker cleanup jobs in skip logs
- Update skip logs UI to display resource column with links to related resources, improving navigation and context
- Add fallback display when linked resources are deleted
- Expand tests to cover both restart scenarios: non-due jobs (should not fire) and due jobs (should fire)
This commit is contained in:
Andras Bacsai 2026-02-28 18:03:29 +01:00
parent 63be5928ab
commit 31555f9e8a
5 changed files with 166 additions and 29 deletions

View file

@ -341,8 +341,19 @@ private function shouldRunNow(string $frequency, string $timezone, ?string $dedu
$lastDispatched = Cache::get($dedupKey);
// Run if: never dispatched before, OR there's been a due time since last dispatch
if ($lastDispatched === null || $previousDue->gt(Carbon::parse($lastDispatched))) {
if ($lastDispatched === null) {
// First run after restart or cache loss: only fire if actually due right now.
// Seed the cache so subsequent runs can use tolerance/catch-up logic.
$isDue = $cron->isDue($executionTime);
if ($isDue) {
Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
}
return $isDue;
}
// Subsequent runs: fire if there's been a due time since last dispatch
if ($previousDue->gt(Carbon::parse($lastDispatched))) {
Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
return true;

View file

@ -3,8 +3,11 @@
namespace App\Livewire\Settings;
use App\Models\DockerCleanupExecution;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledTask;
use App\Models\ScheduledTaskExecution;
use App\Models\Server;
use App\Services\SchedulerLogParser;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
@ -102,13 +105,84 @@ private function loadData(?int $teamId = null): void
$parser = new SchedulerLogParser;
$allSkips = $parser->getRecentSkips(500, $teamId);
$this->skipTotalCount = $allSkips->count();
$this->skipLogs = $allSkips->slice($this->skipPage, $this->skipDefaultTake)->values();
$this->skipLogs = $this->enrichSkipLogsWithLinks(
$allSkips->slice($this->skipPage, $this->skipDefaultTake)->values()
);
$this->showSkipPrev = $this->skipPage > 0;
$this->showSkipNext = ($this->skipPage + $this->skipDefaultTake) < $this->skipTotalCount;
$this->skipCurrentPage = intval($this->skipPage / $this->skipDefaultTake) + 1;
$this->managerRuns = $parser->getRecentRuns(30, $teamId);
}
private function enrichSkipLogsWithLinks(Collection $skipLogs): Collection
{
$taskIds = $skipLogs->where('type', 'task')->pluck('context.task_id')->filter()->unique()->values();
$backupIds = $skipLogs->where('type', 'backup')->pluck('context.backup_id')->filter()->unique()->values();
$serverIds = $skipLogs->where('type', 'docker_cleanup')->pluck('context.server_id')->filter()->unique()->values();
$tasks = $taskIds->isNotEmpty()
? ScheduledTask::with(['application.environment.project', 'service.environment.project'])->whereIn('id', $taskIds)->get()->keyBy('id')
: collect();
$backups = $backupIds->isNotEmpty()
? ScheduledDatabaseBackup::with(['database.environment.project'])->whereIn('id', $backupIds)->get()->keyBy('id')
: collect();
$servers = $serverIds->isNotEmpty()
? Server::whereIn('id', $serverIds)->get()->keyBy('id')
: collect();
return $skipLogs->map(function (array $skip) use ($tasks, $backups, $servers): array {
$skip['link'] = null;
$skip['resource_name'] = null;
if ($skip['type'] === 'task') {
$task = $tasks->get($skip['context']['task_id'] ?? null);
if ($task) {
$skip['resource_name'] = $skip['context']['task_name'] ?? $task->name;
$resource = $task->application ?? $task->service;
$environment = $resource?->environment;
$project = $environment?->project;
if ($project && $environment && $resource) {
$routeName = $task->application_id
? 'project.application.scheduled-tasks'
: 'project.service.scheduled-tasks';
$routeKey = $task->application_id ? 'application_uuid' : 'service_uuid';
$skip['link'] = route($routeName, [
'project_uuid' => $project->uuid,
'environment_uuid' => $environment->uuid,
$routeKey => $resource->uuid,
'task_uuid' => $task->uuid,
]);
}
}
} elseif ($skip['type'] === 'backup') {
$backup = $backups->get($skip['context']['backup_id'] ?? null);
if ($backup) {
$database = $backup->database;
$skip['resource_name'] = $database?->name ?? 'Database backup';
$environment = $database?->environment;
$project = $environment?->project;
if ($project && $environment && $database) {
$skip['link'] = route('project.database.backup.index', [
'project_uuid' => $project->uuid,
'environment_uuid' => $environment->uuid,
'database_uuid' => $database->uuid,
]);
}
}
} elseif ($skip['type'] === 'docker_cleanup') {
$server = $servers->get($skip['context']['server_id'] ?? null);
if ($server) {
$skip['resource_name'] = $server->name;
$skip['link'] = route('server.show', ['server_uuid' => $server->uuid]);
}
}
return $skip;
});
}
private function getExecutions(?int $teamId = null): Collection
{
$dateFrom = $this->getDateFrom();

View file

@ -213,8 +213,8 @@ class="border-b border-gray-200 dark:border-coolgray-400">
<tr>
<th class="px-4 py-3">Time</th>
<th class="px-4 py-3">Type</th>
<th class="px-4 py-3">Resource</th>
<th class="px-4 py-3">Reason</th>
<th class="px-4 py-3">Details</th>
</tr>
</thead>
<tbody>
@ -235,6 +235,17 @@ class="border-b border-gray-200 dark:border-coolgray-400">
{{ ucfirst(str_replace('_', ' ', $skip['type'])) }}
</span>
</td>
<td class="px-4 py-2">
@if($skip['link'] ?? null)
<a href="{{ $skip['link'] }}" class="text-white underline hover:no-underline">
{{ $skip['resource_name'] }}
</a>
@elseif($skip['resource_name'] ?? null)
{{ $skip['resource_name'] }}
@else
<span class="text-gray-500">{{ $skip['context']['task_name'] ?? $skip['context']['server_name'] ?? 'Deleted' }}</span>
@endif
</td>
<td class="px-4 py-2">
@php
$reasonLabel = match($skip['reason']) {
@ -256,15 +267,6 @@ class="border-b border-gray-200 dark:border-coolgray-400">
@endphp
<span class="{{ $reasonBg }}">{{ $reasonLabel }}</span>
</td>
<td class="px-4 py-2 text-xs text-gray-500">
@php
$details = collect($skip['context'])
->except(['type', 'skip_reason', 'execution_time'])
->map(fn($v, $k) => str_replace('_', ' ', $k) . ': ' . $v)
->implode(', ');
@endphp
{{ $details }}
</td>
</tr>
@empty
<tr>

View file

@ -30,8 +30,11 @@
expect($result)->toBeTrue();
});
it('dispatches backup when job is delayed past the cron minute', function () {
// Freeze time at 02:07 — job was delayed 7 minutes past the 02:00 cron
it('catches delayed job when cache has a baseline from previous run', function () {
// Simulate a previous dispatch yesterday at 02:00
Cache::put('test-backup:1', Carbon::create(2026, 2, 27, 2, 0, 0, 'UTC')->toIso8601String(), 86400);
// Freeze time at 02:07 — job was delayed 7 minutes past today's 02:00 cron
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 7, 0, 'UTC'));
$job = new ScheduledJobManager;
@ -45,8 +48,8 @@
$method = $reflection->getMethod('shouldRunNow');
$method->setAccessible(true);
// isDue() would return false at 02:07, but getPreviousRunDate() = 02:00
// No lastDispatched in cache → should run
// isDue() would return false at 02:07, but getPreviousRunDate() = 02:00 today
// lastDispatched = 02:00 yesterday → 02:00 today > yesterday → fires
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1');
expect($result)->toBeTrue();
@ -106,8 +109,27 @@
expect($result3)->toBeTrue();
});
it('re-dispatches after cache loss instead of missing', function () {
// First run at 02:00 — dispatches
it('does not fire non-due jobs on restart when cache is empty', function () {
// Time is 10:00, cron is daily at 02:00 — NOT due right now
Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
$job = new ScheduledJobManager;
$reflection = new ReflectionClass($job);
$executionTimeProp = $reflection->getProperty('executionTime');
$executionTimeProp->setAccessible(true);
$executionTimeProp->setValue($job, Carbon::now());
$method = $reflection->getMethod('shouldRunNow');
$method->setAccessible(true);
// Cache is empty (fresh restart) — should NOT fire daily backup at 10:00
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
expect($result)->toBeFalse();
});
it('fires due jobs on restart when cache is empty', function () {
// Time is exactly 02:00, cron is daily at 02:00 — IS due right now
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
$job = new ScheduledJobManager;
@ -120,16 +142,8 @@
$method = $reflection->getMethod('shouldRunNow');
$method->setAccessible(true);
$method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
// Simulate Redis restart — cache lost
Cache::forget('test-backup:4');
// Run again at 02:01 — should re-dispatch (lastDispatched = null after cache loss)
Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
$executionTimeProp->setValue($job, Carbon::now());
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
// Cache is empty (fresh restart) — but cron IS due → should fire
$result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4b');
expect($result)->toBeTrue();
});

View file

@ -234,3 +234,39 @@
// Cleanup
@unlink($logPath);
});
test('skipped jobs show fallback when resource is deleted', function () {
$this->actingAs($this->rootUser);
session(['currentTeam' => $this->rootTeam]);
$logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log');
$logDir = dirname($logPath);
if (! is_dir($logDir)) {
mkdir($logDir, 0755, true);
}
// Temporarily rename existing logs so they don't interfere
$existingLogs = glob(storage_path('logs/scheduled-*.log'));
$renamed = [];
foreach ($existingLogs as $log) {
$tmp = $log.'.bak';
rename($log, $tmp);
$renamed[$tmp] = $log;
}
$lines = [
'['.now()->format('Y-m-d H:i:s').'] production.INFO: Task skipped {"type":"task","skip_reason":"application_not_running","task_id":99999,"task_name":"my-cron-job","team_id":0}',
];
file_put_contents($logPath, implode("\n", $lines)."\n");
Livewire::test(ScheduledJobs::class)
->assertStatus(200)
->assertSee('my-cron-job')
->assertSee('Application not running');
// Cleanup
@unlink($logPath);
foreach ($renamed as $tmp => $original) {
rename($tmp, $original);
}
});