From 664b31212fecaf464bf719df7f722e55262b1db8 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 18 Feb 2026 15:42:42 +0100 Subject: [PATCH] chore: prepare for PR --- app/Jobs/ScheduledJobManager.php | 115 ++++++-- app/Livewire/Settings/ScheduledJobs.php | 211 ++++++++++++++ app/Services/SchedulerLogParser.php | 188 +++++++++++++ .../components/settings/navbar.blade.php | 4 + .../settings/scheduled-jobs.blade.php | 260 ++++++++++++++++++ routes/web.php | 2 + tests/Feature/ScheduledJobMonitoringTest.php | 200 ++++++++++++++ 7 files changed, 958 insertions(+), 22 deletions(-) create mode 100644 app/Livewire/Settings/ScheduledJobs.php create mode 100644 app/Services/SchedulerLogParser.php create mode 100644 resources/views/livewire/settings/scheduled-jobs.blade.php create mode 100644 tests/Feature/ScheduledJobMonitoringTest.php diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php index 75ff883c2..c9dc20af1 100644 --- a/app/Jobs/ScheduledJobManager.php +++ b/app/Jobs/ScheduledJobManager.php @@ -27,6 +27,10 @@ class ScheduledJobManager implements ShouldQueue */ private ?Carbon $executionTime = null; + private int $dispatchedCount = 0; + + private int $skippedCount = 0; + /** * Create a new job instance. */ @@ -61,6 +65,12 @@ public function handle(): void { // Freeze the execution time at the start of the job $this->executionTime = Carbon::now(); + $this->dispatchedCount = 0; + $this->skippedCount = 0; + + Log::channel('scheduled')->info('ScheduledJobManager started', [ + 'execution_time' => $this->executionTime->toIso8601String(), + ]); // Process backups - don't let failures stop task processing try { @@ -91,6 +101,13 @@ public function handle(): void 'trace' => $e->getTraceAsString(), ]); } + + Log::channel('scheduled')->info('ScheduledJobManager completed', [ + 'execution_time' => $this->executionTime->toIso8601String(), + 'duration_ms' => Carbon::now()->diffInMilliseconds($this->executionTime), + 'dispatched' => $this->dispatchedCount, + 'skipped' => $this->skippedCount, + ]); } private function processScheduledBackups(): void @@ -101,8 +118,16 @@ private function processScheduledBackups(): void foreach ($backups as $backup) { try { - // Apply the same filtering logic as the original - if (! $this->shouldProcessBackup($backup)) { + $skipReason = $this->getBackupSkipReason($backup); + if ($skipReason !== null) { + $this->skippedCount++; + $this->logSkip('backup', $skipReason, [ + 'backup_id' => $backup->id, + 'database_id' => $backup->database_id, + 'database_type' => $backup->database_type, + 'team_id' => $backup->team_id ?? null, + ]); + continue; } @@ -120,6 +145,14 @@ private function processScheduledBackups(): void if ($this->shouldRunNow($frequency, $serverTimezone)) { DatabaseBackupJob::dispatch($backup); + $this->dispatchedCount++; + Log::channel('scheduled')->info('Backup dispatched', [ + 'backup_id' => $backup->id, + 'database_id' => $backup->database_id, + 'database_type' => $backup->database_type, + 'team_id' => $backup->team_id ?? null, + 'server_id' => $server->id, + ]); } } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Error processing backup', [ @@ -138,7 +171,15 @@ private function processScheduledTasks(): void foreach ($tasks as $task) { try { - if (! $this->shouldProcessTask($task)) { + $skipReason = $this->getTaskSkipReason($task); + if ($skipReason !== null) { + $this->skippedCount++; + $this->logSkip('task', $skipReason, [ + 'task_id' => $task->id, + 'task_name' => $task->name, + 'team_id' => $task->server()?->team_id, + ]); + continue; } @@ -156,6 +197,13 @@ private function processScheduledTasks(): void if ($this->shouldRunNow($frequency, $serverTimezone)) { ScheduledTaskJob::dispatch($task); + $this->dispatchedCount++; + Log::channel('scheduled')->info('Task dispatched', [ + 'task_id' => $task->id, + 'task_name' => $task->name, + 'team_id' => $server->team_id, + 'server_id' => $server->id, + ]); } } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Error processing task', [ @@ -166,33 +214,33 @@ private function processScheduledTasks(): void } } - private function shouldProcessBackup(ScheduledDatabaseBackup $backup): bool + private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string { if (blank(data_get($backup, 'database'))) { $backup->delete(); - return false; + return 'database_deleted'; } $server = $backup->server(); if (blank($server)) { $backup->delete(); - return false; + return 'server_deleted'; } if ($server->isFunctional() === false) { - return false; + return 'server_not_functional'; } if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { - return false; + return 'subscription_unpaid'; } - return true; + return null; } - private function shouldProcessTask(ScheduledTask $task): bool + private function getTaskSkipReason(ScheduledTask $task): ?string { $service = $task->service; $application = $task->application; @@ -201,32 +249,32 @@ private function shouldProcessTask(ScheduledTask $task): bool if (blank($server)) { $task->delete(); - return false; + return 'server_deleted'; } if ($server->isFunctional() === false) { - return false; + return 'server_not_functional'; } if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) { - return false; + return 'subscription_unpaid'; } if (! $service && ! $application) { $task->delete(); - return false; + return 'resource_deleted'; } if ($application && str($application->status)->contains('running') === false) { - return false; + return 'application_not_running'; } if ($service && str($service->status)->contains('running') === false) { - return false; + return 'service_not_running'; } - return true; + return null; } private function shouldRunNow(string $frequency, string $timezone): bool @@ -248,7 +296,15 @@ private function processDockerCleanups(): void foreach ($servers as $server) { try { - if (! $this->shouldProcessDockerCleanup($server)) { + $skipReason = $this->getDockerCleanupSkipReason($server); + if ($skipReason !== null) { + $this->skippedCount++; + $this->logSkip('docker_cleanup', $skipReason, [ + 'server_id' => $server->id, + 'server_name' => $server->name, + 'team_id' => $server->team_id, + ]); + continue; } @@ -270,6 +326,12 @@ private function processDockerCleanups(): void $server->settings->delete_unused_volumes, $server->settings->delete_unused_networks ); + $this->dispatchedCount++; + Log::channel('scheduled')->info('Docker cleanup dispatched', [ + 'server_id' => $server->id, + 'server_name' => $server->name, + 'team_id' => $server->team_id, + ]); } } catch (\Exception $e) { Log::channel('scheduled-errors')->error('Error processing docker cleanup', [ @@ -296,19 +358,28 @@ private function getServersForCleanup(): Collection return $query->get(); } - private function shouldProcessDockerCleanup(Server $server): bool + private function getDockerCleanupSkipReason(Server $server): ?string { if (! $server->isFunctional()) { - return false; + return 'server_not_functional'; } // In cloud, check subscription status (except team 0) if (isCloud() && $server->team_id !== 0) { if (data_get($server->team->subscription, 'stripe_invoice_paid', false) === false) { - return false; + return 'subscription_unpaid'; } } - return true; + return null; + } + + private function logSkip(string $type, string $reason, array $context = []): void + { + Log::channel('scheduled')->info(ucfirst(str_replace('_', ' ', $type)).' skipped', array_merge([ + 'type' => $type, + 'skip_reason' => $reason, + 'execution_time' => $this->executionTime?->toIso8601String(), + ], $context)); } } diff --git a/app/Livewire/Settings/ScheduledJobs.php b/app/Livewire/Settings/ScheduledJobs.php new file mode 100644 index 000000000..66480cd8d --- /dev/null +++ b/app/Livewire/Settings/ScheduledJobs.php @@ -0,0 +1,211 @@ +executions = collect(); + $this->skipLogs = collect(); + $this->managerRuns = collect(); + } + + public function mount(): void + { + if (! isInstanceAdmin()) { + redirect()->route('dashboard'); + + return; + } + + $this->loadData(); + } + + public function updatedFilterType(): void + { + $this->loadData(); + } + + public function updatedFilterDate(): void + { + $this->loadData(); + } + + public function refresh(): void + { + $this->loadData(); + } + + public function render() + { + return view('livewire.settings.scheduled-jobs', [ + 'executions' => $this->executions, + 'skipLogs' => $this->skipLogs, + 'managerRuns' => $this->managerRuns, + ]); + } + + private function loadData(?int $teamId = null): void + { + $this->executions = $this->getExecutions($teamId); + + $parser = new SchedulerLogParser; + $this->skipLogs = $parser->getRecentSkips(50, $teamId); + $this->managerRuns = $parser->getRecentRuns(30, $teamId); + } + + private function getExecutions(?int $teamId = null): Collection + { + $dateFrom = $this->getDateFrom(); + + $backups = collect(); + $tasks = collect(); + $cleanups = collect(); + + if ($this->filterType === 'all' || $this->filterType === 'backup') { + $backups = $this->getBackupExecutions($dateFrom, $teamId); + } + + if ($this->filterType === 'all' || $this->filterType === 'task') { + $tasks = $this->getTaskExecutions($dateFrom, $teamId); + } + + if ($this->filterType === 'all' || $this->filterType === 'cleanup') { + $cleanups = $this->getCleanupExecutions($dateFrom, $teamId); + } + + return $backups->concat($tasks)->concat($cleanups) + ->sortByDesc('created_at') + ->values() + ->take(100); + } + + private function getBackupExecutions(?Carbon $dateFrom, ?int $teamId): Collection + { + $query = ScheduledDatabaseBackupExecution::with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.team']) + ->where('status', 'failed') + ->when($dateFrom, fn ($q) => $q->where('created_at', '>=', $dateFrom)) + ->when($teamId, fn ($q) => $q->whereRelation('scheduledDatabaseBackup.team', 'id', $teamId)) + ->orderBy('created_at', 'desc') + ->limit(100) + ->get(); + + return $query->map(function ($execution) { + $backup = $execution->scheduledDatabaseBackup; + $database = $backup?->database; + $server = $backup?->server(); + + return [ + 'id' => $execution->id, + 'type' => 'backup', + 'status' => $execution->status ?? 'unknown', + 'resource_name' => $database?->name ?? 'Deleted database', + 'resource_type' => $database ? class_basename($database) : null, + 'server_name' => $server?->name ?? 'Unknown', + 'server_id' => $server?->id, + 'team_id' => $backup?->team_id, + 'created_at' => $execution->created_at, + 'finished_at' => $execution->updated_at, + 'message' => $execution->message, + 'size' => $execution->size ?? null, + ]; + }); + } + + private function getTaskExecutions(?Carbon $dateFrom, ?int $teamId): Collection + { + $query = ScheduledTaskExecution::with(['scheduledTask.application', 'scheduledTask.service']) + ->where('status', 'failed') + ->when($dateFrom, fn ($q) => $q->where('created_at', '>=', $dateFrom)) + ->when($teamId, function ($q) use ($teamId) { + $q->where(function ($sub) use ($teamId) { + $sub->whereRelation('scheduledTask.application.environment.project.team', 'id', $teamId) + ->orWhereRelation('scheduledTask.service.environment.project.team', 'id', $teamId); + }); + }) + ->orderBy('created_at', 'desc') + ->limit(100) + ->get(); + + return $query->map(function ($execution) { + $task = $execution->scheduledTask; + $resource = $task?->application ?? $task?->service; + $server = $task?->server(); + $teamId = $server?->team_id; + + return [ + 'id' => $execution->id, + 'type' => 'task', + 'status' => $execution->status ?? 'unknown', + 'resource_name' => $task?->name ?? 'Deleted task', + 'resource_type' => $resource ? class_basename($resource) : null, + 'server_name' => $server?->name ?? 'Unknown', + 'server_id' => $server?->id, + 'team_id' => $teamId, + 'created_at' => $execution->created_at, + 'finished_at' => $execution->finished_at, + 'message' => $execution->message, + 'size' => null, + ]; + }); + } + + private function getCleanupExecutions(?Carbon $dateFrom, ?int $teamId): Collection + { + $query = DockerCleanupExecution::with(['server']) + ->where('status', 'failed') + ->when($dateFrom, fn ($q) => $q->where('created_at', '>=', $dateFrom)) + ->when($teamId, fn ($q) => $q->whereRelation('server', 'team_id', $teamId)) + ->orderBy('created_at', 'desc') + ->limit(100) + ->get(); + + return $query->map(function ($execution) { + $server = $execution->server; + + return [ + 'id' => $execution->id, + 'type' => 'cleanup', + 'status' => $execution->status ?? 'unknown', + 'resource_name' => $server?->name ?? 'Deleted server', + 'resource_type' => 'Server', + 'server_name' => $server?->name ?? 'Unknown', + 'server_id' => $server?->id, + 'team_id' => $server?->team_id, + 'created_at' => $execution->created_at, + 'finished_at' => $execution->finished_at ?? $execution->updated_at, + 'message' => $execution->message, + 'size' => null, + ]; + }); + } + + private function getDateFrom(): ?Carbon + { + return match ($this->filterDate) { + 'last_24h' => now()->subDay(), + 'last_7d' => now()->subWeek(), + 'last_30d' => now()->subMonth(), + default => null, + }; + } +} diff --git a/app/Services/SchedulerLogParser.php b/app/Services/SchedulerLogParser.php new file mode 100644 index 000000000..a735a11c3 --- /dev/null +++ b/app/Services/SchedulerLogParser.php @@ -0,0 +1,188 @@ + + */ + public function getRecentSkips(int $limit = 100, ?int $teamId = null): Collection + { + $logFiles = $this->getLogFiles(); + + $skips = collect(); + + foreach ($logFiles as $logFile) { + $lines = $this->readLastLines($logFile, 2000); + + foreach ($lines as $line) { + $entry = $this->parseLogLine($line); + if ($entry === null || ! isset($entry['context']['skip_reason'])) { + continue; + } + + if ($teamId !== null && ($entry['context']['team_id'] ?? null) !== $teamId) { + continue; + } + + $skips->push([ + 'timestamp' => $entry['timestamp'], + 'type' => $entry['context']['type'] ?? 'unknown', + 'reason' => $entry['context']['skip_reason'], + 'team_id' => $entry['context']['team_id'] ?? null, + 'context' => $entry['context'], + ]); + } + } + + return $skips->sortByDesc('timestamp')->values()->take($limit); + } + + /** + * Get recent manager execution logs (start/complete events). + * + * @return Collection + */ + public function getRecentRuns(int $limit = 60, ?int $teamId = null): Collection + { + $logFiles = $this->getLogFiles(); + + $runs = collect(); + + foreach ($logFiles as $logFile) { + $lines = $this->readLastLines($logFile, 2000); + + foreach ($lines as $line) { + $entry = $this->parseLogLine($line); + if ($entry === null) { + continue; + } + + if (! str_contains($entry['message'], 'ScheduledJobManager')) { + continue; + } + + $runs->push([ + 'timestamp' => $entry['timestamp'], + 'message' => $entry['message'], + 'duration_ms' => $entry['context']['duration_ms'] ?? null, + 'dispatched' => $entry['context']['dispatched'] ?? null, + 'skipped' => $entry['context']['skipped'] ?? null, + ]); + } + } + + return $runs->sortByDesc('timestamp')->values()->take($limit); + } + + private function getLogFiles(): array + { + $logDir = storage_path('logs'); + if (! File::isDirectory($logDir)) { + return []; + } + + $files = File::glob($logDir.'/scheduled-*.log'); + + // Sort by modification time, newest first + usort($files, fn ($a, $b) => filemtime($b) - filemtime($a)); + + // Only check last 3 days of logs + return array_slice($files, 0, 3); + } + + /** + * @return array{timestamp: string, level: string, message: string, context: array}|null + */ + private function parseLogLine(string $line): ?array + { + // Laravel daily log format: [2024-01-15 10:30:00] production.INFO: Message {"key":"value"} + if (! preg_match('/^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\] \w+\.(\w+): (.+)$/', $line, $matches)) { + return null; + } + + $timestamp = $matches[1]; + $level = $matches[2]; + $rest = $matches[3]; + + // Extract JSON context if present + $context = []; + if (preg_match('/^(.+?)\s+(\{.+\})\s*$/', $rest, $contextMatches)) { + $message = $contextMatches[1]; + $decoded = json_decode($contextMatches[2], true); + if (is_array($decoded)) { + $context = $decoded; + } + } else { + $message = $rest; + } + + return [ + 'timestamp' => $timestamp, + 'level' => $level, + 'message' => $message, + 'context' => $context, + ]; + } + + /** + * Efficiently read the last N lines of a file. + * + * @return string[] + */ + private function readLastLines(string $filePath, int $lines): array + { + if (! File::exists($filePath)) { + return []; + } + + $fileSize = File::size($filePath); + if ($fileSize === 0) { + return []; + } + + // For small files, read the whole thing + if ($fileSize < 1024 * 1024) { + $content = File::get($filePath); + + return array_filter(explode("\n", $content), fn ($line) => $line !== ''); + } + + // For large files, read from the end + $handle = fopen($filePath, 'r'); + if ($handle === false) { + return []; + } + + $result = []; + $chunkSize = 8192; + $buffer = ''; + $position = $fileSize; + + while ($position > 0 && count($result) < $lines) { + $readSize = min($chunkSize, $position); + $position -= $readSize; + fseek($handle, $position); + $buffer = fread($handle, $readSize).$buffer; + + $bufferLines = explode("\n", $buffer); + $buffer = array_shift($bufferLines); + + $result = array_merge(array_filter($bufferLines, fn ($line) => $line !== ''), $result); + } + + if ($buffer !== '' && count($result) < $lines) { + array_unshift($result, $buffer); + } + + fclose($handle); + + return array_slice($result, -$lines); + } +} diff --git a/resources/views/components/settings/navbar.blade.php b/resources/views/components/settings/navbar.blade.php index 10df20e03..565e485d0 100644 --- a/resources/views/components/settings/navbar.blade.php +++ b/resources/views/components/settings/navbar.blade.php @@ -19,6 +19,10 @@ href="{{ route('settings.oauth') }}"> OAuth + + Scheduled Jobs +
diff --git a/resources/views/livewire/settings/scheduled-jobs.blade.php b/resources/views/livewire/settings/scheduled-jobs.blade.php new file mode 100644 index 000000000..d22aca911 --- /dev/null +++ b/resources/views/livewire/settings/scheduled-jobs.blade.php @@ -0,0 +1,260 @@ +
+ + Scheduled Job Issues | Coolify + + +
+
+
+

Scheduled Job Issues

+ Refresh +
+
Shows failed executions, skipped jobs, and scheduler health.
+
+ + {{-- Tab Buttons --}} +
+
+ Failures ({{ $executions->count() }}) +
+
+ Scheduler Runs ({{ $managerRuns->count() }}) +
+
+ Skipped Jobs ({{ $skipLogs->count() }}) +
+
+ + {{-- Executions Tab --}} +
+ {{-- Filters --}} +
+
+ + +
+
+ + +
+
+ +
+ + + + + + + + + + + + + @forelse($executions as $execution) + + + + + + + + + @empty + + + + @endforelse + +
TypeResourceServerStartedDurationMessage
+ @php + $typeLabel = match($execution['type']) { + 'backup' => 'Backup', + 'task' => 'Task', + 'cleanup' => 'Cleanup', + default => ucfirst($execution['type']), + }; + $typeBg = match($execution['type']) { + 'backup' => 'bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-300', + 'task' => 'bg-purple-100 text-purple-800 dark:bg-purple-900/30 dark:text-purple-300', + 'cleanup' => 'bg-orange-100 text-orange-800 dark:bg-orange-900/30 dark:text-orange-300', + default => 'bg-gray-100 text-gray-800 dark:bg-gray-900/30 dark:text-gray-300', + }; + @endphp + + {{ $typeLabel }} + + + {{ $execution['resource_name'] }} + @if($execution['resource_type']) + ({{ $execution['resource_type'] }}) + @endif + {{ $execution['server_name'] }} + {{ $execution['created_at']->diffForHumans() }} + {{ $execution['created_at']->format('M d H:i') }} + + @if($execution['finished_at'] && $execution['created_at']) + {{ \Carbon\Carbon::parse($execution['created_at'])->diffInSeconds(\Carbon\Carbon::parse($execution['finished_at'])) }}s + @elseif($execution['status'] === 'running') + + @else + - + @endif + + {{ \Illuminate\Support\Str::limit($execution['message'], 80) }} +
+ No failures found for the selected filters. +
+
+
+ + {{-- Scheduler Runs Tab --}} +
+
Shows when the ScheduledJobManager executed. Gaps indicate lock conflicts or missed runs.
+
+ + + + + + + + + + + + @forelse($managerRuns as $run) + + + + + + + + @empty + + + + @endforelse + +
TimeEventDurationDispatchedSkipped
{{ $run['timestamp'] }}{{ $run['message'] }} + @if($run['duration_ms'] !== null) + {{ $run['duration_ms'] }}ms + @else + - + @endif + {{ $run['dispatched'] ?? '-' }} + @if(($run['skipped'] ?? 0) > 0) + {{ $run['skipped'] }} + @else + {{ $run['skipped'] ?? '-' }} + @endif +
+ No scheduler run logs found. Logs appear after the ScheduledJobManager runs. +
+
+
+ + {{-- Skipped Jobs Tab --}} +
+
Jobs that were not dispatched because conditions were not met.
+
+ + + + + + + + + + + @forelse($skipLogs as $skip) + + + + + + + @empty + + + + @endforelse + +
TimeTypeReasonDetails
{{ $skip['timestamp'] }} + @php + $skipTypeBg = match($skip['type']) { + 'backup' => 'bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-300', + 'task' => 'bg-purple-100 text-purple-800 dark:bg-purple-900/30 dark:text-purple-300', + 'docker_cleanup' => 'bg-orange-100 text-orange-800 dark:bg-orange-900/30 dark:text-orange-300', + default => 'bg-gray-100 text-gray-800 dark:bg-gray-900/30 dark:text-gray-300', + }; + @endphp + + {{ ucfirst(str_replace('_', ' ', $skip['type'])) }} + + + @php + $reasonLabel = match($skip['reason']) { + 'server_not_functional' => 'Server not functional', + 'subscription_unpaid' => 'Subscription unpaid', + 'database_deleted' => 'Database deleted', + 'server_deleted' => 'Server deleted', + 'resource_deleted' => 'Resource deleted', + 'application_not_running' => 'Application not running', + 'service_not_running' => 'Service not running', + default => ucfirst(str_replace('_', ' ', $skip['reason'])), + }; + $reasonBg = match($skip['reason']) { + 'server_not_functional', 'database_deleted', 'server_deleted', 'resource_deleted' => 'text-red-600 dark:text-red-400', + 'subscription_unpaid' => 'text-warning', + 'application_not_running', 'service_not_running' => 'text-orange-600 dark:text-orange-400', + default => '', + }; + @endphp + {{ $reasonLabel }} + + @php + $details = collect($skip['context']) + ->except(['type', 'skip_reason', 'execution_time']) + ->map(fn($v, $k) => str_replace('_', ' ', $k) . ': ' . $v) + ->implode(', '); + @endphp + {{ $details }} +
+ No skipped jobs found. This means all scheduled jobs passed their conditions. +
+
+
+
+
diff --git a/routes/web.php b/routes/web.php index e8c738b71..b6c6c95ce 100644 --- a/routes/web.php +++ b/routes/web.php @@ -62,6 +62,7 @@ use App\Livewire\Server\Swarm as ServerSwarm; use App\Livewire\Settings\Advanced as SettingsAdvanced; use App\Livewire\Settings\Index as SettingsIndex; +use App\Livewire\Settings\ScheduledJobs as SettingsScheduledJobs; use App\Livewire\Settings\Updates as SettingsUpdates; use App\Livewire\SettingsBackup; use App\Livewire\SettingsEmail; @@ -119,6 +120,7 @@ Route::get('/settings/backup', SettingsBackup::class)->name('settings.backup'); Route::get('/settings/email', SettingsEmail::class)->name('settings.email'); Route::get('/settings/oauth', SettingsOauth::class)->name('settings.oauth'); + Route::get('/settings/scheduled-jobs', SettingsScheduledJobs::class)->name('settings.scheduled-jobs'); Route::get('/profile', ProfileIndex::class)->name('profile'); diff --git a/tests/Feature/ScheduledJobMonitoringTest.php b/tests/Feature/ScheduledJobMonitoringTest.php new file mode 100644 index 000000000..1348375d4 --- /dev/null +++ b/tests/Feature/ScheduledJobMonitoringTest.php @@ -0,0 +1,200 @@ +rootTeam = Team::factory()->create(['id' => 0, 'name' => 'Root Team']); + $this->rootUser = User::factory()->create(); + $this->rootUser->teams()->attach($this->rootTeam, ['role' => 'owner']); + + // Create regular team and user + $this->regularTeam = Team::factory()->create(); + $this->regularUser = User::factory()->create(); + $this->regularUser->teams()->attach($this->regularTeam, ['role' => 'owner']); +}); + +test('scheduled jobs page requires instance admin access', function () { + $this->actingAs($this->regularUser); + session(['currentTeam' => $this->regularTeam]); + + $response = $this->get(route('settings.scheduled-jobs')); + $response->assertRedirect(route('dashboard')); +}); + +test('scheduled jobs page is accessible by instance admin', function () { + $this->actingAs($this->rootUser); + session(['currentTeam' => $this->rootTeam]); + + Livewire::test(ScheduledJobs::class) + ->assertStatus(200) + ->assertSee('Scheduled Job Issues'); +}); + +test('scheduled jobs page shows failed backup executions', function () { + $this->actingAs($this->rootUser); + session(['currentTeam' => $this->rootTeam]); + + $server = Server::factory()->create(['team_id' => $this->rootTeam->id]); + + $backup = ScheduledDatabaseBackup::create([ + 'team_id' => $this->rootTeam->id, + 'frequency' => '0 * * * *', + 'database_id' => 1, + 'database_type' => 'App\Models\StandalonePostgresql', + 'enabled' => true, + ]); + + ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'failed', + 'message' => 'Backup failed: connection timeout', + ]); + + Livewire::test(ScheduledJobs::class) + ->assertStatus(200) + ->assertSee('Backup'); +}); + +test('scheduled jobs page shows failed cleanup executions', function () { + $this->actingAs($this->rootUser); + session(['currentTeam' => $this->rootTeam]); + + $server = Server::factory()->create([ + 'team_id' => $this->rootTeam->id, + ]); + + DockerCleanupExecution::create([ + 'server_id' => $server->id, + 'status' => 'failed', + 'message' => 'Cleanup failed: disk full', + ]); + + Livewire::test(ScheduledJobs::class) + ->assertStatus(200) + ->assertSee('Cleanup'); +}); + +test('filter by type works', function () { + $this->actingAs($this->rootUser); + session(['currentTeam' => $this->rootTeam]); + + Livewire::test(ScheduledJobs::class) + ->set('filterType', 'backup') + ->assertStatus(200) + ->set('filterType', 'cleanup') + ->assertStatus(200) + ->set('filterType', 'task') + ->assertStatus(200); +}); + +test('only failed executions are shown', function () { + $this->actingAs($this->rootUser); + session(['currentTeam' => $this->rootTeam]); + + $backup = ScheduledDatabaseBackup::create([ + 'team_id' => $this->rootTeam->id, + 'frequency' => '0 * * * *', + 'database_id' => 1, + 'database_type' => 'App\Models\StandalonePostgresql', + 'enabled' => true, + ]); + + ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 'message' => 'Backup completed successfully', + ]); + + ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'failed', + 'message' => 'Backup failed: connection refused', + ]); + + Livewire::test(ScheduledJobs::class) + ->assertSee('Backup failed: connection refused') + ->assertDontSee('Backup completed successfully'); +}); + +test('filter by date range works', function () { + $this->actingAs($this->rootUser); + session(['currentTeam' => $this->rootTeam]); + + Livewire::test(ScheduledJobs::class) + ->set('filterDate', 'last_7d') + ->assertStatus(200) + ->set('filterDate', 'last_30d') + ->assertStatus(200) + ->set('filterDate', 'all') + ->assertStatus(200); +}); + +test('scheduler log parser returns empty collection when no logs exist', function () { + $parser = new SchedulerLogParser; + + $skips = $parser->getRecentSkips(); + expect($skips)->toBeEmpty(); + + $runs = $parser->getRecentRuns(); + expect($runs)->toBeEmpty(); +})->skip(fn () => file_exists(storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log')), 'Skipped: log file already exists from other tests'); + +test('scheduler log parser parses skip entries correctly', function () { + $logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log'); + $logDir = dirname($logPath); + if (! is_dir($logDir)) { + mkdir($logDir, 0755, true); + } + + $logLine = '['.now()->format('Y-m-d H:i:s').'] production.INFO: Backup skipped {"type":"backup","skip_reason":"server_not_functional","execution_time":"'.now()->toIso8601String().'","backup_id":1,"team_id":5}'; + file_put_contents($logPath, $logLine."\n"); + + $parser = new SchedulerLogParser; + $skips = $parser->getRecentSkips(); + + expect($skips)->toHaveCount(1); + expect($skips->first()['type'])->toBe('backup'); + expect($skips->first()['reason'])->toBe('server_not_functional'); + expect($skips->first()['team_id'])->toBe(5); + + // Cleanup + @unlink($logPath); +}); + +test('scheduler log parser filters by team id', function () { + $logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log'); + $logDir = dirname($logPath); + if (! is_dir($logDir)) { + mkdir($logDir, 0755, true); + } + + $lines = [ + '['.now()->format('Y-m-d H:i:s').'] production.INFO: Backup skipped {"type":"backup","skip_reason":"server_not_functional","team_id":1}', + '['.now()->format('Y-m-d H:i:s').'] production.INFO: Backup skipped {"type":"backup","skip_reason":"subscription_unpaid","team_id":2}', + ]; + file_put_contents($logPath, implode("\n", $lines)."\n"); + + $parser = new SchedulerLogParser; + + $allSkips = $parser->getRecentSkips(100); + expect($allSkips)->toHaveCount(2); + + $team1Skips = $parser->getRecentSkips(100, 1); + expect($team1Skips)->toHaveCount(1); + expect($team1Skips->first()['team_id'])->toBe(1); + + // Cleanup + @unlink($logPath); +});