rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(monitoring): add scheduled job monitoring dashboard (#8433)

Browse source
This commit is contained in:
Andras Bacsai 2026-02-18 16:16:56 +01:00 committed by GitHub
commit fd24a54304
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 958 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

115
app/Jobs/ScheduledJobManager.php
View file

@ -27,6 +27,10 @@ class ScheduledJobManager implements ShouldQueue
*/
private ?Carbon $executionTime = null;
private int $dispatchedCount = 0;
private int $skippedCount = 0;
/**
* Create a new job instance.
*/
@ -61,6 +65,12 @@ public function handle(): void
{
// Freeze the execution time at the start of the job
$this->executionTime = Carbon::now();
$this->dispatchedCount = 0;
$this->skippedCount = 0;
Log::channel('scheduled')->info('ScheduledJobManager started', [
'execution_time' => $this->executionTime->toIso8601String(),
]);
// Process backups - don't let failures stop task processing
try {
@ -91,6 +101,13 @@ public function handle(): void
'trace' => $e->getTraceAsString(),
]);
}
Log::channel('scheduled')->info('ScheduledJobManager completed', [
'execution_time' => $this->executionTime->toIso8601String(),
'duration_ms' => Carbon::now()->diffInMilliseconds($this->executionTime),
'dispatched' => $this->dispatchedCount,
'skipped' => $this->skippedCount,
]);
}
private function processScheduledBackups(): void
@ -101,8 +118,16 @@ private function processScheduledBackups(): void
foreach ($backups as $backup) {
try {
// Apply the same filtering logic as the original
if (! $this->shouldProcessBackup($backup)) {
$skipReason = $this->getBackupSkipReason($backup);
if ($skipReason !== null) {
$this->skippedCount++;
$this->logSkip('backup', $skipReason, [
'backup_id' => $backup->id,
'database_id' => $backup->database_id,
'database_type' => $backup->database_type,
'team_id' => $backup->team_id ?? null,
]);
continue;
}
@ -120,6 +145,14 @@ private function processScheduledBackups(): void
if ($this->shouldRunNow($frequency, $serverTimezone)) {
DatabaseBackupJob::dispatch($backup);
$this->dispatchedCount++;
Log::channel('scheduled')->info('Backup dispatched', [
'backup_id' => $backup->id,
'database_id' => $backup->database_id,
'database_type' => $backup->database_type,
'team_id' => $backup->team_id ?? null,
'server_id' => $server->id,
]);
}
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Error processing backup', [
@ -138,7 +171,15 @@ private function processScheduledTasks(): void
foreach ($tasks as $task) {
try {
if (! $this->shouldProcessTask($task)) {
$skipReason = $this->getTaskSkipReason($task);
if ($skipReason !== null) {
$this->skippedCount++;
$this->logSkip('task', $skipReason, [
'task_id' => $task->id,
'task_name' => $task->name,
'team_id' => $task->server()?->team_id,
]);
continue;
}
@ -156,6 +197,13 @@ private function processScheduledTasks(): void
if ($this->shouldRunNow($frequency, $serverTimezone)) {
ScheduledTaskJob::dispatch($task);
$this->dispatchedCount++;
Log::channel('scheduled')->info('Task dispatched', [
'task_id' => $task->id,
'task_name' => $task->name,
'team_id' => $server->team_id,
'server_id' => $server->id,
]);
}
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Error processing task', [
@ -166,33 +214,33 @@ private function processScheduledTasks(): void
}
}
private function shouldProcessBackup(ScheduledDatabaseBackup $backup): bool
private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
{
if (blank(data_get($backup, 'database'))) {
$backup->delete();
return false;
return 'database_deleted';
}
$server = $backup->server();
if (blank($server)) {
$backup->delete();
return false;
return 'server_deleted';
}
if ($server->isFunctional() === false) {
return false;
return 'server_not_functional';
}
if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
return false;
return 'subscription_unpaid';
}
return true;
return null;
}
private function shouldProcessTask(ScheduledTask $task): bool
private function getTaskSkipReason(ScheduledTask $task): ?string
{
$service = $task->service;
$application = $task->application;
@ -201,32 +249,32 @@ private function shouldProcessTask(ScheduledTask $task): bool
if (blank($server)) {
$task->delete();
return false;
return 'server_deleted';
}
if ($server->isFunctional() === false) {
return false;
return 'server_not_functional';
}
if (isCloud() && data_get($server->team->subscription, 'stripe_invoice_paid', false) === false && $server->team->id !== 0) {
return false;
return 'subscription_unpaid';
}
if (! $service && ! $application) {
$task->delete();
return false;
return 'resource_deleted';
}
if ($application && str($application->status)->contains('running') === false) {
return false;
return 'application_not_running';
}
if ($service && str($service->status)->contains('running') === false) {
return false;
return 'service_not_running';
}
return true;
return null;
}
private function shouldRunNow(string $frequency, string $timezone): bool
@ -248,7 +296,15 @@ private function processDockerCleanups(): void
foreach ($servers as $server) {
try {
if (! $this->shouldProcessDockerCleanup($server)) {
$skipReason = $this->getDockerCleanupSkipReason($server);
if ($skipReason !== null) {
$this->skippedCount++;
$this->logSkip('docker_cleanup', $skipReason, [
'server_id' => $server->id,
'server_name' => $server->name,
'team_id' => $server->team_id,
]);
continue;
}
@ -270,6 +326,12 @@ private function processDockerCleanups(): void
$server->settings->delete_unused_volumes,
$server->settings->delete_unused_networks
);
$this->dispatchedCount++;
Log::channel('scheduled')->info('Docker cleanup dispatched', [
'server_id' => $server->id,
'server_name' => $server->name,
'team_id' => $server->team_id,
]);
}
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Error processing docker cleanup', [
@ -296,19 +358,28 @@ private function getServersForCleanup(): Collection
return $query->get();
}
private function shouldProcessDockerCleanup(Server $server): bool
private function getDockerCleanupSkipReason(Server $server): ?string
{
if (! $server->isFunctional()) {
return false;
return 'server_not_functional';
}
// In cloud, check subscription status (except team 0)
if (isCloud() && $server->team_id !== 0) {
if (data_get($server->team->subscription, 'stripe_invoice_paid', false) === false) {
return false;
return 'subscription_unpaid';
}
}
return true;
return null;
}
private function logSkip(string $type, string $reason, array $context = []): void
{
Log::channel('scheduled')->info(ucfirst(str_replace('_', ' ', $type)).' skipped', array_merge([
'type' => $type,
'skip_reason' => $reason,
'execution_time' => $this->executionTime?->toIso8601String(),
], $context));
}
}

211
app/Livewire/Settings/ScheduledJobs.php Normal file
View file

@ -0,0 +1,211 @@
<?php
namespace App\Livewire\Settings;
use App\Models\DockerCleanupExecution;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\ScheduledTaskExecution;
use App\Services\SchedulerLogParser;
use Illuminate\Support\Carbon;
use Illuminate\Support\Collection;
use Livewire\Component;
class ScheduledJobs extends Component
{
public string $filterType = 'all';
public string $filterDate = 'last_24h';
protected Collection $executions;
protected Collection $skipLogs;
protected Collection $managerRuns;
public function boot(): void
{
$this->executions = collect();
$this->skipLogs = collect();
$this->managerRuns = collect();
}
public function mount(): void
{
if (! isInstanceAdmin()) {
redirect()->route('dashboard');
return;
}
$this->loadData();
}
public function updatedFilterType(): void
{
$this->loadData();
}
public function updatedFilterDate(): void
{
$this->loadData();
}
public function refresh(): void
{
$this->loadData();
}
public function render()
{
return view('livewire.settings.scheduled-jobs', [
'executions' => $this->executions,
'skipLogs' => $this->skipLogs,
'managerRuns' => $this->managerRuns,
]);
}
private function loadData(?int $teamId = null): void
{
$this->executions = $this->getExecutions($teamId);
$parser = new SchedulerLogParser;
$this->skipLogs = $parser->getRecentSkips(50, $teamId);
$this->managerRuns = $parser->getRecentRuns(30, $teamId);
}
private function getExecutions(?int $teamId = null): Collection
{
$dateFrom = $this->getDateFrom();
$backups = collect();
$tasks = collect();
$cleanups = collect();
if ($this->filterType === 'all' || $this->filterType === 'backup') {
$backups = $this->getBackupExecutions($dateFrom, $teamId);
}
if ($this->filterType === 'all' || $this->filterType === 'task') {
$tasks = $this->getTaskExecutions($dateFrom, $teamId);
}
if ($this->filterType === 'all' || $this->filterType === 'cleanup') {
$cleanups = $this->getCleanupExecutions($dateFrom, $teamId);
}
return $backups->concat($tasks)->concat($cleanups)
->sortByDesc('created_at')
->values()
->take(100);
}
private function getBackupExecutions(?Carbon $dateFrom, ?int $teamId): Collection
{
$query = ScheduledDatabaseBackupExecution::with(['scheduledDatabaseBackup.database', 'scheduledDatabaseBackup.team'])
->where('status', 'failed')
->when($dateFrom, fn ($q) => $q->where('created_at', '>=', $dateFrom))
->when($teamId, fn ($q) => $q->whereRelation('scheduledDatabaseBackup.team', 'id', $teamId))
->orderBy('created_at', 'desc')
->limit(100)
->get();
return $query->map(function ($execution) {
$backup = $execution->scheduledDatabaseBackup;
$database = $backup?->database;
$server = $backup?->server();
return [
'id' => $execution->id,
'type' => 'backup',
'status' => $execution->status ?? 'unknown',
'resource_name' => $database?->name ?? 'Deleted database',
'resource_type' => $database ? class_basename($database) : null,
'server_name' => $server?->name ?? 'Unknown',
'server_id' => $server?->id,
'team_id' => $backup?->team_id,
'created_at' => $execution->created_at,
'finished_at' => $execution->updated_at,
'message' => $execution->message,
'size' => $execution->size ?? null,
];
});
}
private function getTaskExecutions(?Carbon $dateFrom, ?int $teamId): Collection
{
$query = ScheduledTaskExecution::with(['scheduledTask.application', 'scheduledTask.service'])
->where('status', 'failed')
->when($dateFrom, fn ($q) => $q->where('created_at', '>=', $dateFrom))
->when($teamId, function ($q) use ($teamId) {
$q->where(function ($sub) use ($teamId) {
$sub->whereRelation('scheduledTask.application.environment.project.team', 'id', $teamId)
->orWhereRelation('scheduledTask.service.environment.project.team', 'id', $teamId);
});
})
->orderBy('created_at', 'desc')
->limit(100)
->get();
return $query->map(function ($execution) {
$task = $execution->scheduledTask;
$resource = $task?->application ?? $task?->service;
$server = $task?->server();
$teamId = $server?->team_id;
return [
'id' => $execution->id,
'type' => 'task',
'status' => $execution->status ?? 'unknown',
'resource_name' => $task?->name ?? 'Deleted task',
'resource_type' => $resource ? class_basename($resource) : null,
'server_name' => $server?->name ?? 'Unknown',
'server_id' => $server?->id,
'team_id' => $teamId,
'created_at' => $execution->created_at,
'finished_at' => $execution->finished_at,
'message' => $execution->message,
'size' => null,
];
});
}
private function getCleanupExecutions(?Carbon $dateFrom, ?int $teamId): Collection
{
$query = DockerCleanupExecution::with(['server'])
->where('status', 'failed')
->when($dateFrom, fn ($q) => $q->where('created_at', '>=', $dateFrom))
->when($teamId, fn ($q) => $q->whereRelation('server', 'team_id', $teamId))
->orderBy('created_at', 'desc')
->limit(100)
->get();
return $query->map(function ($execution) {
$server = $execution->server;
return [
'id' => $execution->id,
'type' => 'cleanup',
'status' => $execution->status ?? 'unknown',
'resource_name' => $server?->name ?? 'Deleted server',
'resource_type' => 'Server',
'server_name' => $server?->name ?? 'Unknown',
'server_id' => $server?->id,
'team_id' => $server?->team_id,
'created_at' => $execution->created_at,
'finished_at' => $execution->finished_at ?? $execution->updated_at,
'message' => $execution->message,
'size' => null,
];
});
}
private function getDateFrom(): ?Carbon
{
return match ($this->filterDate) {
'last_24h' => now()->subDay(),
'last_7d' => now()->subWeek(),
'last_30d' => now()->subMonth(),
default => null,
};
}
}

188
app/Services/SchedulerLogParser.php Normal file
View file

@ -0,0 +1,188 @@
<?php
namespace App\Services;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\File;
class SchedulerLogParser
{
/**
* Get recent skip events from the scheduled log files.
*
* @return Collection<int, array{timestamp: string, type: string, reason: string, team_id: ?int, context: array}>
*/
public function getRecentSkips(int $limit = 100, ?int $teamId = null): Collection
{
$logFiles = $this->getLogFiles();
$skips = collect();
foreach ($logFiles as $logFile) {
$lines = $this->readLastLines($logFile, 2000);
foreach ($lines as $line) {
$entry = $this->parseLogLine($line);
if ($entry === null || ! isset($entry['context']['skip_reason'])) {
continue;
}
if ($teamId !== null && ($entry['context']['team_id'] ?? null) !== $teamId) {
continue;
}
$skips->push([
'timestamp' => $entry['timestamp'],
'type' => $entry['context']['type'] ?? 'unknown',
'reason' => $entry['context']['skip_reason'],
'team_id' => $entry['context']['team_id'] ?? null,
'context' => $entry['context'],
]);
}
}
return $skips->sortByDesc('timestamp')->values()->take($limit);
}
/**
* Get recent manager execution logs (start/complete events).
*
* @return Collection<int, array{timestamp: string, message: string, duration_ms: ?int, dispatched: ?int, skipped: ?int}>
*/
public function getRecentRuns(int $limit = 60, ?int $teamId = null): Collection
{
$logFiles = $this->getLogFiles();
$runs = collect();
foreach ($logFiles as $logFile) {
$lines = $this->readLastLines($logFile, 2000);
foreach ($lines as $line) {
$entry = $this->parseLogLine($line);
if ($entry === null) {
continue;
}
if (! str_contains($entry['message'], 'ScheduledJobManager')) {
continue;
}
$runs->push([
'timestamp' => $entry['timestamp'],
'message' => $entry['message'],
'duration_ms' => $entry['context']['duration_ms'] ?? null,
'dispatched' => $entry['context']['dispatched'] ?? null,
'skipped' => $entry['context']['skipped'] ?? null,
]);
}
}
return $runs->sortByDesc('timestamp')->values()->take($limit);
}
private function getLogFiles(): array
{
$logDir = storage_path('logs');
if (! File::isDirectory($logDir)) {
return [];
}
$files = File::glob($logDir.'/scheduled-*.log');
// Sort by modification time, newest first
usort($files, fn ($a, $b) => filemtime($b) - filemtime($a));
// Only check last 3 days of logs
return array_slice($files, 0, 3);
}
/**
* @return array{timestamp: string, level: string, message: string, context: array}|null
*/
private function parseLogLine(string $line): ?array
{
// Laravel daily log format: [2024-01-15 10:30:00] production.INFO: Message {"key":"value"}
if (! preg_match('/^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\] \w+\.(\w+): (.+)$/', $line, $matches)) {
return null;
}
$timestamp = $matches[1];
$level = $matches[2];
$rest = $matches[3];
// Extract JSON context if present
$context = [];
if (preg_match('/^(.+?)\s+(\{.+\})\s*$/', $rest, $contextMatches)) {
$message = $contextMatches[1];
$decoded = json_decode($contextMatches[2], true);
if (is_array($decoded)) {
$context = $decoded;
}
} else {
$message = $rest;
}
return [
'timestamp' => $timestamp,
'level' => $level,
'message' => $message,
'context' => $context,
];
}
/**
* Efficiently read the last N lines of a file.
*
* @return string[]
*/
private function readLastLines(string $filePath, int $lines): array
{
if (! File::exists($filePath)) {
return [];
}
$fileSize = File::size($filePath);
if ($fileSize === 0) {
return [];
}
// For small files, read the whole thing
if ($fileSize < 1024 * 1024) {
$content = File::get($filePath);
return array_filter(explode("\n", $content), fn ($line) => $line !== '');
}
// For large files, read from the end
$handle = fopen($filePath, 'r');
if ($handle === false) {
return [];
}
$result = [];
$chunkSize = 8192;
$buffer = '';
$position = $fileSize;
while ($position > 0 && count($result) < $lines) {
$readSize = min($chunkSize, $position);
$position -= $readSize;
fseek($handle, $position);
$buffer = fread($handle, $readSize).$buffer;
$bufferLines = explode("\n", $buffer);
$buffer = array_shift($bufferLines);
$result = array_merge(array_filter($bufferLines, fn ($line) => $line !== ''), $result);
}
if ($buffer !== '' && count($result) < $lines) {
array_unshift($result, $buffer);
}
fclose($handle);
return array_slice($result, -$lines);
}
}

4
resources/views/components/settings/navbar.blade.php
View file

@ -19,6 +19,10 @@
href="{{ route('settings.oauth') }}">
OAuth
</a>
<a class="{{ request()->routeIs('settings.scheduled-jobs') ? 'dark:text-white' : '' }}" {{ wireNavigate() }}
href="{{ route('settings.scheduled-jobs') }}">
Scheduled Jobs
</a>
<div class="flex-1"></div>
</nav>
</div>

260
resources/views/livewire/settings/scheduled-jobs.blade.php Normal file
View file

@ -0,0 +1,260 @@
<div>
<x-slot:title>
Scheduled Job Issues | Coolify
</x-slot>
<x-settings.navbar />
<div x-data="{ activeTab: window.location.hash ? window.location.hash.substring(1) : 'executions' }"
class="flex flex-col gap-8">
<div>
<div class="flex items-center gap-2">
<h2>Scheduled Job Issues</h2>
<x-forms.button wire:click="refresh">Refresh</x-forms.button>
</div>
<div class="pb-4">Shows failed executions, skipped jobs, and scheduler health.</div>
</div>
{{-- Tab Buttons --}}
<div class="flex flex-row gap-4">
<div @class([
'box-without-bg cursor-pointer dark:bg-coolgray-100 dark:text-white w-full text-center items-center justify-center',
])
:class="activeTab === 'executions' && 'dark:bg-coollabs bg-coollabs text-white'"
@click="activeTab = 'executions'; window.location.hash = 'executions'">
Failures ({{ $executions->count() }})
</div>
<div @class([
'box-without-bg cursor-pointer dark:bg-coolgray-100 dark:text-white w-full text-center items-center justify-center',
])
:class="activeTab === 'scheduler-runs' && 'dark:bg-coollabs bg-coollabs text-white'"
@click="activeTab = 'scheduler-runs'; window.location.hash = 'scheduler-runs'">
Scheduler Runs ({{ $managerRuns->count() }})
</div>
<div @class([
'box-without-bg cursor-pointer dark:bg-coolgray-100 dark:text-white w-full text-center items-center justify-center',
])
:class="activeTab === 'skipped-jobs' && 'dark:bg-coollabs bg-coollabs text-white'"
@click="activeTab = 'skipped-jobs'; window.location.hash = 'skipped-jobs'">
Skipped Jobs ({{ $skipLogs->count() }})
</div>
</div>
{{-- Executions Tab --}}
<div x-show="activeTab === 'executions'" x-cloak>
{{-- Filters --}}
<div class="flex gap-4 flex-wrap mb-4">
<div class="flex flex-col gap-1">
<label class="text-sm font-medium">Type</label>
<select wire:model.live="filterType"
class="w-40 border bg-white dark:bg-coolgray-100 border-gray-300 dark:border-coolgray-400 rounded-md text-sm">
<option value="all">All Types</option>
<option value="backup">Backups</option>
<option value="task">Tasks</option>
<option value="cleanup">Docker Cleanup</option>
</select>
</div>
<div class="flex flex-col gap-1">
<label class="text-sm font-medium">Time Range</label>
<select wire:model.live="filterDate"
class="w-40 border bg-white dark:bg-coolgray-100 border-gray-300 dark:border-coolgray-400 rounded-md text-sm">
<option value="last_24h">Last 24 Hours</option>
<option value="last_7d">Last 7 Days</option>
<option value="last_30d">Last 30 Days</option>
<option value="all">All Time</option>
</select>
</div>
</div>
<div class="overflow-x-auto">
<table class="w-full text-sm text-left">
<thead class="text-xs uppercase bg-gray-50 dark:bg-coolgray-200">
<tr>
<th class="px-4 py-3">Type</th>
<th class="px-4 py-3">Resource</th>
<th class="px-4 py-3">Server</th>
<th class="px-4 py-3">Started</th>
<th class="px-4 py-3">Duration</th>
<th class="px-4 py-3">Message</th>
</tr>
</thead>
<tbody>
@forelse($executions as $execution)
<tr wire:key="exec-{{ $execution['type'] }}-{{ $execution['id'] }}"
class="border-b border-gray-200 dark:border-coolgray-400 hover:bg-gray-50 dark:hover:bg-coolgray-200">
<td class="px-4 py-3">
@php
$typeLabel = match($execution['type']) {
'backup' => 'Backup',
'task' => 'Task',
'cleanup' => 'Cleanup',
default => ucfirst($execution['type']),
};
$typeBg = match($execution['type']) {
'backup' => 'bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-300',
'task' => 'bg-purple-100 text-purple-800 dark:bg-purple-900/30 dark:text-purple-300',
'cleanup' => 'bg-orange-100 text-orange-800 dark:bg-orange-900/30 dark:text-orange-300',
default => 'bg-gray-100 text-gray-800 dark:bg-gray-900/30 dark:text-gray-300',
};
@endphp
<span class="px-2 py-1 rounded-md text-xs font-medium {{ $typeBg }}">
{{ $typeLabel }}
</span>
</td>
<td class="px-4 py-3">
{{ $execution['resource_name'] }}
@if($execution['resource_type'])
<span class="text-xs text-gray-500">({{ $execution['resource_type'] }})</span>
@endif
</td>
<td class="px-4 py-3">{{ $execution['server_name'] }}</td>
<td class="px-4 py-3 whitespace-nowrap">
{{ $execution['created_at']->diffForHumans() }}
<span class="block text-xs text-gray-500">{{ $execution['created_at']->format('M d H:i') }}</span>
</td>
<td class="px-4 py-3 whitespace-nowrap">
@if($execution['finished_at'] && $execution['created_at'])
{{ \Carbon\Carbon::parse($execution['created_at'])->diffInSeconds(\Carbon\Carbon::parse($execution['finished_at'])) }}s
@elseif($execution['status'] === 'running')
<x-loading class="w-4 h-4" />
@else
-
@endif
</td>
<td class="px-4 py-3 max-w-xs truncate" title="{{ $execution['message'] }}">
{{ \Illuminate\Support\Str::limit($execution['message'], 80) }}
</td>
</tr>
@empty
<tr>
<td colspan="6" class="px-4 py-8 text-center text-gray-500">
No failures found for the selected filters.
</td>
</tr>
@endforelse
</tbody>
</table>
</div>
</div>
{{-- Scheduler Runs Tab --}}
<div x-show="activeTab === 'scheduler-runs'" x-cloak>
<div class="pb-4 text-sm text-gray-500">Shows when the ScheduledJobManager executed. Gaps indicate lock conflicts or missed runs.</div>
<div class="overflow-x-auto">
<table class="w-full text-sm text-left">
<thead class="text-xs uppercase bg-gray-50 dark:bg-coolgray-200">
<tr>
<th class="px-4 py-3">Time</th>
<th class="px-4 py-3">Event</th>
<th class="px-4 py-3">Duration</th>
<th class="px-4 py-3">Dispatched</th>
<th class="px-4 py-3">Skipped</th>
</tr>
</thead>
<tbody>
@forelse($managerRuns as $run)
<tr wire:key="run-{{ $loop->index }}"
class="border-b border-gray-200 dark:border-coolgray-400">
<td class="px-4 py-2 whitespace-nowrap text-xs">{{ $run['timestamp'] }}</td>
<td class="px-4 py-2">{{ $run['message'] }}</td>
<td class="px-4 py-2">
@if($run['duration_ms'] !== null)
{{ $run['duration_ms'] }}ms
@else
-
@endif
</td>
<td class="px-4 py-2">{{ $run['dispatched'] ?? '-' }}</td>
<td class="px-4 py-2">
@if(($run['skipped'] ?? 0) > 0)
<span class="text-warning">{{ $run['skipped'] }}</span>
@else
{{ $run['skipped'] ?? '-' }}
@endif
</td>
</tr>
@empty
<tr>
<td colspan="5" class="px-4 py-4 text-center text-gray-500">
No scheduler run logs found. Logs appear after the ScheduledJobManager runs.
</td>
</tr>
@endforelse
</tbody>
</table>
</div>
</div>
{{-- Skipped Jobs Tab --}}
<div x-show="activeTab === 'skipped-jobs'" x-cloak>
<div class="pb-4 text-sm text-gray-500">Jobs that were not dispatched because conditions were not met.</div>
<div class="overflow-x-auto">
<table class="w-full text-sm text-left">
<thead class="text-xs uppercase bg-gray-50 dark:bg-coolgray-200">
<tr>
<th class="px-4 py-3">Time</th>
<th class="px-4 py-3">Type</th>
<th class="px-4 py-3">Reason</th>
<th class="px-4 py-3">Details</th>
</tr>
</thead>
<tbody>
@forelse($skipLogs as $skip)
<tr wire:key="skip-{{ $loop->index }}"
class="border-b border-gray-200 dark:border-coolgray-400">
<td class="px-4 py-2 whitespace-nowrap text-xs">{{ $skip['timestamp'] }}</td>
<td class="px-4 py-2">
@php
$skipTypeBg = match($skip['type']) {
'backup' => 'bg-blue-100 text-blue-800 dark:bg-blue-900/30 dark:text-blue-300',
'task' => 'bg-purple-100 text-purple-800 dark:bg-purple-900/30 dark:text-purple-300',
'docker_cleanup' => 'bg-orange-100 text-orange-800 dark:bg-orange-900/30 dark:text-orange-300',
default => 'bg-gray-100 text-gray-800 dark:bg-gray-900/30 dark:text-gray-300',
};
@endphp
<span class="px-2 py-1 rounded-md text-xs font-medium {{ $skipTypeBg }}">
{{ ucfirst(str_replace('_', ' ', $skip['type'])) }}
</span>
</td>
<td class="px-4 py-2">
@php
$reasonLabel = match($skip['reason']) {
'server_not_functional' => 'Server not functional',
'subscription_unpaid' => 'Subscription unpaid',
'database_deleted' => 'Database deleted',
'server_deleted' => 'Server deleted',
'resource_deleted' => 'Resource deleted',
'application_not_running' => 'Application not running',
'service_not_running' => 'Service not running',
default => ucfirst(str_replace('_', ' ', $skip['reason'])),
};
$reasonBg = match($skip['reason']) {
'server_not_functional', 'database_deleted', 'server_deleted', 'resource_deleted' => 'text-red-600 dark:text-red-400',
'subscription_unpaid' => 'text-warning',
'application_not_running', 'service_not_running' => 'text-orange-600 dark:text-orange-400',
default => '',
};
@endphp
<span class="{{ $reasonBg }}">{{ $reasonLabel }}</span>
</td>
<td class="px-4 py-2 text-xs text-gray-500">
@php
$details = collect($skip['context'])
->except(['type', 'skip_reason', 'execution_time'])
->map(fn($v, $k) => str_replace('_', ' ', $k) . ': ' . $v)
->implode(', ');
@endphp
{{ $details }}
</td>
</tr>
@empty
<tr>
<td colspan="4" class="px-4 py-4 text-center text-gray-500">
No skipped jobs found. This means all scheduled jobs passed their conditions.
</td>
</tr>
@endforelse
</tbody>
</table>
</div>
</div>
</div>
</div>

2
routes/web.php
View file

@ -62,6 +62,7 @@
use App\Livewire\Server\Swarm as ServerSwarm;
use App\Livewire\Settings\Advanced as SettingsAdvanced;
use App\Livewire\Settings\Index as SettingsIndex;
use App\Livewire\Settings\ScheduledJobs as SettingsScheduledJobs;
use App\Livewire\Settings\Updates as SettingsUpdates;
use App\Livewire\SettingsBackup;
use App\Livewire\SettingsEmail;
@ -119,6 +120,7 @@
Route::get('/settings/backup', SettingsBackup::class)->name('settings.backup');
Route::get('/settings/email', SettingsEmail::class)->name('settings.email');
Route::get('/settings/oauth', SettingsOauth::class)->name('settings.oauth');
Route::get('/settings/scheduled-jobs', SettingsScheduledJobs::class)->name('settings.scheduled-jobs');
Route::get('/profile', ProfileIndex::class)->name('profile');

200
tests/Feature/ScheduledJobMonitoringTest.php Normal file
View file

@ -0,0 +1,200 @@
<?php
use App\Livewire\Settings\ScheduledJobs;
use App\Models\DockerCleanupExecution;
use App\Models\ScheduledDatabaseBackup;
use App\Models\ScheduledDatabaseBackupExecution;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use App\Services\SchedulerLogParser;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Livewire\Livewire;
uses(RefreshDatabase::class);
beforeEach(function () {
// Create root team (id 0) and root user
$this->rootTeam = Team::factory()->create(['id' => 0, 'name' => 'Root Team']);
$this->rootUser = User::factory()->create();
$this->rootUser->teams()->attach($this->rootTeam, ['role' => 'owner']);
// Create regular team and user
$this->regularTeam = Team::factory()->create();
$this->regularUser = User::factory()->create();
$this->regularUser->teams()->attach($this->regularTeam, ['role' => 'owner']);
});
test('scheduled jobs page requires instance admin access', function () {
$this->actingAs($this->regularUser);
session(['currentTeam' => $this->regularTeam]);
$response = $this->get(route('settings.scheduled-jobs'));
$response->assertRedirect(route('dashboard'));
});
test('scheduled jobs page is accessible by instance admin', function () {
$this->actingAs($this->rootUser);
session(['currentTeam' => $this->rootTeam]);
Livewire::test(ScheduledJobs::class)
->assertStatus(200)
->assertSee('Scheduled Job Issues');
});
test('scheduled jobs page shows failed backup executions', function () {
$this->actingAs($this->rootUser);
session(['currentTeam' => $this->rootTeam]);
$server = Server::factory()->create(['team_id' => $this->rootTeam->id]);
$backup = ScheduledDatabaseBackup::create([
'team_id' => $this->rootTeam->id,
'frequency' => '0 * * * *',
'database_id' => 1,
'database_type' => 'App\Models\StandalonePostgresql',
'enabled' => true,
]);
ScheduledDatabaseBackupExecution::create([
'scheduled_database_backup_id' => $backup->id,
'status' => 'failed',
'message' => 'Backup failed: connection timeout',
]);
Livewire::test(ScheduledJobs::class)
->assertStatus(200)
->assertSee('Backup');
});
test('scheduled jobs page shows failed cleanup executions', function () {
$this->actingAs($this->rootUser);
session(['currentTeam' => $this->rootTeam]);
$server = Server::factory()->create([
'team_id' => $this->rootTeam->id,
]);
DockerCleanupExecution::create([
'server_id' => $server->id,
'status' => 'failed',
'message' => 'Cleanup failed: disk full',
]);
Livewire::test(ScheduledJobs::class)
->assertStatus(200)
->assertSee('Cleanup');
});
test('filter by type works', function () {
$this->actingAs($this->rootUser);
session(['currentTeam' => $this->rootTeam]);
Livewire::test(ScheduledJobs::class)
->set('filterType', 'backup')
->assertStatus(200)
->set('filterType', 'cleanup')
->assertStatus(200)
->set('filterType', 'task')
->assertStatus(200);
});
test('only failed executions are shown', function () {
$this->actingAs($this->rootUser);
session(['currentTeam' => $this->rootTeam]);
$backup = ScheduledDatabaseBackup::create([
'team_id' => $this->rootTeam->id,
'frequency' => '0 * * * *',
'database_id' => 1,
'database_type' => 'App\Models\StandalonePostgresql',
'enabled' => true,
]);
ScheduledDatabaseBackupExecution::create([
'scheduled_database_backup_id' => $backup->id,
'status' => 'success',
'message' => 'Backup completed successfully',
]);
ScheduledDatabaseBackupExecution::create([
'scheduled_database_backup_id' => $backup->id,
'status' => 'failed',
'message' => 'Backup failed: connection refused',
]);
Livewire::test(ScheduledJobs::class)
->assertSee('Backup failed: connection refused')
->assertDontSee('Backup completed successfully');
});
test('filter by date range works', function () {
$this->actingAs($this->rootUser);
session(['currentTeam' => $this->rootTeam]);
Livewire::test(ScheduledJobs::class)
->set('filterDate', 'last_7d')
->assertStatus(200)
->set('filterDate', 'last_30d')
->assertStatus(200)
->set('filterDate', 'all')
->assertStatus(200);
});
test('scheduler log parser returns empty collection when no logs exist', function () {
$parser = new SchedulerLogParser;
$skips = $parser->getRecentSkips();
expect($skips)->toBeEmpty();
$runs = $parser->getRecentRuns();
expect($runs)->toBeEmpty();
})->skip(fn () => file_exists(storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log')), 'Skipped: log file already exists from other tests');
test('scheduler log parser parses skip entries correctly', function () {
$logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log');
$logDir = dirname($logPath);
if (! is_dir($logDir)) {
mkdir($logDir, 0755, true);
}
$logLine = '['.now()->format('Y-m-d H:i:s').'] production.INFO: Backup skipped {"type":"backup","skip_reason":"server_not_functional","execution_time":"'.now()->toIso8601String().'","backup_id":1,"team_id":5}';
file_put_contents($logPath, $logLine."\n");
$parser = new SchedulerLogParser;
$skips = $parser->getRecentSkips();
expect($skips)->toHaveCount(1);
expect($skips->first()['type'])->toBe('backup');
expect($skips->first()['reason'])->toBe('server_not_functional');
expect($skips->first()['team_id'])->toBe(5);
// Cleanup
@unlink($logPath);
});
test('scheduler log parser filters by team id', function () {
$logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log');
$logDir = dirname($logPath);
if (! is_dir($logDir)) {
mkdir($logDir, 0755, true);
}
$lines = [
'['.now()->format('Y-m-d H:i:s').'] production.INFO: Backup skipped {"type":"backup","skip_reason":"server_not_functional","team_id":1}',
'['.now()->format('Y-m-d H:i:s').'] production.INFO: Backup skipped {"type":"backup","skip_reason":"subscription_unpaid","team_id":2}',
];
file_put_contents($logPath, implode("\n", $lines)."\n");
$parser = new SchedulerLogParser;
$allSkips = $parser->getRecentSkips(100);
expect($allSkips)->toHaveCount(2);
$team1Skips = $parser->getRecentSkips(100, 1);
expect($team1Skips)->toHaveCount(1);
expect($team1Skips->first()['team_id'])->toBe(1);
// Cleanup
@unlink($logPath);
});