From 0b462f6a58641b40f2c49b3045dc28cb426a9e7e Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Mon, 7 Sep 2026 14:09:42 +0200 Subject: [PATCH] feat(backups): alert when scheduled backups are missing (#11433) --- app/Console/Kernel.php | 2 + .../Controllers/Api/DatabasesController.php | 10 +- app/Jobs/CheckMissingDatabaseBackupsJob.php | 59 ++++++++ app/Livewire/Project/Database/BackupEdit.php | 5 + app/Models/ScheduledDatabaseBackup.php | 4 + .../ScheduledDatabaseBackupExecution.php | 7 + .../Channels/TelegramChannel.php | 4 +- app/Notifications/Database/BackupMissing.php | 83 ++++++++++++ ...ds_to_scheduled_database_backups_table.php | 28 ++++ .../views/emails/backup-missing.blade.php | 7 + .../database/backup-edit/general.blade.php | 3 + tests/Feature/Api/LifecycleApisTest.php | 15 ++- .../MissingDatabaseBackupNotificationTest.php | 126 ++++++++++++++++++ 13 files changed, 349 insertions(+), 4 deletions(-) create mode 100644 app/Jobs/CheckMissingDatabaseBackupsJob.php create mode 100644 app/Notifications/Database/BackupMissing.php create mode 100644 database/migrations/2026_08_20_150000_add_missing_backup_notification_fields_to_scheduled_database_backups_table.php create mode 100644 resources/views/emails/backup-missing.blade.php create mode 100644 tests/Feature/MissingDatabaseBackupNotificationTest.php diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 8eb010b8a..79660c249 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -5,6 +5,7 @@ use App\Jobs\ApiTokenExpirationWarningJob; use App\Jobs\CheckForUpdatesJob; use App\Jobs\CheckHelperImageJob; +use App\Jobs\CheckMissingDatabaseBackupsJob; use App\Jobs\CheckTraefikVersionJob; use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\CleanupOrphanedPreviewContainersJob; @@ -53,6 +54,7 @@ protected function schedule(Schedule $schedule): void ->runInBackground(); $this->scheduleInstance->command('sanctum:prune-expired --hours=1')->hourly()->onOneServer(); $this->scheduleInstance->job(new ApiTokenExpirationWarningJob)->hourly()->onOneServer(); + $this->scheduleInstance->job(new CheckMissingDatabaseBackupsJob)->hourly()->onOneServer(); if (isDev()) { // Instance Jobs diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index aeb69ac8b..881aa5a89 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -769,6 +769,7 @@ public function update_by_uuid(Request $request) 'database_backup_retention_days_s3' => ['type' => 'integer', 'description' => 'Number of days to retain backups in S3'], 'database_backup_retention_max_storage_s3' => ['type' => 'number', 'description' => 'Max storage (GB) for S3 backups'], 'timeout' => ['type' => 'integer', 'description' => 'Backup job timeout in seconds (min: 60, max: 36000)', 'default' => 3600], + 'missing_backup_notification_days' => ['type' => 'integer', 'description' => 'Alert after this many days without an execution; 0 disables alerts', 'minimum' => 0, 'maximum' => 365, 'default' => 0], ], ), ) @@ -805,7 +806,7 @@ public function update_by_uuid(Request $request) )] public function create_backup(Request $request) { - $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout']; + $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days']; $teamId = getTeamIdFromToken(); if (is_null($teamId)) { @@ -833,6 +834,7 @@ public function create_backup(Request $request) 'database_backup_retention_days_s3' => 'integer|min:0', 'database_backup_retention_max_storage_s3' => 'numeric|min:0', 'timeout' => 'integer|min:60|max:36000', + 'missing_backup_notification_days' => 'integer|min:0|max:365', ]); if ($validator->fails()) { @@ -1025,6 +1027,7 @@ public function create_backup(Request $request) 'database_backup_retention_days_s3' => ['type' => 'integer', 'description' => 'Retention days of the backup in s3'], 'database_backup_retention_max_storage_s3' => ['type' => 'number', 'description' => 'Max storage of the backup in S3'], 'timeout' => ['type' => 'integer', 'description' => 'Backup job timeout in seconds (min: 60, max: 36000)', 'default' => 3600], + 'missing_backup_notification_days' => ['type' => 'integer', 'description' => 'Alert after this many days without an execution; 0 disables alerts', 'minimum' => 0, 'maximum' => 365], ], ), ) @@ -1054,7 +1057,7 @@ public function create_backup(Request $request) )] public function update_backup(Request $request) { - $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout']; + $backupConfigFields = ['save_s3', 'enabled', 'dump_all', 'frequency', 'databases_to_backup', 'database_backup_retention_amount_locally', 'database_backup_retention_days_locally', 'database_backup_retention_max_storage_locally', 'database_backup_retention_amount_s3', 'database_backup_retention_days_s3', 'database_backup_retention_max_storage_s3', 's3_storage_uuid', 'timeout', 'missing_backup_notification_days']; $teamId = getTeamIdFromToken(); if (is_null($teamId)) { @@ -1080,6 +1083,7 @@ public function update_backup(Request $request) 'database_backup_retention_days_s3' => 'integer|min:0', 'database_backup_retention_max_storage_s3' => 'numeric|min:0', 'timeout' => 'integer|min:60|max:36000', + 'missing_backup_notification_days' => 'integer|min:0|max:365', ]); if ($validator->fails()) { return response()->json([ @@ -4885,6 +4889,8 @@ function () use ($database) { 'id', 'created_at', 'updated_at', + 'last_execution_at', + 'missing_backup_notification_sent_at', ])->fill([ 'uuid' => new_public_id(), 'database_id' => $newDatabase->id, diff --git a/app/Jobs/CheckMissingDatabaseBackupsJob.php b/app/Jobs/CheckMissingDatabaseBackupsJob.php new file mode 100644 index 000000000..06ba79a33 --- /dev/null +++ b/app/Jobs/CheckMissingDatabaseBackupsJob.php @@ -0,0 +1,59 @@ +with(['team', 'database', 'latest_log']) + ->where('enabled', true) + ->where('missing_backup_notification_days', '>', 0) + ->chunkById(100, function ($backups): void { + foreach ($backups as $backup) { + $this->notifyIfMissing($backup); + } + }); + } + + private function notifyIfMissing(ScheduledDatabaseBackup $backup): void + { + $lastExecutionAt = $backup->last_execution_at ?? $backup->latest_log?->created_at; + $lastActivityAt = $lastExecutionAt ?? $backup->created_at; + + if (! $lastActivityAt || $lastActivityAt->isAfter(now()->subDays($backup->missing_backup_notification_days))) { + return; + } + + if ($backup->missing_backup_notification_sent_at?->greaterThanOrEqualTo($lastActivityAt)) { + return; + } + + if (! $backup->team) { + Log::warning("Cannot send missing backup notification for backup {$backup->id}: team not found"); + + return; + } + + if ($backup->team->getEnabledChannels('backup_failure') === []) { + return; + } + + $backup->team->notify(new BackupMissing($backup, $lastExecutionAt)); + $backup->forceFill(['missing_backup_notification_sent_at' => now()])->save(); + } +} diff --git a/app/Livewire/Project/Database/BackupEdit.php b/app/Livewire/Project/Database/BackupEdit.php index 28f7e0255..64f684475 100644 --- a/app/Livewire/Project/Database/BackupEdit.php +++ b/app/Livewire/Project/Database/BackupEdit.php @@ -85,6 +85,9 @@ class BackupEdit extends Component #[Validate(['required', 'int', 'min:60', 'max:36000'])] public int|string $timeout = 3600; + #[Validate(['required', 'integer', 'min:0', 'max:365'])] + public int $missingBackupNotificationDays = 0; + public function getListeners(): array { // Keep "Backup Now" in sync when the database starts/stops without a full page refresh. @@ -152,6 +155,7 @@ private function syncData(bool $toModel = false): void $this->backup->databases_to_backup = $this->databasesToBackup; $this->backup->dump_all = $this->dumpAll; $this->backup->timeout = $this->timeout; + $this->backup->missing_backup_notification_days = $this->missingBackupNotificationDays; $this->customValidate(); $this->backup->save(); } else { @@ -170,6 +174,7 @@ private function syncData(bool $toModel = false): void $this->databasesToBackup = $this->backup->databases_to_backup; $this->dumpAll = $this->backup->dump_all; $this->timeout = $this->backup->timeout; + $this->missingBackupNotificationDays = $this->backup->missing_backup_notification_days; } } diff --git a/app/Models/ScheduledDatabaseBackup.php b/app/Models/ScheduledDatabaseBackup.php index e41c793c8..7a26658e4 100644 --- a/app/Models/ScheduledDatabaseBackup.php +++ b/app/Models/ScheduledDatabaseBackup.php @@ -14,6 +14,9 @@ protected function casts(): array 'dump_all' => 'boolean', 'database_backup_retention_max_storage_locally' => 'float', 'database_backup_retention_max_storage_s3' => 'float', + 'missing_backup_notification_days' => 'integer', + 'missing_backup_notification_sent_at' => 'datetime', + 'last_execution_at' => 'datetime', ]; } @@ -37,6 +40,7 @@ protected function casts(): array 'database_backup_retention_max_storage_s3', 'timeout', 'disable_local_backup', + 'missing_backup_notification_days', ]; public static function ownedByCurrentTeam() diff --git a/app/Models/ScheduledDatabaseBackupExecution.php b/app/Models/ScheduledDatabaseBackupExecution.php index 8c5de1e8b..1a479772c 100644 --- a/app/Models/ScheduledDatabaseBackupExecution.php +++ b/app/Models/ScheduledDatabaseBackupExecution.php @@ -6,6 +6,13 @@ class ScheduledDatabaseBackupExecution extends BaseModel { + protected static function booted(): void + { + static::created(function (ScheduledDatabaseBackupExecution $execution): void { + $execution->scheduledDatabaseBackup()->update(['last_execution_at' => $execution->created_at ?? now()]); + }); + } + protected $fillable = [ 'uuid', 'scheduled_database_backup_id', diff --git a/app/Notifications/Channels/TelegramChannel.php b/app/Notifications/Channels/TelegramChannel.php index a52feda08..118ad4269 100644 --- a/app/Notifications/Channels/TelegramChannel.php +++ b/app/Notifications/Channels/TelegramChannel.php @@ -9,6 +9,7 @@ use App\Notifications\Application\StatusChanged; use App\Notifications\Container\ContainerRestarted; use App\Notifications\Database\BackupFailed; +use App\Notifications\Database\BackupMissing; use App\Notifications\Database\BackupSuccess; use App\Notifications\ScheduledTask\TaskFailed; use App\Notifications\ScheduledTask\TaskSuccess; @@ -40,7 +41,8 @@ public function send($notifiable, $notification): void RestartLimitReached::class => $settings->telegram_notifications_restart_limit_reached_thread_id, BackupSuccess::class => $settings->telegram_notifications_backup_success_thread_id, - BackupFailed::class => $settings->telegram_notifications_backup_failure_thread_id, + BackupFailed::class, + BackupMissing::class => $settings->telegram_notifications_backup_failure_thread_id, TaskSuccess::class => $settings->telegram_notifications_scheduled_task_success_thread_id, TaskFailed::class => $settings->telegram_notifications_scheduled_task_failure_thread_id, diff --git a/app/Notifications/Database/BackupMissing.php b/app/Notifications/Database/BackupMissing.php new file mode 100644 index 000000000..d7f127ce3 --- /dev/null +++ b/app/Notifications/Database/BackupMissing.php @@ -0,0 +1,83 @@ +onQueue('high'); + $this->databaseName = $backup->database?->name ?? $backup->description ?? $backup->uuid; + } + + public function via(object $notifiable): array + { + return $notifiable->getEnabledChannels('backup_failure'); + } + + public function toMail(): MailMessage + { + return (new MailMessage) + ->subject("Coolify: [ACTION REQUIRED] No recent backup for {$this->databaseName}") + ->view('emails.backup-missing', $this->messageData()); + } + + public function toDiscord(): DiscordMessage + { + return new DiscordMessage( + title: ':warning: Scheduled database backup missing', + description: $this->description(), + color: DiscordMessage::errorColor(), + isCritical: true, + ); + } + + public function toTelegram(): array + { + return ['message' => 'Coolify: '.$this->description()]; + } + + public function toPushover(): PushoverMessage + { + return new PushoverMessage(title: 'Scheduled database backup missing', level: 'error', message: $this->description()); + } + + public function toSlack(): SlackMessage + { + return new SlackMessage(title: 'Scheduled database backup missing', description: $this->description(), color: SlackMessage::errorColor()); + } + + public function toWebhook(): array + { + return array_merge($this->messageData(), [ + 'success' => false, + 'message' => 'Scheduled database backup missing', + 'event' => 'backup_missing', + 'backup_uuid' => $this->backup->uuid, + ]); + } + + private function description(): string + { + return "The enabled backup schedule for {$this->databaseName} has produced no executions in the last {$this->backup->missing_backup_notification_days} day(s)."; + } + + private function messageData(): array + { + return [ + 'database_name' => $this->databaseName, + 'days' => $this->backup->missing_backup_notification_days, + 'last_execution_at' => $this->lastExecutionAt?->toDateTimeString(), + ]; + } +} diff --git a/database/migrations/2026_08_20_150000_add_missing_backup_notification_fields_to_scheduled_database_backups_table.php b/database/migrations/2026_08_20_150000_add_missing_backup_notification_fields_to_scheduled_database_backups_table.php new file mode 100644 index 000000000..fe7c1e629 --- /dev/null +++ b/database/migrations/2026_08_20_150000_add_missing_backup_notification_fields_to_scheduled_database_backups_table.php @@ -0,0 +1,28 @@ +unsignedInteger('missing_backup_notification_days')->default(0); + $table->timestamp('missing_backup_notification_sent_at')->nullable(); + $table->timestamp('last_execution_at')->nullable(); + }); + } + + public function down(): void + { + Schema::table('scheduled_database_backups', function (Blueprint $table) { + $table->dropColumn([ + 'missing_backup_notification_days', + 'missing_backup_notification_sent_at', + 'last_execution_at', + ]); + }); + } +}; diff --git a/resources/views/emails/backup-missing.blade.php b/resources/views/emails/backup-missing.blade.php new file mode 100644 index 000000000..69728c26f --- /dev/null +++ b/resources/views/emails/backup-missing.blade.php @@ -0,0 +1,7 @@ +The enabled backup schedule for {{ $database_name }} has produced no executions in the last {{ $days }} day(s). + +@if ($last_execution_at) +The last execution was at {{ $last_execution_at }}. +@else +This schedule has never produced an execution. +@endif diff --git a/resources/views/livewire/project/database/backup-edit/general.blade.php b/resources/views/livewire/project/database/backup-edit/general.blade.php index caabf249a..5dddec8ed 100644 --- a/resources/views/livewire/project/database/backup-edit/general.blade.php +++ b/resources/views/livewire/project/database/backup-edit/general.blade.php @@ -91,6 +91,9 @@ class="chip-remove" required /> + diff --git a/tests/Feature/Api/LifecycleApisTest.php b/tests/Feature/Api/LifecycleApisTest.php index 4973453e9..49d6f6635 100644 --- a/tests/Feature/Api/LifecycleApisTest.php +++ b/tests/Feature/Api/LifecycleApisTest.php @@ -138,6 +138,16 @@ 'destination_id' => $this->destination->id, 'destination_type' => $this->destination->getMorphClass(), ]); + $backup = $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'enabled' => true, + 'frequency' => '0 0 * * *', + 'save_s3' => false, + ]); + $backup->forceFill([ + 'last_execution_at' => now()->subDay(), + 'missing_backup_notification_sent_at' => now(), + ])->save(); $response = $this->withHeaders($this->headers) ->postJson("/api/v1/databases/{$database->uuid}/clone", [ @@ -149,11 +159,14 @@ ->assertJsonPath('message', 'Database cloned.'); $cloned = StandalonePostgresql::where('uuid', $response->json('uuid'))->first(); + $clonedBackup = $cloned->scheduledBackups()->sole(); expect($cloned)->not->toBeNull() ->and($cloned->name)->toBe('cloned-db') ->and($cloned->environment_id)->toBe($database->environment_id) ->and($cloned->destination_id)->toBe($this->destination->id) - ->and(str($cloned->status)->startsWith('exited'))->toBeTrue(); + ->and(str($cloned->status)->startsWith('exited'))->toBeTrue() + ->and($clonedBackup->last_execution_at)->toBeNull() + ->and($clonedBackup->missing_backup_notification_sent_at)->toBeNull(); }); test('creates renamed volumes when cloning a database with clone_volumes', function () { diff --git a/tests/Feature/MissingDatabaseBackupNotificationTest.php b/tests/Feature/MissingDatabaseBackupNotificationTest.php new file mode 100644 index 000000000..f4f2efd19 --- /dev/null +++ b/tests/Feature/MissingDatabaseBackupNotificationTest.php @@ -0,0 +1,126 @@ + Carbon::setTestNow()); + +function missingBackupSchedule(Team $team, array $attributes = []): ScheduledDatabaseBackup +{ + $backup = ScheduledDatabaseBackup::create(array_merge([ + 'enabled' => true, + 'frequency' => '0 0 * * *', + 'save_s3' => false, + 'database_type' => 'App\\Models\\StandalonePostgresql', + 'database_id' => 999, + 'team_id' => $team->id, + 'missing_backup_notification_days' => 2, + ], $attributes)); + + ScheduledDatabaseBackup::whereKey($backup->id)->update(['created_at' => now()->subDays(3)]); + + return $backup->refresh(); +} + +function teamWithBackupFailureNotifications(): Team +{ + $team = Team::create(['name' => 'Test team']); + $team->emailNotificationSettings->update([ + 'smtp_enabled' => true, + 'backup_failure_email_notifications' => true, + ]); + + return $team; +} + +it('notifies the team when an enabled backup has no executions for the configured days', function () { + Notification::fake(); + $team = teamWithBackupFailureNotifications(); + $backup = missingBackupSchedule($team); + + (new CheckMissingDatabaseBackupsJob)->handle(); + + Notification::assertSentTo($team, BackupMissing::class, fn (BackupMissing $notification) => $notification->backup->is($backup)); + expect($backup->fresh()->missing_backup_notification_sent_at)->not->toBeNull(); +}); + +it('does not notify for recent disabled or unconfigured backup schedules', function () { + Notification::fake(); + $team = teamWithBackupFailureNotifications(); + missingBackupSchedule($team, ['enabled' => false]); + missingBackupSchedule($team, ['missing_backup_notification_days' => 0]); + $recent = missingBackupSchedule($team); + ScheduledDatabaseBackup::whereKey($recent->id)->update(['created_at' => now()->subDay()]); + + (new CheckMissingDatabaseBackupsJob)->handle(); + + Notification::assertNothingSent(); +}); + +it('notifies once per period without executions and rearms after another execution', function () { + Notification::fake(); + $team = teamWithBackupFailureNotifications(); + $backup = missingBackupSchedule($team); + + (new CheckMissingDatabaseBackupsJob)->handle(); + (new CheckMissingDatabaseBackupsJob)->handle(); + + Notification::assertSentToTimes($team, BackupMissing::class, 1); + + Carbon::setTestNow(now()->addMinute()); + $execution = ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 'database_name' => 'app', + ]); + Carbon::setTestNow(now()->addDays(3)); + + (new CheckMissingDatabaseBackupsJob)->handle(); + + Notification::assertSentToTimes($team, BackupMissing::class, 2); +}); + +it('preserves the last execution checkpoint when execution history is deleted', function () { + Notification::fake(); + $team = teamWithBackupFailureNotifications(); + $backup = missingBackupSchedule($team); + + (new CheckMissingDatabaseBackupsJob)->handle(); + Carbon::setTestNow(now()->addMinute()); + $execution = ScheduledDatabaseBackupExecution::create([ + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 'database_name' => 'app', + ]); + $execution->delete(); + Carbon::setTestNow(now()->addDays(3)); + + (new CheckMissingDatabaseBackupsJob)->handle(); + + Notification::assertSentToTimes($team, BackupMissing::class, 2); +}); + +it('waits to mark an incident sent until a notification channel is enabled', function () { + Notification::fake(); + $team = Team::create(['name' => 'Test team']); + $backup = missingBackupSchedule($team); + + (new CheckMissingDatabaseBackupsJob)->handle(); + expect($backup->fresh()->missing_backup_notification_sent_at)->toBeNull(); + + $team->emailNotificationSettings->update([ + 'smtp_enabled' => true, + 'backup_failure_email_notifications' => true, + ]); + (new CheckMissingDatabaseBackupsJob)->handle(); + + Notification::assertSentTo($team, BackupMissing::class); +});