rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(backup): enhance backup job with S3 upload handling and notifications

Browse source 
- Introduced a new notification class, BackupSuccessWithS3Warning, to alert users when local backups succeed but S3 uploads fail.
- Updated DatabaseBackupJob to track local backup success and handle S3 upload errors, improving error reporting and user notifications.
- Modified ScheduledDatabaseBackupExecution model to include a new s3_uploaded boolean field for tracking S3 upload status.
- Adjusted views and validation logic to reflect changes in backup execution status and S3 handling.
- Added tests to ensure the new s3_uploaded column is correctly implemented and validated.
This commit is contained in:
Andras Bacsai 2025-10-07 15:02:23 +02:00
parent bc8cf8ed84
commit 2c64136503
10 changed files with 303 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

86
app/Jobs/DatabaseBackupJob.php
View file

@ -15,6 +15,7 @@
use App\Models\Team;
use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupSuccess;
use App\Notifications\Database\BackupSuccessWithS3Warning;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@ -74,6 +75,7 @@ public function __construct(public ScheduledDatabaseBackup $backup)
{
$this->onQueue('high');
$this->timeout = $backup->timeout;
$this->backup_log_uuid = (string) new Cuid2;
}
public function handle(): void
@ -298,6 +300,10 @@ public function handle(): void
} while ($exists);
$size = 0;
$localBackupSucceeded = false;
$s3UploadError = null;
// Step 1: Create local backup
try {
if (str($databaseType)->contains('postgres')) {
$this->backup_file = "/pg-dump-$database-".Carbon::now()->timestamp.'.dmp';
@ -310,6 +316,7 @@ public function handle(): void
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'local_storage_deleted' => false,
]);
$this->backup_standalone_postgresql($database);
} elseif (str($databaseType)->contains('mongo')) {
@ -330,6 +337,7 @@ public function handle(): void
'database_name' => $databaseName,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'local_storage_deleted' => false,
]);
$this->backup_standalone_mongodb($database);
} elseif (str($databaseType)->contains('mysql')) {
@ -343,6 +351,7 @@ public function handle(): void
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'local_storage_deleted' => false,
]);
$this->backup_standalone_mysql($database);
} elseif (str($databaseType)->contains('mariadb')) {
@ -356,56 +365,77 @@ public function handle(): void
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
'local_storage_deleted' => false,
]);
$this->backup_standalone_mariadb($database);
} else {
throw new \Exception('Unsupported database type');
}
$size = $this->calculate_size();
if ($this->backup->save_s3) {
// Verify local backup succeeded
if ($size > 0) {
$localBackupSucceeded = true;
} else {
throw new \Exception('Local backup file is empty or was not created');
}
} catch (\Throwable $e) {
// Local backup failed
if ($this->backup_log) {
$this->backup_log->update([
'status' => 'failed',
'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
'size' => $size,
'filename' => null,
's3_uploaded' => null,
]);
}
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
continue;
}
// Step 2: Upload to S3 if enabled (independent of local backup)
$localStorageDeleted = false;
if ($this->backup->save_s3 && $localBackupSucceeded) {
try {
$this->upload_to_s3();
// If local backup is disabled, delete the local file immediately after S3 upload
if ($this->backup->disable_local_backup) {
deleteBackupsLocally($this->backup_location, $this->server);
$localStorageDeleted = true;
}
} catch (\Throwable $e) {
// S3 upload failed but local backup succeeded
$s3UploadError = $e->getMessage();
}
}
$this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
// Step 3: Update status and send notifications based on results
if ($localBackupSucceeded) {
$message = $this->backup_output;
if ($s3UploadError) {
$message = $message
? $message."\n\nWarning: S3 upload failed: ".$s3UploadError
: 'Warning: S3 upload failed: '.$s3UploadError;
}
$this->backup_log->update([
'status' => 'success',
'message' => $this->backup_output,
'message' => $message,
'size' => $size,
's3_uploaded' => $this->backup->save_s3 ? $this->s3_uploaded : null,
'local_storage_deleted' => $localStorageDeleted,
]);
} catch (\Throwable $e) {
// Check if backup actually failed or if it's just a post-backup issue
$actualBackupFailed = ! $this->s3_uploaded && $this->backup->save_s3;
if ($actualBackupFailed || $size === 0) {
// Real backup failure
if ($this->backup_log) {
$this->backup_log->update([
'status' => 'failed',
'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
'size' => $size,
'filename' => null,
]);
}
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
// Send appropriate notification
if ($s3UploadError) {
$this->team->notify(new BackupSuccessWithS3Warning($this->backup, $this->database, $database, $s3UploadError));
} else {
// Backup succeeded but post-processing failed (cleanup, notification, etc.)
if ($this->backup_log) {
$this->backup_log->update([
'status' => 'success',
'message' => $this->backup_output ? $this->backup_output."\nWarning: Post-backup cleanup encountered an issue: ".$e->getMessage() : 'Warning: '.$e->getMessage(),
'size' => $size,
]);
}
// Send success notification since the backup itself succeeded
$this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
// Log the post-backup issue
ray('Post-backup operation failed but backup was successful: '.$e->getMessage());
}
}
}

2
app/Livewire/Project/Database/BackupEdit.php
View file

@ -208,7 +208,7 @@ private function customValidate()
// Validate that disable_local_backup can only be true when S3 backup is enabled
if ($this->backup->disable_local_backup && ! $this->backup->save_s3) {
throw new \Exception('Local backup can only be disabled when S3 backup is enabled.');
$this->backup->disable_local_backup = $this->disableLocalBackup = false;
}
$isValid = validate_cron_expression($this->backup->frequency);

7
app/Livewire/Project/Database/BackupExecutions.php
View file

@ -202,11 +202,6 @@ public function server()
public function render()
{
return view('livewire.project.database.backup-executions', [
'checkboxes' => [
['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently from S3 Storage'],
// ['id' => 'delete_backup_sftp', 'label' => 'Delete the selected backup permanently from SFTP Storage'],
],
]);
return view('livewire.project.database.backup-executions');
}
}

9
app/Models/ScheduledDatabaseBackupExecution.php
View file

@ -8,6 +8,15 @@ class ScheduledDatabaseBackupExecution extends BaseModel
{
protected $guarded = [];
protected function casts(): array
{
return [
's3_uploaded' => 'boolean',
'local_storage_deleted' => 'boolean',
's3_storage_deleted' => 'boolean',
];
}
public function scheduledDatabaseBackup(): BelongsTo
{
return $this->belongsTo(ScheduledDatabaseBackup::class);

116
app/Notifications/Database/BackupSuccessWithS3Warning.php Normal file
View file

@ -0,0 +1,116 @@
<?php
namespace App\Notifications\Database;
use App\Models\ScheduledDatabaseBackup;
use App\Notifications\CustomEmailNotification;
use App\Notifications\Dto\DiscordMessage;
use App\Notifications\Dto\PushoverMessage;
use App\Notifications\Dto\SlackMessage;
use Illuminate\Notifications\Messages\MailMessage;
class BackupSuccessWithS3Warning extends CustomEmailNotification
{
public string $name;
public string $frequency;
public ?string $s3_storage_url = null;
public function __construct(ScheduledDatabaseBackup $backup, public $database, public $database_name, public $s3_error)
{
$this->onQueue('high');
$this->name = $database->name;
$this->frequency = $backup->frequency;
if ($backup->s3) {
$this->s3_storage_url = base_url().'/storages/'.$backup->s3->uuid;
}
}
public function via(object $notifiable): array
{
return $notifiable->getEnabledChannels('backup_failure');
}
public function toMail(): MailMessage
{
$mail = new MailMessage;
$mail->subject("Coolify: Backup succeeded locally but S3 upload failed for {$this->database->name}");
$mail->view('emails.backup-success-with-s3-warning', [
'name' => $this->name,
'database_name' => $this->database_name,
'frequency' => $this->frequency,
's3_error' => $this->s3_error,
's3_storage_url' => $this->s3_storage_url,
]);
return $mail;
}
public function toDiscord(): DiscordMessage
{
$message = new DiscordMessage(
title: ':warning: Database backup succeeded locally, S3 upload failed',
description: "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.",
color: DiscordMessage::warningColor(),
);
$message->addField('Frequency', $this->frequency, true);
$message->addField('S3 Error', $this->s3_error);
if ($this->s3_storage_url) {
$message->addField('S3 Storage', '[Check Configuration]('.$this->s3_storage_url.')');
}
return $message;
}
public function toTelegram(): array
{
$message = "Coolify: Database backup for {$this->name} (db:{$this->database_name}) with frequency of {$this->frequency} succeeded locally but failed to upload to S3.\n\nS3 Error:\n{$this->s3_error}";
if ($this->s3_storage_url) {
$message .= "\n\nCheck S3 Configuration: {$this->s3_storage_url}";
}
return [
'message' => $message,
];
}
public function toPushover(): PushoverMessage
{
$message = "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.<br/><br/><b>Frequency:</b> {$this->frequency}.<br/><b>S3 Error:</b> {$this->s3_error}";
if ($this->s3_storage_url) {
$message .= "<br/><br/><a href=\"{$this->s3_storage_url}\">Check S3 Configuration</a>";
}
return new PushoverMessage(
title: 'Database backup succeeded locally, S3 upload failed',
level: 'warning',
message: $message,
);
}
public function toSlack(): SlackMessage
{
$title = 'Database backup succeeded locally, S3 upload failed';
$description = "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.";
$description .= "\n\n*Frequency:* {$this->frequency}";
$description .= "\n\n*S3 Error:* {$this->s3_error}";
if ($this->s3_storage_url) {
$description .= "\n\n*S3 Storage:* <{$this->s3_storage_url}|Check Configuration>";
}
return new SlackMessage(
title: $title,
description: $description,
color: SlackMessage::warningColor()
);
}
}

35
bootstrap/helpers/databases.php
View file

@ -237,12 +237,11 @@ function removeOldBackups($backup): void
{
try {
if ($backup->executions) {
// If local backup is disabled, mark all executions as having local storage deleted
if ($backup->disable_local_backup && $backup->save_s3) {
$backup->executions()
->where('local_storage_deleted', false)
->update(['local_storage_deleted' => true]);
} else {
// Delete old local backups (only if local backup is NOT disabled)
// Note: When disable_local_backup is enabled, each execution already marks its own
// local_storage_deleted status at the time of backup, so we don't need to retroactively
// update old executions
if (! $backup->disable_local_backup) {
$localBackupsToDelete = deleteOldBackupsLocally($backup);
if ($localBackupsToDelete->isNotEmpty()) {
$backup->executions()
@ -261,18 +260,18 @@ function removeOldBackups($backup): void
}
}
// Delete executions where both local and S3 storage are marked as deleted
// or where only S3 is enabled and S3 storage is deleted
if ($backup->disable_local_backup && $backup->save_s3) {
$backup->executions()
->where('s3_storage_deleted', true)
->delete();
} else {
$backup->executions()
->where('local_storage_deleted', true)
->where('s3_storage_deleted', true)
->delete();
}
// Delete execution records where all backup copies are gone
// Case 1: Both local and S3 backups are deleted
$backup->executions()
->where('local_storage_deleted', true)
->where('s3_storage_deleted', true)
->delete();
// Case 2: Local backup is deleted and S3 was never used (s3_uploaded is null)
$backup->executions()
->where('local_storage_deleted', true)
->whereNull('s3_uploaded')
->delete();
} catch (\Exception $e) {
throw $e;

28
database/migrations/2025_10_07_120723_add_s3_uploaded_to_scheduled_database_backup_executions_table.php Normal file
View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('scheduled_database_backup_executions', function (Blueprint $table) {
$table->boolean('s3_uploaded')->nullable()->after('filename');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('scheduled_database_backup_executions', function (Blueprint $table) {
$table->dropColumn('s3_uploaded');
});
}
};

9
resources/views/emails/backup-success-with-s3-warning.blade.php Normal file
View file

@ -0,0 +1,9 @@
<x-emails.layout>
Database backup for {{ $name }} @if($database_name)(db:{{ $database_name }})@endif with frequency of {{ $frequency }} succeeded locally but failed to upload to S3.
S3 Error: {{ $s3_error }}
@if($s3_storage_url)
Check S3 Configuration: {{ $s3_storage_url }}
@endif
</x-emails.layout>

41
resources/views/livewire/project/database/backup-executions.blade.php
View file

@ -51,12 +51,14 @@ class="flex flex-col gap-4">
data_get($execution, 'status') === 'running',
'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200 dark:shadow-red-900/5' =>
data_get($execution, 'status') === 'failed',
'bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-200 dark:shadow-amber-900/5' =>
data_get($execution, 'status') === 'success' && data_get($execution, 's3_uploaded') === false,
'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200 dark:shadow-green-900/5' =>
data_get($execution, 'status') === 'success',
data_get($execution, 'status') === 'success' && data_get($execution, 's3_uploaded') !== false,
])>
@php
$statusText = match (data_get($execution, 'status')) {
'success' => 'Success',
'success' => data_get($execution, 's3_uploaded') === false ? 'Success (S3 Warning)' : 'Success',
'running' => 'In Progress',
'failed' => 'Failed',
default => ucfirst(data_get($execution, 'status')),
@ -120,20 +122,15 @@ class="flex flex-col gap-4">
Local Storage
</span>
</span>
@if ($backup->save_s3)
@if (data_get($execution, 's3_uploaded') !== null)
<span @class([
'px-2 py-1 rounded-sm text-xs font-medium',
'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => !data_get(
$execution,
's3_storage_deleted',
false),
'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => data_get(
$execution,
's3_storage_deleted',
false),
'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200' => data_get($execution, 's3_uploaded') === false && !data_get($execution, 's3_storage_deleted', false),
'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => data_get($execution, 's3_uploaded') === true && !data_get($execution, 's3_storage_deleted', false),
'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => data_get($execution, 's3_storage_deleted', false),
])>
<span class="flex items-center gap-1">
@if (!data_get($execution, 's3_storage_deleted', false))
@if (data_get($execution, 's3_uploaded') === true && !data_get($execution, 's3_storage_deleted', false))
<svg class="w-3 h-3" fill="currentColor" viewBox="0 0 20 20"
xmlns="http://www.w3.org/2000/svg">
<path fill-rule="evenodd"
@ -163,9 +160,25 @@ class="flex flex-col gap-4">
<x-forms.button class="dark:hover:bg-coolgray-400"
x-on:click="download_file('{{ data_get($execution, 'id') }}')">Download</x-forms.button>
@endif
@php
$executionCheckboxes = [];
$deleteActions = [];
if (!data_get($execution, 'local_storage_deleted', false)) {
$deleteActions[] = 'This backup will be permanently deleted from local storage.';
}
if (data_get($execution, 's3_uploaded') === true && !data_get($execution, 's3_storage_deleted', false)) {
$executionCheckboxes[] = ['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently from S3 Storage'];
}
if (empty($deleteActions)) {
$deleteActions[] = 'This backup execution record will be deleted.';
}
@endphp
<x-modal-confirmation title="Confirm Backup Deletion?" buttonTitle="Delete" isErrorButton
submitAction="deleteBackup({{ data_get($execution, 'id') }})" :checkboxes="$checkboxes"
:actions="['This backup will be permanently deleted from local storage.']" confirmationText="{{ data_get($execution, 'filename') }}"
submitAction="deleteBackup({{ data_get($execution, 'id') }})" :checkboxes="$executionCheckboxes"
:actions="$deleteActions" confirmationText="{{ data_get($execution, 'filename') }}"
confirmationLabel="Please confirm the execution of the actions by entering the Backup Filename below"
shortConfirmationLabel="Backup Filename" 1 />
</div>

37
tests/Feature/DatabaseBackupJobTest.php Normal file
View file

@ -0,0 +1,37 @@
<?php
use App\Models\ScheduledDatabaseBackupExecution;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Schema;
uses(RefreshDatabase::class);
test('scheduled_database_backup_executions table has s3_uploaded column', function () {
expect(Schema::hasColumn('scheduled_database_backup_executions', 's3_uploaded'))->toBeTrue();
});
test('s3_uploaded column is nullable', function () {
$columns = Schema::getColumns('scheduled_database_backup_executions');
$s3UploadedColumn = collect($columns)->firstWhere('name', 's3_uploaded');
expect($s3UploadedColumn)->not->toBeNull();
expect($s3UploadedColumn['nullable'])->toBeTrue();
});
test('scheduled database backup execution model casts s3_uploaded correctly', function () {
$model = new ScheduledDatabaseBackupExecution;
$casts = $model->getCasts();
expect($casts)->toHaveKey('s3_uploaded');
expect($casts['s3_uploaded'])->toBe('boolean');
});
test('scheduled database backup execution model casts storage deletion fields correctly', function () {
$model = new ScheduledDatabaseBackupExecution;
$casts = $model->getCasts();
expect($casts)->toHaveKey('local_storage_deleted');
expect($casts['local_storage_deleted'])->toBe('boolean');
expect($casts)->toHaveKey('s3_storage_deleted');
expect($casts['s3_storage_deleted'])->toBe('boolean');
});