diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php
index 9734d16af..3cc372fd1 100644
--- a/app/Jobs/DatabaseBackupJob.php
+++ b/app/Jobs/DatabaseBackupJob.php
@@ -15,6 +15,7 @@
use App\Models\Team;
use App\Notifications\Database\BackupFailed;
use App\Notifications\Database\BackupSuccess;
+use App\Notifications\Database\BackupSuccessWithS3Warning;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
@@ -74,6 +75,7 @@ public function __construct(public ScheduledDatabaseBackup $backup)
{
$this->onQueue('high');
$this->timeout = $backup->timeout;
+ $this->backup_log_uuid = (string) new Cuid2;
}
public function handle(): void
@@ -298,6 +300,10 @@ public function handle(): void
} while ($exists);
$size = 0;
+ $localBackupSucceeded = false;
+ $s3UploadError = null;
+
+ // Step 1: Create local backup
try {
if (str($databaseType)->contains('postgres')) {
$this->backup_file = "/pg-dump-$database-".Carbon::now()->timestamp.'.dmp';
@@ -310,6 +316,7 @@ public function handle(): void
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
+ 'local_storage_deleted' => false,
]);
$this->backup_standalone_postgresql($database);
} elseif (str($databaseType)->contains('mongo')) {
@@ -330,6 +337,7 @@ public function handle(): void
'database_name' => $databaseName,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
+ 'local_storage_deleted' => false,
]);
$this->backup_standalone_mongodb($database);
} elseif (str($databaseType)->contains('mysql')) {
@@ -343,6 +351,7 @@ public function handle(): void
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
+ 'local_storage_deleted' => false,
]);
$this->backup_standalone_mysql($database);
} elseif (str($databaseType)->contains('mariadb')) {
@@ -356,56 +365,77 @@ public function handle(): void
'database_name' => $database,
'filename' => $this->backup_location,
'scheduled_database_backup_id' => $this->backup->id,
+ 'local_storage_deleted' => false,
]);
$this->backup_standalone_mariadb($database);
} else {
throw new \Exception('Unsupported database type');
}
+
$size = $this->calculate_size();
- if ($this->backup->save_s3) {
+
+ // Verify local backup succeeded
+ if ($size > 0) {
+ $localBackupSucceeded = true;
+ } else {
+ throw new \Exception('Local backup file is empty or was not created');
+ }
+ } catch (\Throwable $e) {
+ // Local backup failed
+ if ($this->backup_log) {
+ $this->backup_log->update([
+ 'status' => 'failed',
+ 'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
+ 'size' => $size,
+ 'filename' => null,
+ 's3_uploaded' => null,
+ ]);
+ }
+ $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
+
+ continue;
+ }
+
+ // Step 2: Upload to S3 if enabled (independent of local backup)
+ $localStorageDeleted = false;
+ if ($this->backup->save_s3 && $localBackupSucceeded) {
+ try {
$this->upload_to_s3();
// If local backup is disabled, delete the local file immediately after S3 upload
if ($this->backup->disable_local_backup) {
deleteBackupsLocally($this->backup_location, $this->server);
+ $localStorageDeleted = true;
}
+ } catch (\Throwable $e) {
+ // S3 upload failed but local backup succeeded
+ $s3UploadError = $e->getMessage();
}
+ }
- $this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
+ // Step 3: Update status and send notifications based on results
+ if ($localBackupSucceeded) {
+ $message = $this->backup_output;
+
+ if ($s3UploadError) {
+ $message = $message
+ ? $message."\n\nWarning: S3 upload failed: ".$s3UploadError
+ : 'Warning: S3 upload failed: '.$s3UploadError;
+ }
$this->backup_log->update([
'status' => 'success',
- 'message' => $this->backup_output,
+ 'message' => $message,
'size' => $size,
+ 's3_uploaded' => $this->backup->save_s3 ? $this->s3_uploaded : null,
+ 'local_storage_deleted' => $localStorageDeleted,
]);
- } catch (\Throwable $e) {
- // Check if backup actually failed or if it's just a post-backup issue
- $actualBackupFailed = ! $this->s3_uploaded && $this->backup->save_s3;
- if ($actualBackupFailed || $size === 0) {
- // Real backup failure
- if ($this->backup_log) {
- $this->backup_log->update([
- 'status' => 'failed',
- 'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
- 'size' => $size,
- 'filename' => null,
- ]);
- }
- $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
+ // Send appropriate notification
+ if ($s3UploadError) {
+ $this->team->notify(new BackupSuccessWithS3Warning($this->backup, $this->database, $database, $s3UploadError));
} else {
- // Backup succeeded but post-processing failed (cleanup, notification, etc.)
- if ($this->backup_log) {
- $this->backup_log->update([
- 'status' => 'success',
- 'message' => $this->backup_output ? $this->backup_output."\nWarning: Post-backup cleanup encountered an issue: ".$e->getMessage() : 'Warning: '.$e->getMessage(),
- 'size' => $size,
- ]);
- }
- // Send success notification since the backup itself succeeded
$this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
- // Log the post-backup issue
- ray('Post-backup operation failed but backup was successful: '.$e->getMessage());
}
}
}
diff --git a/app/Livewire/Project/Database/BackupEdit.php b/app/Livewire/Project/Database/BackupEdit.php
index 98d076ac0..b3df79008 100644
--- a/app/Livewire/Project/Database/BackupEdit.php
+++ b/app/Livewire/Project/Database/BackupEdit.php
@@ -208,7 +208,7 @@ private function customValidate()
// Validate that disable_local_backup can only be true when S3 backup is enabled
if ($this->backup->disable_local_backup && ! $this->backup->save_s3) {
- throw new \Exception('Local backup can only be disabled when S3 backup is enabled.');
+ $this->backup->disable_local_backup = $this->disableLocalBackup = false;
}
$isValid = validate_cron_expression($this->backup->frequency);
diff --git a/app/Livewire/Project/Database/BackupExecutions.php b/app/Livewire/Project/Database/BackupExecutions.php
index 2f3aae8cf..0b6d8338b 100644
--- a/app/Livewire/Project/Database/BackupExecutions.php
+++ b/app/Livewire/Project/Database/BackupExecutions.php
@@ -202,11 +202,6 @@ public function server()
public function render()
{
- return view('livewire.project.database.backup-executions', [
- 'checkboxes' => [
- ['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently from S3 Storage'],
- // ['id' => 'delete_backup_sftp', 'label' => 'Delete the selected backup permanently from SFTP Storage'],
- ],
- ]);
+ return view('livewire.project.database.backup-executions');
}
}
diff --git a/app/Models/ScheduledDatabaseBackupExecution.php b/app/Models/ScheduledDatabaseBackupExecution.php
index b06dd5b45..c0298ecc8 100644
--- a/app/Models/ScheduledDatabaseBackupExecution.php
+++ b/app/Models/ScheduledDatabaseBackupExecution.php
@@ -8,6 +8,15 @@ class ScheduledDatabaseBackupExecution extends BaseModel
{
protected $guarded = [];
+ protected function casts(): array
+ {
+ return [
+ 's3_uploaded' => 'boolean',
+ 'local_storage_deleted' => 'boolean',
+ 's3_storage_deleted' => 'boolean',
+ ];
+ }
+
public function scheduledDatabaseBackup(): BelongsTo
{
return $this->belongsTo(ScheduledDatabaseBackup::class);
diff --git a/app/Notifications/Database/BackupSuccessWithS3Warning.php b/app/Notifications/Database/BackupSuccessWithS3Warning.php
new file mode 100644
index 000000000..75ae2824c
--- /dev/null
+++ b/app/Notifications/Database/BackupSuccessWithS3Warning.php
@@ -0,0 +1,116 @@
+onQueue('high');
+
+ $this->name = $database->name;
+ $this->frequency = $backup->frequency;
+
+ if ($backup->s3) {
+ $this->s3_storage_url = base_url().'/storages/'.$backup->s3->uuid;
+ }
+ }
+
+ public function via(object $notifiable): array
+ {
+ return $notifiable->getEnabledChannels('backup_failure');
+ }
+
+ public function toMail(): MailMessage
+ {
+ $mail = new MailMessage;
+ $mail->subject("Coolify: Backup succeeded locally but S3 upload failed for {$this->database->name}");
+ $mail->view('emails.backup-success-with-s3-warning', [
+ 'name' => $this->name,
+ 'database_name' => $this->database_name,
+ 'frequency' => $this->frequency,
+ 's3_error' => $this->s3_error,
+ 's3_storage_url' => $this->s3_storage_url,
+ ]);
+
+ return $mail;
+ }
+
+ public function toDiscord(): DiscordMessage
+ {
+ $message = new DiscordMessage(
+ title: ':warning: Database backup succeeded locally, S3 upload failed',
+ description: "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.",
+ color: DiscordMessage::warningColor(),
+ );
+
+ $message->addField('Frequency', $this->frequency, true);
+ $message->addField('S3 Error', $this->s3_error);
+
+ if ($this->s3_storage_url) {
+ $message->addField('S3 Storage', '[Check Configuration]('.$this->s3_storage_url.')');
+ }
+
+ return $message;
+ }
+
+ public function toTelegram(): array
+ {
+ $message = "Coolify: Database backup for {$this->name} (db:{$this->database_name}) with frequency of {$this->frequency} succeeded locally but failed to upload to S3.\n\nS3 Error:\n{$this->s3_error}";
+
+ if ($this->s3_storage_url) {
+ $message .= "\n\nCheck S3 Configuration: {$this->s3_storage_url}";
+ }
+
+ return [
+ 'message' => $message,
+ ];
+ }
+
+ public function toPushover(): PushoverMessage
+ {
+ $message = "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.
Frequency: {$this->frequency}.
S3 Error: {$this->s3_error}";
+
+ if ($this->s3_storage_url) {
+ $message .= "
s3_storage_url}\">Check S3 Configuration";
+ }
+
+ return new PushoverMessage(
+ title: 'Database backup succeeded locally, S3 upload failed',
+ level: 'warning',
+ message: $message,
+ );
+ }
+
+ public function toSlack(): SlackMessage
+ {
+ $title = 'Database backup succeeded locally, S3 upload failed';
+ $description = "Database backup for {$this->name} (db:{$this->database_name}) was created successfully on local storage, but failed to upload to S3.";
+
+ $description .= "\n\n*Frequency:* {$this->frequency}";
+ $description .= "\n\n*S3 Error:* {$this->s3_error}";
+
+ if ($this->s3_storage_url) {
+ $description .= "\n\n*S3 Storage:* <{$this->s3_storage_url}|Check Configuration>";
+ }
+
+ return new SlackMessage(
+ title: $title,
+ description: $description,
+ color: SlackMessage::warningColor()
+ );
+ }
+}
diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php
index 5dbd46b5e..aa7be3236 100644
--- a/bootstrap/helpers/databases.php
+++ b/bootstrap/helpers/databases.php
@@ -237,12 +237,11 @@ function removeOldBackups($backup): void
{
try {
if ($backup->executions) {
- // If local backup is disabled, mark all executions as having local storage deleted
- if ($backup->disable_local_backup && $backup->save_s3) {
- $backup->executions()
- ->where('local_storage_deleted', false)
- ->update(['local_storage_deleted' => true]);
- } else {
+ // Delete old local backups (only if local backup is NOT disabled)
+ // Note: When disable_local_backup is enabled, each execution already marks its own
+ // local_storage_deleted status at the time of backup, so we don't need to retroactively
+ // update old executions
+ if (! $backup->disable_local_backup) {
$localBackupsToDelete = deleteOldBackupsLocally($backup);
if ($localBackupsToDelete->isNotEmpty()) {
$backup->executions()
@@ -261,18 +260,18 @@ function removeOldBackups($backup): void
}
}
- // Delete executions where both local and S3 storage are marked as deleted
- // or where only S3 is enabled and S3 storage is deleted
- if ($backup->disable_local_backup && $backup->save_s3) {
- $backup->executions()
- ->where('s3_storage_deleted', true)
- ->delete();
- } else {
- $backup->executions()
- ->where('local_storage_deleted', true)
- ->where('s3_storage_deleted', true)
- ->delete();
- }
+ // Delete execution records where all backup copies are gone
+ // Case 1: Both local and S3 backups are deleted
+ $backup->executions()
+ ->where('local_storage_deleted', true)
+ ->where('s3_storage_deleted', true)
+ ->delete();
+
+ // Case 2: Local backup is deleted and S3 was never used (s3_uploaded is null)
+ $backup->executions()
+ ->where('local_storage_deleted', true)
+ ->whereNull('s3_uploaded')
+ ->delete();
} catch (\Exception $e) {
throw $e;
diff --git a/database/migrations/2025_10_07_120723_add_s3_uploaded_to_scheduled_database_backup_executions_table.php b/database/migrations/2025_10_07_120723_add_s3_uploaded_to_scheduled_database_backup_executions_table.php
new file mode 100644
index 000000000..d80f2621b
--- /dev/null
+++ b/database/migrations/2025_10_07_120723_add_s3_uploaded_to_scheduled_database_backup_executions_table.php
@@ -0,0 +1,28 @@
+boolean('s3_uploaded')->nullable()->after('filename');
+ });
+ }
+
+ /**
+ * Reverse the migrations.
+ */
+ public function down(): void
+ {
+ Schema::table('scheduled_database_backup_executions', function (Blueprint $table) {
+ $table->dropColumn('s3_uploaded');
+ });
+ }
+};
diff --git a/resources/views/emails/backup-success-with-s3-warning.blade.php b/resources/views/emails/backup-success-with-s3-warning.blade.php
new file mode 100644
index 000000000..5d2f25851
--- /dev/null
+++ b/resources/views/emails/backup-success-with-s3-warning.blade.php
@@ -0,0 +1,9 @@
+
+Database backup for {{ $name }} @if($database_name)(db:{{ $database_name }})@endif with frequency of {{ $frequency }} succeeded locally but failed to upload to S3.
+
+S3 Error: {{ $s3_error }}
+
+@if($s3_storage_url)
+Check S3 Configuration: {{ $s3_storage_url }}
+@endif
+
diff --git a/resources/views/livewire/project/database/backup-executions.blade.php b/resources/views/livewire/project/database/backup-executions.blade.php
index 94245643a..30eef5976 100644
--- a/resources/views/livewire/project/database/backup-executions.blade.php
+++ b/resources/views/livewire/project/database/backup-executions.blade.php
@@ -51,12 +51,14 @@ class="flex flex-col gap-4">
data_get($execution, 'status') === 'running',
'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200 dark:shadow-red-900/5' =>
data_get($execution, 'status') === 'failed',
+ 'bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-200 dark:shadow-amber-900/5' =>
+ data_get($execution, 'status') === 'success' && data_get($execution, 's3_uploaded') === false,
'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200 dark:shadow-green-900/5' =>
- data_get($execution, 'status') === 'success',
+ data_get($execution, 'status') === 'success' && data_get($execution, 's3_uploaded') !== false,
])>
@php
$statusText = match (data_get($execution, 'status')) {
- 'success' => 'Success',
+ 'success' => data_get($execution, 's3_uploaded') === false ? 'Success (S3 Warning)' : 'Success',
'running' => 'In Progress',
'failed' => 'Failed',
default => ucfirst(data_get($execution, 'status')),
@@ -120,20 +122,15 @@ class="flex flex-col gap-4">
Local Storage
- @if ($backup->save_s3)
+ @if (data_get($execution, 's3_uploaded') !== null)
!data_get(
- $execution,
- 's3_storage_deleted',
- false),
- 'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => data_get(
- $execution,
- 's3_storage_deleted',
- false),
+ 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200' => data_get($execution, 's3_uploaded') === false && !data_get($execution, 's3_storage_deleted', false),
+ 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => data_get($execution, 's3_uploaded') === true && !data_get($execution, 's3_storage_deleted', false),
+ 'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => data_get($execution, 's3_storage_deleted', false),
])>
- @if (!data_get($execution, 's3_storage_deleted', false))
+ @if (data_get($execution, 's3_uploaded') === true && !data_get($execution, 's3_storage_deleted', false))