diff --git a/app/Actions/Shared/DeleteScheduledVolumeBackup.php b/app/Actions/Shared/DeleteScheduledVolumeBackup.php new file mode 100644 index 000000000..55972ab52 --- /dev/null +++ b/app/Actions/Shared/DeleteScheduledVolumeBackup.php @@ -0,0 +1,71 @@ +id), $backup->timeout + 300); + + if (! $lock->get()) { + throw new \RuntimeException('Wait for the queued or running storage backup to finish before deleting this schedule.'); + } + + try { + if ($backup->executions() + ->where(fn ($query) => $query + ->where('status', 'running') + ->orWhere('stop_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->exists()) { + throw new \RuntimeException('Wait for the running storage backup and recovery operations to finish before deleting this schedule.'); + } + + $localFilenames = $backup->executions() + ->where('local_storage_deleted', false) + ->pluck('filename') + ->filter() + ->all(); + + if ($localFilenames !== []) { + $server ??= $backup->server(); + if (! $server) { + throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.'); + } + + deleteBackupsLocally($localFilenames, $server, throwError: true); + } + + $s3Executions = $backup->executions() + ->with('s3') + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false) + ->get(); + + foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { + $s3 = $executions->first()->s3; + if (! $s3) { + throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + } + + $filenames = $executions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsS3($filenames, $s3); + } + } + + $backup->delete(); + } finally { + $lock->release(); + } + } +} diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index f25158d34..c80da0cab 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -497,7 +497,7 @@ public function handle(): void } } if ($this->backup_log && $this->backup_log->status === 'success') { - removeOldBackups($this->backup); + $this->removeExpiredBackups(); } } catch (Throwable $e) { throw $e; @@ -513,6 +513,19 @@ public function handle(): void } } + private function removeExpiredBackups(): void + { + try { + removeOldBackups($this->backup); + } catch (Throwable $exception) { + Log::channel('scheduled-errors')->warning('Database backup retention cleanup failed', [ + 'backup_id' => $this->backup->id, + 'execution_id' => $this->backup_log?->id, + 'error' => $exception->getMessage(), + ]); + } + } + private function backup_standalone_mongodb(string $databaseWithCollections): void { try { diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.php index 825604910..656e5c409 100644 --- a/app/Jobs/DeleteResourceJob.php +++ b/app/Jobs/DeleteResourceJob.php @@ -7,7 +7,10 @@ use App\Actions\Server\CleanupDocker; use App\Actions\Service\DeleteService; use App\Actions\Service\StopService; +use App\Actions\Shared\DeleteScheduledVolumeBackup; +use App\Enums\ApplicationDeploymentStatus; use App\Models\Application; +use App\Models\ApplicationDeploymentQueue; use App\Models\ApplicationPreview; use App\Models\Service; use App\Models\StandaloneClickhouse; @@ -42,6 +45,10 @@ public function __construct( public function handle() { + if (! $this->resource instanceof ApplicationPreview) { + $this->deleteScheduledVolumeBackups(); + } + try { // Handle ApplicationPreview instances separately if ($this->resource instanceof ApplicationPreview) { @@ -113,6 +120,24 @@ public function handle() } } + private function deleteScheduledVolumeBackups(): void + { + $server = data_get($this->resource, 'server') ?? data_get($this->resource, 'destination.server'); + $resources = $this->resource instanceof Service + ? $this->resource->applications()->get()->concat($this->resource->databases()->get()) + : collect([$this->resource]); + + foreach ($resources as $resource) { + $storages = $resource->persistentStorages()->get()->concat($resource->fileStorages()->get()); + + foreach ($storages as $storage) { + foreach ($storage->scheduledBackups()->get() as $backup) { + DeleteScheduledVolumeBackup::run($backup, $server); + } + } + } + } + private function deleteApplicationPreview() { $application = $this->resource->application; @@ -125,11 +150,11 @@ private function deleteApplicationPreview() } // Cancel any active deployments for this PR (same logic as API cancel_deployment) - $activeDeployments = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id) + $activeDeployments = ApplicationDeploymentQueue::where('application_id', $application->id) ->where('pull_request_id', $pull_request_id) ->whereIn('status', [ - \App\Enums\ApplicationDeploymentStatus::QUEUED->value, - \App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value, + ApplicationDeploymentStatus::QUEUED->value, + ApplicationDeploymentStatus::IN_PROGRESS->value, ]) ->get(); @@ -137,7 +162,7 @@ private function deleteApplicationPreview() try { // Mark deployment as cancelled $activeDeployment->update([ - 'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value, + 'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value, ]); // Add cancellation log entry diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index 98b5373bf..bb9be3a8f 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -63,7 +63,9 @@ public function handle(): void throw new \RuntimeException('The storage backup resource, team, or server no longer exists.'); } - $this->execution = $this->backup->executions()->create(); + $this->execution = $this->backup->executions()->create([ + 's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null, + ]); BackupCreated::dispatch($team->id); $backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid; @@ -352,6 +354,7 @@ private function removeExpiredBackups(Server $server): void if ($this->backup->save_s3 && $this->backup->s3) { $s3Executions = $this->backup->executions() + ->with('s3') ->where('status', 'success') ->where('s3_uploaded', true) ->where('s3_storage_deleted', false) @@ -363,11 +366,18 @@ private function removeExpiredBackups(Server $server): void $this->backup->retention_max_storage_s3, ); - $filenames = $s3Executions->pluck('filename')->filter()->all(); - if ($filenames !== []) { - deleteBackupsS3($filenames, $this->backup->s3); - $this->backup->executions()->whereKey($s3Executions->pluck('id')->all()) - ->update(['s3_storage_deleted' => true]); + foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { + $s3 = $executions->first()->s3; + if (! $s3) { + throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + } + + $filenames = $executions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsS3($filenames, $s3); + $this->backup->executions()->whereKey($executions->pluck('id')->all()) + ->update(['s3_storage_deleted' => true]); + } } } diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php index 48db9b0a8..a3465b192 100644 --- a/app/Jobs/VolumeBackupRecoveryJob.php +++ b/app/Jobs/VolumeBackupRecoveryJob.php @@ -96,8 +96,8 @@ private static function recoverContainers(ScheduledVolumeBackupExecution $execut public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void { - $execution->loadMissing('scheduledVolumeBackup.s3'); - $s3 = $execution->scheduledVolumeBackup?->s3; + $execution->loadMissing('s3'); + $s3 = $execution->s3; if (! $s3 || blank($execution->filename)) { throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.'); diff --git a/app/Livewire/Project/Database/ScheduledBackups.php b/app/Livewire/Project/Database/ScheduledBackups.php index 77a380ea0..aa7e9f43c 100644 --- a/app/Livewire/Project/Database/ScheduledBackups.php +++ b/app/Livewire/Project/Database/ScheduledBackups.php @@ -3,6 +3,7 @@ namespace App\Livewire\Project\Database; use App\Models\ServiceDatabase; +use Illuminate\Contracts\View\View; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; @@ -63,4 +64,11 @@ public function refreshScheduledBackups(?int $id = null): void $this->database->refresh(); $this->dispatch('refreshScheduledBackups'); } + + public function render(): View + { + $this->database->loadMissing('scheduledBackups.s3'); + + return view('livewire.project.database.scheduled-backups'); + } } diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php index e85036c55..d16a5e2c8 100644 --- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php +++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php @@ -2,6 +2,7 @@ namespace App\Livewire\Project\Shared\Storages; +use App\Actions\Shared\DeleteScheduledVolumeBackup; use App\Jobs\VolumeBackupJob; use App\Models\LocalFileVolume; use App\Models\LocalPersistentVolume; @@ -9,7 +10,6 @@ use App\Models\ScheduledVolumeBackup; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Cache; use Livewire\Component; use Livewire\WithPagination; use Throwable; @@ -214,57 +214,8 @@ public function delete(?string $password = null, array $selectedActions = []) return false; } - $lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300); - - if (! $lock->get()) { - $this->dispatch('error', 'Wait for the queued or running storage backup to finish before deleting this schedule.'); - - return false; - } - try { - if ($this->backup->executions() - ->where(fn ($query) => $query - ->where('status', 'running') - ->orWhere('stop_recovery_pending', true) - ->orWhere('s3_cleanup_pending', true)) - ->exists()) { - $this->dispatch('error', 'Wait for the running storage backup and container recovery to finish before deleting this schedule.'); - - return false; - } - - $localFilenames = $this->backup->executions() - ->where('local_storage_deleted', false) - ->pluck('filename') - ->filter() - ->all(); - $server = $this->backup->server(); - - if ($localFilenames !== []) { - if (! $server) { - throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.'); - } - - deleteBackupsLocally($localFilenames, $server, throwError: true); - } - - $s3Filenames = $this->backup->executions() - ->where('s3_uploaded', true) - ->where('s3_storage_deleted', false) - ->pluck('filename') - ->filter() - ->all(); - - if ($s3Filenames !== []) { - if (! $this->backup->s3) { - throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.'); - } - - deleteBackupsS3($s3Filenames, $this->backup->s3); - } - - $this->backup->delete(); + DeleteScheduledVolumeBackup::run($this->backup); $this->backup = null; $this->dispatch('success', 'Storage backup schedule and archives deleted.'); $this->redirectRoute('project.application.backup.index', [ @@ -278,8 +229,6 @@ public function delete(?string $password = null, array $selectedActions = []) $this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage()); return false; - } finally { - $lock->release(); } } diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index f8b4bdd56..82a358344 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -69,13 +69,12 @@ protected static function boot(): void 'save_s3' => false, 's3_storage_id' => null, ]); - $volumeBackupIds = ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->pluck('id'); - ScheduledVolumeBackupExecution::whereIn('scheduled_volume_backup_id', $volumeBackupIds) + ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id) ->update([ 's3_storage_deleted' => true, 's3_cleanup_pending' => false, ]); - ScheduledVolumeBackup::whereKey($volumeBackupIds)->update([ + ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([ 'save_s3' => false, 's3_storage_id' => null, ]); diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php index 1bcf5e6bc..528734446 100644 --- a/app/Models/ScheduledVolumeBackupExecution.php +++ b/app/Models/ScheduledVolumeBackupExecution.php @@ -9,6 +9,7 @@ class ScheduledVolumeBackupExecution extends BaseModel protected $fillable = [ 'uuid', 'scheduled_volume_backup_id', + 's3_storage_id', 'status', 'message', 'size', @@ -40,4 +41,9 @@ public function scheduledVolumeBackup(): BelongsTo { return $this->belongsTo(ScheduledVolumeBackup::class); } + + public function s3(): BelongsTo + { + return $this->belongsTo(S3Storage::class, 's3_storage_id'); + } } diff --git a/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php index a8a044373..236c937dd 100644 --- a/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php +++ b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php @@ -15,6 +15,7 @@ public function up(): void $table->id(); $table->string('uuid')->unique(); $table->foreignId('scheduled_volume_backup_id')->constrained()->cascadeOnDelete(); + $table->foreignId('s3_storage_id')->nullable()->constrained()->nullOnDelete(); $table->enum('status', ['success', 'failed', 'running'])->default('running'); $table->longText('message')->nullable(); $table->unsignedBigInteger('size')->default(0); diff --git a/resources/views/livewire/project/database/scheduled-backups.blade.php b/resources/views/livewire/project/database/scheduled-backups.blade.php index 963ba232d..f93fc050f 100644 --- a/resources/views/livewire/project/database/scheduled-backups.blade.php +++ b/resources/views/livewire/project/database/scheduled-backups.blade.php @@ -3,11 +3,12 @@ backups: @js($database->scheduledBackups->map(fn ($backup) => [ 'name' => strtolower($database->name), 'frequency' => strtolower($backup->frequency), + 's3_storage' => strtolower($backup->s3?->name ?? ''), ])->values()), hasMatches() { const query = this.search.toLowerCase(); - return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query)); + return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query) || backup.s3_storage.includes(query)); }, }">
@@ -31,14 +32,14 @@ @else
+ placeholder="Search by database name, frequency, or S3 storage..." />
No scheduled backups match your search.
@forelse($database->scheduledBackups as $backup) @if ($type == 'database') - $backup->latest_log && @@ -112,18 +113,18 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @endif @endif @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @else Last Run: Never • Total Executions: 0 @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @endif
@else - $backup->latest_log && @@ -197,7 +198,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @endif @endif @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif
Total Executions: {{ $backup->executions()->count() }} @php @@ -217,7 +218,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @else Last Run: Never • Total Executions: 0 @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @endif diff --git a/tests/Feature/BackupSearchTest.php b/tests/Feature/BackupSearchTest.php index af4092617..c9edadd97 100644 --- a/tests/Feature/BackupSearchTest.php +++ b/tests/Feature/BackupSearchTest.php @@ -5,6 +5,7 @@ use App\Models\InstanceSettings; use App\Models\LocalPersistentVolume; use App\Models\Project; +use App\Models\S3Storage; use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; @@ -70,9 +71,21 @@ 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), ]); + $storage = S3Storage::create([ + 'name' => 'Archive-Bucket', + 'region' => 'us-east-1', + 'key' => 'test-key', + 'secret' => 'test-secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://s3.example.com', + 'is_usable' => true, + 'team_id' => $this->team->id, + ]); $database->scheduledBackups()->create([ 'team_id' => $this->team->id, 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $storage->id, ]); $database->scheduledBackups()->create([ 'team_id' => $this->team->id, @@ -89,11 +102,56 @@ ->assertOk() ->assertSee('

0 3 * * 1

', false) ->assertSee('

daily

', false) + ->assertSee('S3: Archive-Bucket') + ->assertSee('archive-bucket', false) + ->assertSee('backup.s3_storage.includes(query)', false) + ->assertSee('Search by database name, frequency, or S3 storage...') ->assertSee('x-model="search"', false) ->assertSee('x-show=', false) ->assertSee('No scheduled backups match your search.'); }); +it('renders unavailable S3 storage only for S3-enabled database backups', function () { + $server = Server::factory()->create(['team_id' => $this->team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $database = StandalonePostgresql::create([ + 'name' => 'Orders-Primary', + 'image' => 'postgres:16-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => null, + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'weekly', + 'save_s3' => false, + 's3_storage_id' => null, + ]); + + $parameters = [ + 'project_uuid' => $project->uuid, + 'environment_uuid' => $environment->uuid, + 'database_uuid' => $database->uuid, + ]; + + $response = $this->get(route('project.database.backup.index', $parameters)) + ->assertOk() + ->assertSee('S3: Storage unavailable'); + + expect(substr_count($response->getContent(), 'S3:'))->toBe(1); +}); + function createBackupSearchApplication(Team $team): Application { $server = Server::factory()->create(['team_id' => $team->id]); diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.php index fd55aabd2..c21e1be22 100644 --- a/tests/Feature/Jobs/DatabaseBackupJobTest.php +++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php @@ -7,6 +7,7 @@ use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Schema; +use Illuminate\Support\Facades\Storage; uses(RefreshDatabase::class); @@ -238,6 +239,57 @@ expect($log->message)->toContain('Some real failure'); }); +test('retention cleanup failure does not fail a successful database backup', function () { + $team = Team::factory()->create(); + $s3 = S3Storage::create([ + 'name' => 'Test S3', + 'region' => 'us-east-1', + 'key' => 'test-key', + 'secret' => 'test-secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $s3->id, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'database_backup_retention_amount_s3' => 1, + ]); + $oldExecution = ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'old-backup', + 'database_name' => 'database', + 'filename' => '/backup/old.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + 'local_storage_deleted' => true, + 'created_at' => now()->subDay(), + ]); + ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'new-backup', + 'database_name' => 'database', + 'filename' => '/backup/new.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + 'local_storage_deleted' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + $job = new DatabaseBackupJob($backup); + + expect(fn () => (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job)) + ->not->toThrow(Throwable::class); + + expect($oldExecution->fresh()->s3_storage_deleted)->toBeFalse(); +}); + test('s3 storage has scheduled backups relationship', function () { $team = Team::factory()->create(); diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php index fe2aac015..17194a187 100644 --- a/tests/Feature/VolumeBackupTest.php +++ b/tests/Feature/VolumeBackupTest.php @@ -1,5 +1,6 @@ toBeTrue(); }); +it('records the S3 storage used by each volume backup execution', function () { + expect(Schema::hasColumn('scheduled_volume_backup_executions', 's3_storage_id'))->toBeTrue() + ->and((new ScheduledVolumeBackupExecution)->s3())->not->toBeNull(); +}); + it('exposes actions to manage and run volume backups', function () { expect(method_exists(VolumeBackups::class, 'save'))->toBeTrue() ->and(method_exists(VolumeBackups::class, 'backupNow'))->toBeTrue() @@ -998,6 +1004,7 @@ function signInForVolumeBackups($testCase, Team $team): User ]); $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, 'status' => 'success', 'filename' => '/data/coolify/backups/volumes/test/s3.tar.gz', 's3_uploaded' => true, @@ -1012,6 +1019,50 @@ function signInForVolumeBackups($testCase, Team $team): User ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse(); }); +it('marks historical executions deleted when their recorded S3 storage is removed', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz', + 's3_uploaded' => true, + 's3_cleanup_pending' => true, + ]); + + $originalStorage->delete(); + + expect($backup->fresh()->s3_storage_id)->toBe($newStorage->id) + ->and($execution->fresh()->s3_storage_id)->toBeNull() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue() + ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse(); +}); + it('queues a manual backup before a schedule has been saved', function () { Queue::fake(); $team = Team::factory()->create(); @@ -1100,6 +1151,61 @@ function signInForVolumeBackups($testCase, Team $team): User && str_contains($process->command, 'archive.tar.gz')); }); +it('deletes S3 archives from the storage recorded on each execution', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz', + 'local_storage_deleted' => true, + 's3_uploaded' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete') + ->once() + ->with(['/data/coolify/backups/volumes/test/historical.tar.gz']) + ->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('success'); + + expect($backup->fresh())->toBeNull(); +}); + it('refuses to delete a schedule while its backup is running', function () { Process::fake(); $team = Team::factory()->create(); @@ -1156,6 +1262,31 @@ function signInForVolumeBackups($testCase, Team $team): User expect($volume->fresh())->not->toBeNull(); }); +it('deletes disabled volume backup archives before deleting their application', function () { + Process::fake(); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => false, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/application-delete.tar.gz', + ]); + $application->delete(); + + (new DeleteResourceJob($application))->handle(); + + expect($backup->fresh())->toBeNull() + ->and($execution->fresh())->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'application-delete.tar.gz')); +}); + it('marks a running execution failed even when the job instance lost its execution state', function () { Process::fake(); $team = Team::factory()->create(); @@ -1222,6 +1353,46 @@ function signInForVolumeBackups($testCase, Team $team): User && ! str_contains($process->command, ':/backup')); }); +it('keeps the upload destination on the volume backup execution', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Execution destination', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $s3Storage->id, + ]); + $sshDisk = Storage::fake('ssh-keys'); + $disk = Mockery::mock(); + $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]); + $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); + Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk); + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + + expect($execution->s3_storage_id)->toBe($s3Storage->id) + ->and($execution->s3->is($s3Storage))->toBeTrue(); +}); + it('removes local volume backups older than the configured retention days', function () { config(['broadcasting.default' => 'null']); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); @@ -1295,6 +1466,70 @@ function signInForVolumeBackups($testCase, Team $team): User && str_contains($process->command, 'over-limit.tar.gz')); }); +it('removes retained S3 archives from the storage recorded on each execution', function () { + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0, + 'retention_amount_s3' => 1, + 'retention_days_s3' => 0, + 'retention_max_storage_s3' => 0, + ]); + $oldExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/old.tar.gz', + 's3_uploaded' => true, + 'created_at' => now()->subDay(), + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $newStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/new.tar.gz', + 's3_uploaded' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + $job = new VolumeBackupJob($backup); + $method = (new ReflectionClass($job))->getMethod('removeExpiredBackups'); + + $method->invoke($job, $server); + + expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + it('keeps a successful backup successful when retention cleanup fails', function () { config(['broadcasting.default' => 'null']); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); @@ -1511,6 +1746,7 @@ function signInForVolumeBackups($testCase, Team $team): User ]); $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, 'status' => 'failed', 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', 's3_cleanup_pending' => true, @@ -1527,6 +1763,55 @@ function signInForVolumeBackups($testCase, Team $team): User ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); }); +it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + + VolumeBackupRecoveryJob::cleanupS3Upload($execution); + + expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () { Queue::fake(); Process::fake(); @@ -1550,6 +1835,7 @@ function signInForVolumeBackups($testCase, Team $team): User ]); $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, 'status' => 'running', 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', 's3_cleanup_pending' => true,