From ac36a590881c89f3a0fe1453d1be1dbc7e0ae1ae Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 16 Jul 2026 14:30:58 +0200 Subject: [PATCH] fix(backups): pin S3 per volume execution and harden deletes Store s3_storage_id on scheduled volume backup executions so retention and recovery use the S3 that received the upload. Extract DeleteScheduledVolumeBackup for UI and resource deletion, and isolate database backup retention failures so cleanup errors do not fail the job. --- .../Shared/DeleteScheduledVolumeBackup.php | 71 +++++ app/Jobs/DatabaseBackupJob.php | 15 +- app/Jobs/DeleteResourceJob.php | 33 +- app/Jobs/VolumeBackupJob.php | 22 +- app/Jobs/VolumeBackupRecoveryJob.php | 4 +- .../Project/Database/ScheduledBackups.php | 8 + .../Project/Shared/Storages/VolumeBackups.php | 55 +--- app/Models/S3Storage.php | 5 +- app/Models/ScheduledVolumeBackupExecution.php | 6 + ...heduled_volume_backup_executions_table.php | 1 + .../database/scheduled-backups.blade.php | 17 +- tests/Feature/BackupSearchTest.php | 58 ++++ tests/Feature/Jobs/DatabaseBackupJobTest.php | 52 ++++ tests/Feature/VolumeBackupTest.php | 286 ++++++++++++++++++ 14 files changed, 556 insertions(+), 77 deletions(-) create mode 100644 app/Actions/Shared/DeleteScheduledVolumeBackup.php diff --git a/app/Actions/Shared/DeleteScheduledVolumeBackup.php b/app/Actions/Shared/DeleteScheduledVolumeBackup.php new file mode 100644 index 000000000..55972ab52 --- /dev/null +++ b/app/Actions/Shared/DeleteScheduledVolumeBackup.php @@ -0,0 +1,71 @@ +id), $backup->timeout + 300); + + if (! $lock->get()) { + throw new \RuntimeException('Wait for the queued or running storage backup to finish before deleting this schedule.'); + } + + try { + if ($backup->executions() + ->where(fn ($query) => $query + ->where('status', 'running') + ->orWhere('stop_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->exists()) { + throw new \RuntimeException('Wait for the running storage backup and recovery operations to finish before deleting this schedule.'); + } + + $localFilenames = $backup->executions() + ->where('local_storage_deleted', false) + ->pluck('filename') + ->filter() + ->all(); + + if ($localFilenames !== []) { + $server ??= $backup->server(); + if (! $server) { + throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.'); + } + + deleteBackupsLocally($localFilenames, $server, throwError: true); + } + + $s3Executions = $backup->executions() + ->with('s3') + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false) + ->get(); + + foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { + $s3 = $executions->first()->s3; + if (! $s3) { + throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + } + + $filenames = $executions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsS3($filenames, $s3); + } + } + + $backup->delete(); + } finally { + $lock->release(); + } + } +} diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index f25158d34..c80da0cab 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -497,7 +497,7 @@ public function handle(): void } } if ($this->backup_log && $this->backup_log->status === 'success') { - removeOldBackups($this->backup); + $this->removeExpiredBackups(); } } catch (Throwable $e) { throw $e; @@ -513,6 +513,19 @@ public function handle(): void } } + private function removeExpiredBackups(): void + { + try { + removeOldBackups($this->backup); + } catch (Throwable $exception) { + Log::channel('scheduled-errors')->warning('Database backup retention cleanup failed', [ + 'backup_id' => $this->backup->id, + 'execution_id' => $this->backup_log?->id, + 'error' => $exception->getMessage(), + ]); + } + } + private function backup_standalone_mongodb(string $databaseWithCollections): void { try { diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.php index 825604910..656e5c409 100644 --- a/app/Jobs/DeleteResourceJob.php +++ b/app/Jobs/DeleteResourceJob.php @@ -7,7 +7,10 @@ use App\Actions\Server\CleanupDocker; use App\Actions\Service\DeleteService; use App\Actions\Service\StopService; +use App\Actions\Shared\DeleteScheduledVolumeBackup; +use App\Enums\ApplicationDeploymentStatus; use App\Models\Application; +use App\Models\ApplicationDeploymentQueue; use App\Models\ApplicationPreview; use App\Models\Service; use App\Models\StandaloneClickhouse; @@ -42,6 +45,10 @@ public function __construct( public function handle() { + if (! $this->resource instanceof ApplicationPreview) { + $this->deleteScheduledVolumeBackups(); + } + try { // Handle ApplicationPreview instances separately if ($this->resource instanceof ApplicationPreview) { @@ -113,6 +120,24 @@ public function handle() } } + private function deleteScheduledVolumeBackups(): void + { + $server = data_get($this->resource, 'server') ?? data_get($this->resource, 'destination.server'); + $resources = $this->resource instanceof Service + ? $this->resource->applications()->get()->concat($this->resource->databases()->get()) + : collect([$this->resource]); + + foreach ($resources as $resource) { + $storages = $resource->persistentStorages()->get()->concat($resource->fileStorages()->get()); + + foreach ($storages as $storage) { + foreach ($storage->scheduledBackups()->get() as $backup) { + DeleteScheduledVolumeBackup::run($backup, $server); + } + } + } + } + private function deleteApplicationPreview() { $application = $this->resource->application; @@ -125,11 +150,11 @@ private function deleteApplicationPreview() } // Cancel any active deployments for this PR (same logic as API cancel_deployment) - $activeDeployments = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id) + $activeDeployments = ApplicationDeploymentQueue::where('application_id', $application->id) ->where('pull_request_id', $pull_request_id) ->whereIn('status', [ - \App\Enums\ApplicationDeploymentStatus::QUEUED->value, - \App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value, + ApplicationDeploymentStatus::QUEUED->value, + ApplicationDeploymentStatus::IN_PROGRESS->value, ]) ->get(); @@ -137,7 +162,7 @@ private function deleteApplicationPreview() try { // Mark deployment as cancelled $activeDeployment->update([ - 'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value, + 'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value, ]); // Add cancellation log entry diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index 98b5373bf..bb9be3a8f 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -63,7 +63,9 @@ public function handle(): void throw new \RuntimeException('The storage backup resource, team, or server no longer exists.'); } - $this->execution = $this->backup->executions()->create(); + $this->execution = $this->backup->executions()->create([ + 's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null, + ]); BackupCreated::dispatch($team->id); $backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid; @@ -352,6 +354,7 @@ private function removeExpiredBackups(Server $server): void if ($this->backup->save_s3 && $this->backup->s3) { $s3Executions = $this->backup->executions() + ->with('s3') ->where('status', 'success') ->where('s3_uploaded', true) ->where('s3_storage_deleted', false) @@ -363,11 +366,18 @@ private function removeExpiredBackups(Server $server): void $this->backup->retention_max_storage_s3, ); - $filenames = $s3Executions->pluck('filename')->filter()->all(); - if ($filenames !== []) { - deleteBackupsS3($filenames, $this->backup->s3); - $this->backup->executions()->whereKey($s3Executions->pluck('id')->all()) - ->update(['s3_storage_deleted' => true]); + foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { + $s3 = $executions->first()->s3; + if (! $s3) { + throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + } + + $filenames = $executions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsS3($filenames, $s3); + $this->backup->executions()->whereKey($executions->pluck('id')->all()) + ->update(['s3_storage_deleted' => true]); + } } } diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php index 48db9b0a8..a3465b192 100644 --- a/app/Jobs/VolumeBackupRecoveryJob.php +++ b/app/Jobs/VolumeBackupRecoveryJob.php @@ -96,8 +96,8 @@ private static function recoverContainers(ScheduledVolumeBackupExecution $execut public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void { - $execution->loadMissing('scheduledVolumeBackup.s3'); - $s3 = $execution->scheduledVolumeBackup?->s3; + $execution->loadMissing('s3'); + $s3 = $execution->s3; if (! $s3 || blank($execution->filename)) { throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.'); diff --git a/app/Livewire/Project/Database/ScheduledBackups.php b/app/Livewire/Project/Database/ScheduledBackups.php index 77a380ea0..aa7e9f43c 100644 --- a/app/Livewire/Project/Database/ScheduledBackups.php +++ b/app/Livewire/Project/Database/ScheduledBackups.php @@ -3,6 +3,7 @@ namespace App\Livewire\Project\Database; use App\Models\ServiceDatabase; +use Illuminate\Contracts\View\View; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; @@ -63,4 +64,11 @@ public function refreshScheduledBackups(?int $id = null): void $this->database->refresh(); $this->dispatch('refreshScheduledBackups'); } + + public function render(): View + { + $this->database->loadMissing('scheduledBackups.s3'); + + return view('livewire.project.database.scheduled-backups'); + } } diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php index e85036c55..d16a5e2c8 100644 --- a/app/Livewire/Project/Shared/Storages/VolumeBackups.php +++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php @@ -2,6 +2,7 @@ namespace App\Livewire\Project\Shared\Storages; +use App\Actions\Shared\DeleteScheduledVolumeBackup; use App\Jobs\VolumeBackupJob; use App\Models\LocalFileVolume; use App\Models\LocalPersistentVolume; @@ -9,7 +10,6 @@ use App\Models\ScheduledVolumeBackup; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; -use Illuminate\Support\Facades\Cache; use Livewire\Component; use Livewire\WithPagination; use Throwable; @@ -214,57 +214,8 @@ public function delete(?string $password = null, array $selectedActions = []) return false; } - $lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300); - - if (! $lock->get()) { - $this->dispatch('error', 'Wait for the queued or running storage backup to finish before deleting this schedule.'); - - return false; - } - try { - if ($this->backup->executions() - ->where(fn ($query) => $query - ->where('status', 'running') - ->orWhere('stop_recovery_pending', true) - ->orWhere('s3_cleanup_pending', true)) - ->exists()) { - $this->dispatch('error', 'Wait for the running storage backup and container recovery to finish before deleting this schedule.'); - - return false; - } - - $localFilenames = $this->backup->executions() - ->where('local_storage_deleted', false) - ->pluck('filename') - ->filter() - ->all(); - $server = $this->backup->server(); - - if ($localFilenames !== []) { - if (! $server) { - throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.'); - } - - deleteBackupsLocally($localFilenames, $server, throwError: true); - } - - $s3Filenames = $this->backup->executions() - ->where('s3_uploaded', true) - ->where('s3_storage_deleted', false) - ->pluck('filename') - ->filter() - ->all(); - - if ($s3Filenames !== []) { - if (! $this->backup->s3) { - throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.'); - } - - deleteBackupsS3($s3Filenames, $this->backup->s3); - } - - $this->backup->delete(); + DeleteScheduledVolumeBackup::run($this->backup); $this->backup = null; $this->dispatch('success', 'Storage backup schedule and archives deleted.'); $this->redirectRoute('project.application.backup.index', [ @@ -278,8 +229,6 @@ public function delete(?string $password = null, array $selectedActions = []) $this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage()); return false; - } finally { - $lock->release(); } } diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index f8b4bdd56..82a358344 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -69,13 +69,12 @@ protected static function boot(): void 'save_s3' => false, 's3_storage_id' => null, ]); - $volumeBackupIds = ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->pluck('id'); - ScheduledVolumeBackupExecution::whereIn('scheduled_volume_backup_id', $volumeBackupIds) + ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id) ->update([ 's3_storage_deleted' => true, 's3_cleanup_pending' => false, ]); - ScheduledVolumeBackup::whereKey($volumeBackupIds)->update([ + ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([ 'save_s3' => false, 's3_storage_id' => null, ]); diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php index 1bcf5e6bc..528734446 100644 --- a/app/Models/ScheduledVolumeBackupExecution.php +++ b/app/Models/ScheduledVolumeBackupExecution.php @@ -9,6 +9,7 @@ class ScheduledVolumeBackupExecution extends BaseModel protected $fillable = [ 'uuid', 'scheduled_volume_backup_id', + 's3_storage_id', 'status', 'message', 'size', @@ -40,4 +41,9 @@ public function scheduledVolumeBackup(): BelongsTo { return $this->belongsTo(ScheduledVolumeBackup::class); } + + public function s3(): BelongsTo + { + return $this->belongsTo(S3Storage::class, 's3_storage_id'); + } } diff --git a/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php index a8a044373..236c937dd 100644 --- a/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php +++ b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php @@ -15,6 +15,7 @@ public function up(): void $table->id(); $table->string('uuid')->unique(); $table->foreignId('scheduled_volume_backup_id')->constrained()->cascadeOnDelete(); + $table->foreignId('s3_storage_id')->nullable()->constrained()->nullOnDelete(); $table->enum('status', ['success', 'failed', 'running'])->default('running'); $table->longText('message')->nullable(); $table->unsignedBigInteger('size')->default(0); diff --git a/resources/views/livewire/project/database/scheduled-backups.blade.php b/resources/views/livewire/project/database/scheduled-backups.blade.php index 963ba232d..f93fc050f 100644 --- a/resources/views/livewire/project/database/scheduled-backups.blade.php +++ b/resources/views/livewire/project/database/scheduled-backups.blade.php @@ -3,11 +3,12 @@ backups: @js($database->scheduledBackups->map(fn ($backup) => [ 'name' => strtolower($database->name), 'frequency' => strtolower($backup->frequency), + 's3_storage' => strtolower($backup->s3?->name ?? ''), ])->values()), hasMatches() { const query = this.search.toLowerCase(); - return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query)); + return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query) || backup.s3_storage.includes(query)); }, }">
@@ -31,14 +32,14 @@ @else
+ placeholder="Search by database name, frequency, or S3 storage..." />
No scheduled backups match your search.
@forelse($database->scheduledBackups as $backup) @if ($type == 'database') - $backup->latest_log && @@ -112,18 +113,18 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @endif @endif @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @else Last Run: Never • Total Executions: 0 @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @endif
@else - $backup->latest_log && @@ -197,7 +198,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @endif @endif @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif
Total Executions: {{ $backup->executions()->count() }} @php @@ -217,7 +218,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @else Last Run: Never • Total Executions: 0 @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @endif diff --git a/tests/Feature/BackupSearchTest.php b/tests/Feature/BackupSearchTest.php index af4092617..c9edadd97 100644 --- a/tests/Feature/BackupSearchTest.php +++ b/tests/Feature/BackupSearchTest.php @@ -5,6 +5,7 @@ use App\Models\InstanceSettings; use App\Models\LocalPersistentVolume; use App\Models\Project; +use App\Models\S3Storage; use App\Models\Server; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; @@ -70,9 +71,21 @@ 'destination_id' => $destination->id, 'destination_type' => $destination->getMorphClass(), ]); + $storage = S3Storage::create([ + 'name' => 'Archive-Bucket', + 'region' => 'us-east-1', + 'key' => 'test-key', + 'secret' => 'test-secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://s3.example.com', + 'is_usable' => true, + 'team_id' => $this->team->id, + ]); $database->scheduledBackups()->create([ 'team_id' => $this->team->id, 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $storage->id, ]); $database->scheduledBackups()->create([ 'team_id' => $this->team->id, @@ -89,11 +102,56 @@ ->assertOk() ->assertSee('

0 3 * * 1

', false) ->assertSee('

daily

', false) + ->assertSee('S3: Archive-Bucket') + ->assertSee('archive-bucket', false) + ->assertSee('backup.s3_storage.includes(query)', false) + ->assertSee('Search by database name, frequency, or S3 storage...') ->assertSee('x-model="search"', false) ->assertSee('x-show=', false) ->assertSee('No scheduled backups match your search.'); }); +it('renders unavailable S3 storage only for S3-enabled database backups', function () { + $server = Server::factory()->create(['team_id' => $this->team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $database = StandalonePostgresql::create([ + 'name' => 'Orders-Primary', + 'image' => 'postgres:16-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => null, + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'weekly', + 'save_s3' => false, + 's3_storage_id' => null, + ]); + + $parameters = [ + 'project_uuid' => $project->uuid, + 'environment_uuid' => $environment->uuid, + 'database_uuid' => $database->uuid, + ]; + + $response = $this->get(route('project.database.backup.index', $parameters)) + ->assertOk() + ->assertSee('S3: Storage unavailable'); + + expect(substr_count($response->getContent(), 'S3:'))->toBe(1); +}); + function createBackupSearchApplication(Team $team): Application { $server = Server::factory()->create(['team_id' => $team->id]); diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.php index fd55aabd2..c21e1be22 100644 --- a/tests/Feature/Jobs/DatabaseBackupJobTest.php +++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php @@ -7,6 +7,7 @@ use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Schema; +use Illuminate\Support\Facades\Storage; uses(RefreshDatabase::class); @@ -238,6 +239,57 @@ expect($log->message)->toContain('Some real failure'); }); +test('retention cleanup failure does not fail a successful database backup', function () { + $team = Team::factory()->create(); + $s3 = S3Storage::create([ + 'name' => 'Test S3', + 'region' => 'us-east-1', + 'key' => 'test-key', + 'secret' => 'test-secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $s3->id, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'database_backup_retention_amount_s3' => 1, + ]); + $oldExecution = ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'old-backup', + 'database_name' => 'database', + 'filename' => '/backup/old.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + 'local_storage_deleted' => true, + 'created_at' => now()->subDay(), + ]); + ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'new-backup', + 'database_name' => 'database', + 'filename' => '/backup/new.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + 'local_storage_deleted' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + $job = new DatabaseBackupJob($backup); + + expect(fn () => (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job)) + ->not->toThrow(Throwable::class); + + expect($oldExecution->fresh()->s3_storage_deleted)->toBeFalse(); +}); + test('s3 storage has scheduled backups relationship', function () { $team = Team::factory()->create(); diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php index fe2aac015..17194a187 100644 --- a/tests/Feature/VolumeBackupTest.php +++ b/tests/Feature/VolumeBackupTest.php @@ -1,5 +1,6 @@ toBeTrue(); }); +it('records the S3 storage used by each volume backup execution', function () { + expect(Schema::hasColumn('scheduled_volume_backup_executions', 's3_storage_id'))->toBeTrue() + ->and((new ScheduledVolumeBackupExecution)->s3())->not->toBeNull(); +}); + it('exposes actions to manage and run volume backups', function () { expect(method_exists(VolumeBackups::class, 'save'))->toBeTrue() ->and(method_exists(VolumeBackups::class, 'backupNow'))->toBeTrue() @@ -998,6 +1004,7 @@ function signInForVolumeBackups($testCase, Team $team): User ]); $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, 'status' => 'success', 'filename' => '/data/coolify/backups/volumes/test/s3.tar.gz', 's3_uploaded' => true, @@ -1012,6 +1019,50 @@ function signInForVolumeBackups($testCase, Team $team): User ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse(); }); +it('marks historical executions deleted when their recorded S3 storage is removed', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz', + 's3_uploaded' => true, + 's3_cleanup_pending' => true, + ]); + + $originalStorage->delete(); + + expect($backup->fresh()->s3_storage_id)->toBe($newStorage->id) + ->and($execution->fresh()->s3_storage_id)->toBeNull() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue() + ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse(); +}); + it('queues a manual backup before a schedule has been saved', function () { Queue::fake(); $team = Team::factory()->create(); @@ -1100,6 +1151,61 @@ function signInForVolumeBackups($testCase, Team $team): User && str_contains($process->command, 'archive.tar.gz')); }); +it('deletes S3 archives from the storage recorded on each execution', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz', + 'local_storage_deleted' => true, + 's3_uploaded' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete') + ->once() + ->with(['/data/coolify/backups/volumes/test/historical.tar.gz']) + ->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('success'); + + expect($backup->fresh())->toBeNull(); +}); + it('refuses to delete a schedule while its backup is running', function () { Process::fake(); $team = Team::factory()->create(); @@ -1156,6 +1262,31 @@ function signInForVolumeBackups($testCase, Team $team): User expect($volume->fresh())->not->toBeNull(); }); +it('deletes disabled volume backup archives before deleting their application', function () { + Process::fake(); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => false, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/application-delete.tar.gz', + ]); + $application->delete(); + + (new DeleteResourceJob($application))->handle(); + + expect($backup->fresh())->toBeNull() + ->and($execution->fresh())->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'application-delete.tar.gz')); +}); + it('marks a running execution failed even when the job instance lost its execution state', function () { Process::fake(); $team = Team::factory()->create(); @@ -1222,6 +1353,46 @@ function signInForVolumeBackups($testCase, Team $team): User && ! str_contains($process->command, ':/backup')); }); +it('keeps the upload destination on the volume backup execution', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Execution destination', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $s3Storage->id, + ]); + $sshDisk = Storage::fake('ssh-keys'); + $disk = Mockery::mock(); + $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]); + $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); + Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk); + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + + expect($execution->s3_storage_id)->toBe($s3Storage->id) + ->and($execution->s3->is($s3Storage))->toBeTrue(); +}); + it('removes local volume backups older than the configured retention days', function () { config(['broadcasting.default' => 'null']); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); @@ -1295,6 +1466,70 @@ function signInForVolumeBackups($testCase, Team $team): User && str_contains($process->command, 'over-limit.tar.gz')); }); +it('removes retained S3 archives from the storage recorded on each execution', function () { + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0, + 'retention_amount_s3' => 1, + 'retention_days_s3' => 0, + 'retention_max_storage_s3' => 0, + ]); + $oldExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/old.tar.gz', + 's3_uploaded' => true, + 'created_at' => now()->subDay(), + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $newStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/new.tar.gz', + 's3_uploaded' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + $job = new VolumeBackupJob($backup); + $method = (new ReflectionClass($job))->getMethod('removeExpiredBackups'); + + $method->invoke($job, $server); + + expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + it('keeps a successful backup successful when retention cleanup fails', function () { config(['broadcasting.default' => 'null']); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); @@ -1511,6 +1746,7 @@ function signInForVolumeBackups($testCase, Team $team): User ]); $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, 'status' => 'failed', 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', 's3_cleanup_pending' => true, @@ -1527,6 +1763,55 @@ function signInForVolumeBackups($testCase, Team $team): User ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); }); +it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + + VolumeBackupRecoveryJob::cleanupS3Upload($execution); + + expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () { Queue::fake(); Process::fake(); @@ -1550,6 +1835,7 @@ function signInForVolumeBackups($testCase, Team $team): User ]); $execution = ScheduledVolumeBackupExecution::create([ 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, 'status' => 'running', 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', 's3_cleanup_pending' => true,