From 63961e0799fa5730d6854d30bbfd150fc33e3e77 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:47:57 +0200 Subject: [PATCH] feat(backups): add scheduled persistent volume backups --- app/Jobs/ScheduledJobManager.php | 107 ++ app/Jobs/VolumeBackupJob.php | 414 +++++++ app/Jobs/VolumeBackupRecoveryJob.php | 117 ++ .../Project/Application/Backup/Create.php | 130 ++ .../Project/Application/Backup/Index.php | 55 + .../Project/Application/Backup/Show.php | 46 + app/Livewire/Project/Database/BackupEdit.php | 13 + app/Livewire/Project/Shared/Storages/Show.php | 20 +- .../Project/Shared/Storages/VolumeBackups.php | 394 ++++++ app/Livewire/Storage/Show.php | 4 +- app/Models/LocalPersistentVolume.php | 6 + app/Models/S3Storage.php | 15 + app/Models/ScheduledVolumeBackup.php | 82 ++ app/Models/ScheduledVolumeBackupExecution.php | 43 + app/View/Components/Forms/Checkbox.php | 1 + bootstrap/helpers/databases.php | 10 +- ..._create_scheduled_volume_backups_table.php | 39 + ...heduled_volume_backup_executions_table.php | 42 + ...mits_to_scheduled_volume_backups_table.php | 30 + .../views/components/forms/checkbox.blade.php | 3 +- .../views/components/forms/input.blade.php | 3 + .../application/backup/create.blade.php | 36 + .../application/backup/index.blade.php | 88 ++ .../project/application/backup/show.blade.php | 11 + .../project/application/heading.blade.php | 9 + .../project/database/backup-edit.blade.php | 11 +- .../project/database/heading.blade.php | 24 +- .../database/scheduled-backups.blade.php | 24 +- .../project/shared/storages/show.blade.php | 74 +- .../shared/storages/volume-backups.blade.php | 338 +++++ .../views/livewire/storage/show.blade.php | 2 +- routes/web.php | 67 + tests/Feature/BackupEditValidationTest.php | 53 + tests/Feature/BackupSearchTest.php | 121 ++ tests/Feature/MobileResourceMenuTest.php | 13 + tests/Feature/VolumeBackupTest.php | 1092 +++++++++++++++++ 36 files changed, 3506 insertions(+), 31 deletions(-) create mode 100644 app/Jobs/VolumeBackupJob.php create mode 100644 app/Jobs/VolumeBackupRecoveryJob.php create mode 100644 app/Livewire/Project/Application/Backup/Create.php create mode 100644 app/Livewire/Project/Application/Backup/Index.php create mode 100644 app/Livewire/Project/Application/Backup/Show.php create mode 100644 app/Livewire/Project/Shared/Storages/VolumeBackups.php create mode 100644 app/Models/ScheduledVolumeBackup.php create mode 100644 app/Models/ScheduledVolumeBackupExecution.php create mode 100644 database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php create mode 100644 database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php create mode 100644 database/migrations/2026_07_15_125721_add_retention_limits_to_scheduled_volume_backups_table.php create mode 100644 resources/views/livewire/project/application/backup/create.blade.php create mode 100644 resources/views/livewire/project/application/backup/index.blade.php create mode 100644 resources/views/livewire/project/application/backup/show.blade.php create mode 100644 resources/views/livewire/project/shared/storages/volume-backups.blade.php create mode 100644 tests/Feature/BackupSearchTest.php create mode 100644 tests/Feature/VolumeBackupTest.php diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php index 46bc89d42..92a0db748 100644 --- a/app/Jobs/ScheduledJobManager.php +++ b/app/Jobs/ScheduledJobManager.php @@ -4,6 +4,8 @@ use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledTask; +use App\Models\ScheduledVolumeBackup; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\Team; use Cron\CronExpression; @@ -109,6 +111,16 @@ public function handle(): void ]); } + try { + $this->recoverPausedVolumeBackupContainers(); + $this->processScheduledVolumeBackups(); + } catch (\Exception $e) { + Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + } + // Process Docker cleanups - don't let failures stop the job manager try { $this->processDockerCleanups(); @@ -341,6 +353,101 @@ private function processScheduledTask(ScheduledTask $task, ?Server $precheckedSe } } + private function processScheduledVolumeBackups(): void + { + ScheduledVolumeBackup::query() + ->with(['volume', 'team.subscription']) + ->where('enabled', true) + ->chunkById(self::CHUNK_SIZE, function ($backups): void { + foreach ($backups as $backup) { + $this->processScheduledVolumeBackup($backup); + } + }); + } + + private function recoverPausedVolumeBackupContainers(): void + { + ScheduledVolumeBackupExecution::query() + ->where(fn (Builder $query) => $query + ->where('pause_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->chunkById(self::CHUNK_SIZE, function ($executions): void { + foreach ($executions as $execution) { + VolumeBackupRecoveryJob::dispatch($execution); + } + }); + } + + private function processScheduledVolumeBackup(ScheduledVolumeBackup $backup): void + { + try { + $server = $backup->server(); + + if ($backup->executions() + ->where(fn (Builder $query) => $query + ->where('pause_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->exists()) { + $this->skippedCount++; + $this->logSkip('volume_backup', 'container_recovery_pending', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + ]); + + return; + } + + if (! $backup->volume || ! $server) { + $backup->delete(); + $this->skippedCount++; + $this->logSkip('volume_backup', 'resource_deleted', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + ]); + + return; + } + + if (! $server->isFunctional()) { + $this->skippedCount++; + $this->logSkip('volume_backup', 'server_not_functional', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + 'server_id' => $server->id, + ]); + + return; + } + + if (isCloud() && $backup->team_id !== 0 && ! data_get($backup, 'team.subscription.stripe_invoice_paid', false)) { + $this->skippedCount++; + $this->logSkip('volume_backup', 'subscription_unpaid', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + 'server_id' => $server->id, + ]); + + return; + } + + if ($this->shouldDispatch($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) { + VolumeBackupJob::dispatch($backup); + $this->dispatchedCount++; + Log::channel('scheduled')->info('Volume backup dispatched', [ + 'backup_id' => $backup->id, + 'volume_id' => $backup->local_persistent_volume_id, + 'team_id' => $backup->team_id, + 'server_id' => $server->id, + ]); + } + } catch (\Exception $e) { + Log::channel('scheduled-errors')->error('Error processing volume backup', [ + 'backup_id' => $backup->id, + 'error' => $e->getMessage(), + ]); + } + } + private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string { if (blank(data_get($backup, 'database'))) { diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php new file mode 100644 index 000000000..fe1668001 --- /dev/null +++ b/app/Jobs/VolumeBackupJob.php @@ -0,0 +1,414 @@ +onQueue(crons_queue()); + $this->timeout = $backup->timeout ?? 3600; + } + + public function middleware(): array + { + return [ + (new WithoutOverlapping('volume-backup-'.$this->backup->id)) + ->shared() + ->expireAfter($this->timeout + 60) + ->dontRelease(), + ]; + } + + public static function lockKey(int $backupId): string + { + return 'laravel-queue-overlap:volume-backup-'.$backupId; + } + + public function handle(): void + { + $this->backup->loadMissing(['volume.resource', 'team', 's3']); + $server = $this->backup->server(); + $volume = $this->backup->volume; + $team = $this->backup->team; + + if (! $server || ! $volume || ! $team) { + throw new \RuntimeException('The volume backup resource, team, or server no longer exists.'); + } + + $this->execution = $this->backup->executions()->create(); + BackupCreated::dispatch($team->id); + + $backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$volume->uuid; + $filename = 'volume-'.str($volume->name)->slug().'-'.Carbon::now()->timestamp.'.tar.gz'; + $backupLocation = $backupDirectory.'/'.$filename; + $this->execution->update(['filename' => $backupLocation]); + + try { + $source = filled($volume->host_path) ? $volume->host_path : $volume->name; + $containerName = 'volume-backup-'.$this->execution->uuid; + $image = coolifyHelperImage().':'.getHelperVersion(); + $verifySourceCommand = filled($volume->host_path) + ? 'test -d '.escapeshellarg($source) + : 'docker volume inspect '.escapeshellarg($source).' >/dev/null'; + + $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName) + .' -v '.escapeshellarg($source.':/volume:ro') + .' '.escapeshellarg($image) + .' tar -czf - -C /volume . > '.escapeshellarg($backupLocation); + + if ($this->backup->pause_during_backup) { + $containers = $this->containersUsingVolume($source, $server); + if ($containers !== []) { + $this->execution->update(['pause_recovery_pending' => true]); + $archiveCommand = $this->archiveWithPausedContainers( + $archiveCommand, + $containers, + VolumeBackupRecoveryJob::stateFile($this->execution), + ); + } + } + + instant_remote_process([ + $verifySourceCommand, + 'mkdir -p '.escapeshellarg($backupDirectory), + $archiveCommand, + ], $server, timeout: $this->timeout, disableMultiplexing: true); + $this->execution->update([ + 'pause_container_ids' => null, + 'pause_recovery_pending' => false, + ]); + + $size = (int) instant_remote_process( + ['du -b '.escapeshellarg($backupLocation).' | cut -f1'], + $server, + disableMultiplexing: true, + ); + + if ($size <= 0) { + throw new \RuntimeException('The volume backup archive is empty or was not created.'); + } + + $warning = null; + $s3Uploaded = null; + $s3CleanupPending = false; + $localStorageDeleted = false; + + if ($this->backup->save_s3) { + $s3CleanupPending = true; + $this->execution->update(['s3_cleanup_pending' => true]); + + try { + $this->uploadToS3($backupLocation, $backupDirectory, $server); + $s3Uploaded = true; + $s3CleanupPending = false; + } catch (Throwable $exception) { + $s3Uploaded = false; + $warning = 'S3 upload failed: '.$exception->getMessage(); + + try { + VolumeBackupRecoveryJob::cleanupS3Upload($this->execution); + $s3CleanupPending = false; + } catch (Throwable $cleanupException) { + VolumeBackupRecoveryJob::dispatch($this->execution); + $warning .= ' Partial S3 upload cleanup failed: '.$cleanupException->getMessage(); + } + } + + if ($s3Uploaded && $this->backup->disable_local_backup) { + try { + deleteBackupsLocally($backupLocation, $server, throwError: true); + $localStorageDeleted = true; + } catch (Throwable $exception) { + $warning = 'S3 upload succeeded, but the local archive could not be deleted: '.$exception->getMessage(); + } + } + } + + $this->execution->update([ + 'status' => 'success', + 'message' => $warning, + 'size' => $size, + 'filename' => $backupLocation, + 's3_uploaded' => $s3Uploaded, + 's3_cleanup_pending' => $s3CleanupPending, + 'local_storage_deleted' => $localStorageDeleted, + ]); + + try { + $this->removeExpiredBackups($server); + } catch (Throwable $exception) { + Log::channel('scheduled-errors')->warning('Volume backup retention cleanup failed', [ + 'backup_id' => $this->backup->id, + 'execution_id' => $this->execution->id, + 'error' => $exception->getMessage(), + ]); + } + } catch (Throwable $exception) { + $recoveryError = $this->recoverIncompleteBackup($this->execution); + $archiveDeleted = false; + + try { + deleteBackupsLocally($backupLocation, $server, throwError: true); + $archiveDeleted = true; + } catch (Throwable $cleanupException) { + $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage(); + } + + $s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending; + + $this->execution->update([ + 'status' => 'failed', + 'message' => $exception->getMessage().$recoveryError, + 'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation, + 'local_storage_deleted' => $archiveDeleted, + ]); + + throw $exception; + } finally { + $this->execution->update(['finished_at' => now()]); + BackupCreated::dispatch($team->id); + } + } + + public function failed(?Throwable $exception): void + { + $execution = $this->execution ?? $this->backup->executions() + ->where('status', 'running') + ->latest('id') + ->first(); + + if ($execution) { + $recoveryError = $this->recoverIncompleteBackup($execution); + $server = $this->backup->server(); + $filename = $execution->filename; + $localStorageDeleted = $execution->local_storage_deleted; + + if ($server && filled($filename)) { + try { + deleteBackupsLocally($filename, $server, throwError: true); + $localStorageDeleted = true; + if (! $execution->fresh()->s3_cleanup_pending) { + $filename = null; + } + } catch (Throwable $cleanupException) { + $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage(); + } + } + + $execution->update([ + 'status' => 'failed', + 'message' => ($exception?->getMessage() ?? 'Volume backup timed out or was terminated.').$recoveryError, + 'finished_at' => now(), + 'filename' => $filename, + 'local_storage_deleted' => $localStorageDeleted, + ]); + } + } + + /** + * @return array + */ + private function containersUsingVolume(string $source, Server $server): array + { + $output = instant_remote_process( + ["containers=\$(docker ps -q) || exit 1; for container in \$containers; do paused=\$(docker inspect --format '{{.State.Paused}}' \"\$container\") || exit 1; [ \"\$paused\" = true ] && continue; mounts=\$(docker inspect --format '{{range .Mounts}}{{println .Source}}{{println .Name}}{{end}}' \"\$container\") || exit 1; if printf '%s\\n' \"\$mounts\" | grep -Fqx -- ".escapeshellarg($source).'; then echo "$container"; fi; done'], + $server, + disableMultiplexing: true, + ); + $containers = collect(preg_split('/\s+/', trim((string) $output))) + ->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1) + ->values() + ->all(); + + return $containers; + } + + /** + * @param array $containers + */ + private function archiveWithPausedContainers(string $archiveCommand, array $containers, string $stateFile): string + { + if ($containers === []) { + return $archiveCommand; + } + + $containerList = implode(' ', $containers); + $script = "set -eu\n" + .'state_file='.escapeshellarg($stateFile)."\n" + .": > \"\$state_file\"\n" + ."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker unpause \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n" + ."trap cleanup EXIT HUP INT TERM\n" + ."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker pause \"\$container\" >/dev/null; done\n" + .$archiveCommand; + + return 'sh -c '.escapeshellarg($script); + } + + private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string + { + if (! $execution->pause_recovery_pending && ! $execution->s3_cleanup_pending) { + return ''; + } + + try { + VolumeBackupRecoveryJob::recover($execution); + + return ''; + } catch (Throwable $exception) { + VolumeBackupRecoveryJob::dispatch($execution); + + return ' Container recovery failed: '.$exception->getMessage(); + } + } + + private function uploadToS3(string $backupLocation, string $backupDirectory, Server $server): void + { + $s3 = $this->backup->s3; + + if (! $s3) { + $this->backup->update(['save_s3' => false, 's3_storage_id' => null]); + + throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.'); + } + + $s3->testConnection(shouldSave: true); + $containerName = 'volume-upload-'.$this->execution->uuid; + $image = coolifyHelperImage().':'.getHelperVersion(); + $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint)) + ->map(fn (string $option): string => '--resolve '.escapeshellarg($option)) + ->implode(' '); + $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions; + + try { + instant_remote_process([ + 'docker rm -f '.escapeshellarg($containerName).' >/dev/null 2>&1 || true', + 'docker run -d --name '.escapeshellarg($containerName).' --rm -v ' + .escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image), + 'docker exec '.escapeshellarg($containerName).' mc alias set'.$resolveOptions.' temporary ' + .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret), + 'docker exec '.escapeshellarg($containerName).' mc cp '.escapeshellarg($backupLocation).' ' + .escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'), + ], $server, timeout: $this->timeout, disableMultiplexing: true); + } finally { + instant_remote_process( + ['docker rm -f '.escapeshellarg($containerName)], + $server, + throwError: false, + disableMultiplexing: true, + ); + } + } + + private function removeExpiredBackups(Server $server): void + { + $localExecutions = $this->backup->executions() + ->where('status', 'success') + ->where('local_storage_deleted', false) + ->get(); + $localExecutions = $this->executionsOutsideRetention( + $localExecutions, + $this->backup->retention_amount_locally, + $this->backup->retention_days_locally, + $this->backup->retention_max_storage_locally, + ); + + $filenames = $localExecutions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsLocally($filenames, $server, throwError: true); + $this->backup->executions()->whereKey($localExecutions->pluck('id')->all()) + ->update(['local_storage_deleted' => true]); + } + + if ($this->backup->save_s3 && $this->backup->s3) { + $s3Executions = $this->backup->executions() + ->where('status', 'success') + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false) + ->get(); + $s3Executions = $this->executionsOutsideRetention( + $s3Executions, + $this->backup->retention_amount_s3, + $this->backup->retention_days_s3, + $this->backup->retention_max_storage_s3, + ); + + $filenames = $s3Executions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsS3($filenames, $this->backup->s3); + $this->backup->executions()->whereKey($s3Executions->pluck('id')->all()) + ->update(['s3_storage_deleted' => true]); + } + } + + $this->backup->executions() + ->where('local_storage_deleted', true) + ->where(function (Builder $query): void { + $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded'); + }) + ->delete(); + } + + private function executionsOutsideRetention(Collection $executions, int $amount, int $days, float $maxStorageGb): Collection + { + if ($amount === 0 && $days === 0 && $maxStorageGb == 0) { + return collect(); + } + + $executionsToDelete = collect(); + + if ($amount > 0) { + $executionsToDelete = $executionsToDelete->merge($executions->skip($amount)); + } + + if ($days > 0) { + $oldestAllowedDate = now()->subDays($days); + $executionsToDelete = $executionsToDelete->merge( + $executions->filter(fn (ScheduledVolumeBackupExecution $execution): bool => $execution->created_at->isBefore($oldestAllowedDate)), + ); + } + + if ($maxStorageGb > 0) { + $maxStorageBytes = $maxStorageGb * 1024 ** 3; + $totalSize = 0; + + foreach ($executions as $index => $execution) { + $totalSize += (int) $execution->size; + + if ($index > 0 && $totalSize > $maxStorageBytes) { + $executionsToDelete = $executionsToDelete->merge($executions->slice($index)); + + break; + } + } + } + + return $executionsToDelete->unique('id')->values(); + } +} diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php new file mode 100644 index 000000000..1749b5309 --- /dev/null +++ b/app/Jobs/VolumeBackupRecoveryJob.php @@ -0,0 +1,117 @@ +onQueue(crons_queue()); + } + + public function middleware(): array + { + return [ + (new WithoutOverlapping('volume-backup-'.$this->execution->scheduled_volume_backup_id)) + ->shared() + ->expireAfter(300) + ->dontRelease(), + (new WithoutOverlapping('volume-backup-recovery-'.$this->execution->id)) + ->expireAfter(300) + ->dontRelease(), + ]; + } + + public function handle(): void + { + self::recover($this->execution); + } + + public static function recover(ScheduledVolumeBackupExecution $execution): void + { + $execution->loadMissing('scheduledVolumeBackup.volume.resource'); + + if ($execution->pause_recovery_pending) { + self::recoverContainers($execution); + } + + if ($execution->s3_cleanup_pending) { + self::cleanupS3Upload($execution); + } + } + + private static function recoverContainers(ScheduledVolumeBackupExecution $execution): void + { + $server = $execution->scheduledVolumeBackup?->server(); + + if (! $server) { + throw new \RuntimeException('The server is unavailable for container recovery.'); + } + + $stateFile = self::stateFile($execution); + $output = instant_remote_process( + ['cat '.escapeshellarg($stateFile).' 2>/dev/null || true'], + $server, + disableMultiplexing: true, + ); + $containers = collect(preg_split('/\s+/', trim((string) $output))) + ->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1) + ->values() + ->all(); + $execution->update(['pause_container_ids' => $containers]); + + $remainingFile = $stateFile.'.remaining'; + $script = 'status=0; : > '.escapeshellarg($remainingFile).'; ' + .'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do ' + .'[ -z "$container" ] && continue; paused=$(docker inspect --format \'{{.State.Paused}}\' "$container" 2>/dev/null) ' + .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; ' + .'if [ "$paused" = true ] && ! docker unpause "$container" >/dev/null; then echo "$container" >> ' + .escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; ' + .'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile) + .'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status'; + + instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true); + $execution->update([ + 'pause_container_ids' => null, + 'pause_recovery_pending' => false, + ]); + } + + public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void + { + $execution->loadMissing('scheduledVolumeBackup.s3'); + $s3 = $execution->scheduledVolumeBackup?->s3; + + if (! $s3 || blank($execution->filename)) { + throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.'); + } + + deleteBackupsS3($execution->filename, $s3); + $execution->update([ + 's3_cleanup_pending' => false, + 's3_storage_deleted' => true, + ]); + } + + public static function stateFile(ScheduledVolumeBackupExecution $execution): string + { + return '/tmp/coolify-volume-backup-'.$execution->uuid.'.paused'; + } +} diff --git a/app/Livewire/Project/Application/Backup/Create.php b/app/Livewire/Project/Application/Backup/Create.php new file mode 100644 index 000000000..c69421778 --- /dev/null +++ b/app/Livewire/Project/Application/Backup/Create.php @@ -0,0 +1,130 @@ + ['required', 'integer'], + 'frequency' => ['required', 'string'], + 'saveToS3' => ['required', 'boolean'], + 's3StorageId' => ['nullable', 'integer'], + ]; + } + + public function mount(): void + { + $this->authorize('view', $this->application); + $this->volumes = $this->application->persistentStorages()->orderBy('name')->get(); + $this->definedS3s = S3Storage::ownedByCurrentTeam()->where('is_usable', true)->get(); + $this->s3StorageId = $this->definedS3s->first()?->id; + $this->volumeLocked = $this->selectedVolumeId !== null; + $this->volumeId = $this->selectedVolumeId ?? $this->volumes->first()?->id; + $this->loadSelectedBackup(); + } + + public function updatedVolumeId(): void + { + $this->loadSelectedBackup(); + } + + public function submit(): void + { + $this->authorize('update', $this->application); + $this->validate(); + + $volume = $this->application->persistentStorages()->whereKey($this->volumeId)->first(); + if (! $volume) { + $this->addError('volumeId', 'Select a volume owned by this application.'); + + return; + } + + if (! validate_cron_expression($this->frequency)) { + $this->addError('frequency', 'The frequency must be a valid cron or human expression.'); + + return; + } + + if ($this->saveToS3 && ! $this->validS3StorageExists()) { + $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.'); + + return; + } + + $backup = ScheduledVolumeBackup::query()->updateOrCreate( + ['local_persistent_volume_id' => $volume->id], + [ + 'team_id' => currentTeam()->id, + 'frequency' => $this->frequency, + 'enabled' => true, + 'save_s3' => $this->saveToS3, + 's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null, + ], + ); + + $this->dispatch('success', $backup->wasRecentlyCreated ? 'Scheduled volume backup created.' : 'Scheduled volume backup updated.'); + $this->dispatch('refreshVolumeBackups'); + $this->dispatch('close-modal'); + } + + public function render() + { + return view('livewire.project.application.backup.create'); + } + + private function loadSelectedBackup(): void + { + $volume = $this->volumes->firstWhere('id', $this->volumeId); + if (! $volume) { + return; + } + + $backup = $volume->scheduledBackups()->first(); + if ($backup) { + $this->frequency = $backup->frequency; + $this->saveToS3 = $backup->save_s3; + $this->s3StorageId = $backup->s3_storage_id ?? $this->definedS3s->first()?->id; + } + } + + private function validS3StorageExists(): bool + { + return $this->s3StorageId !== null + && S3Storage::ownedByCurrentTeam() + ->where('is_usable', true) + ->whereKey($this->s3StorageId) + ->exists(); + } +} diff --git a/app/Livewire/Project/Application/Backup/Index.php b/app/Livewire/Project/Application/Backup/Index.php new file mode 100644 index 000000000..e5dd3c304 --- /dev/null +++ b/app/Livewire/Project/Application/Backup/Index.php @@ -0,0 +1,55 @@ + '$refresh']; + + public function mount(): void + { + $this->application = $this->findApplication(); + $this->authorize('view', $this->application); + $this->parameters = get_route_parameters(); + } + + public function render(): View + { + $volumeIds = $this->application->persistentStorages()->pluck('id'); + $backups = ScheduledVolumeBackup::query() + ->with(['volume', 'latestExecution']) + ->withCount('executions') + ->whereIn('local_persistent_volume_id', $volumeIds) + ->latest() + ->get(); + + return view('livewire.project.application.backup.index', ['backups' => $backups]); + } + + private function findApplication(): Application + { + $project = currentTeam()->projects() + ->where('uuid', request()->route('project_uuid')) + ->firstOrFail(); + $environment = $project->environments() + ->where('uuid', request()->route('environment_uuid')) + ->firstOrFail(); + + return $environment->applications() + ->with('destination.server') + ->where('uuid', request()->route('application_uuid')) + ->firstOrFail(); + } +} diff --git a/app/Livewire/Project/Application/Backup/Show.php b/app/Livewire/Project/Application/Backup/Show.php new file mode 100644 index 000000000..f43593390 --- /dev/null +++ b/app/Livewire/Project/Application/Backup/Show.php @@ -0,0 +1,46 @@ +projects() + ->where('uuid', request()->route('project_uuid')) + ->firstOrFail(); + $environment = $project->environments() + ->where('uuid', request()->route('environment_uuid')) + ->firstOrFail(); + $this->application = $environment->applications() + ->with('destination.server') + ->where('uuid', request()->route('application_uuid')) + ->firstOrFail(); + $this->authorize('view', $this->application); + + $this->backup = ScheduledVolumeBackup::query() + ->with('volume') + ->where('uuid', request()->route('backup_uuid')) + ->whereIn('local_persistent_volume_id', $this->application->persistentStorages()->pluck('id')) + ->firstOrFail(); + $this->parameters = get_route_parameters(); + } + + public function render() + { + return view('livewire.project.application.backup.show'); + } +} diff --git a/app/Livewire/Project/Database/BackupEdit.php b/app/Livewire/Project/Database/BackupEdit.php index 1c1bea2f6..d8ce9c97f 100644 --- a/app/Livewire/Project/Database/BackupEdit.php +++ b/app/Livewire/Project/Database/BackupEdit.php @@ -217,6 +217,19 @@ public function instantSave() } } + public function toggleEnabled(): void + { + try { + $this->authorize('manageBackups', $this->backup->database); + + $this->backupEnabled = ! $this->backupEnabled; + $this->backup->update(['enabled' => $this->backupEnabled]); + $this->dispatch('success', $this->backupEnabled ? 'Backup enabled.' : 'Backup disabled.'); + } catch (\Throwable $e) { + $this->dispatch('error', $e->getMessage()); + } + } + public function updatedS3StorageId(): void { $this->instantSave(); diff --git a/app/Livewire/Project/Shared/Storages/Show.php b/app/Livewire/Project/Shared/Storages/Show.php index 2aaca5e6f..66ab4a114 100644 --- a/app/Livewire/Project/Shared/Storages/Show.php +++ b/app/Livewire/Project/Shared/Storages/Show.php @@ -5,6 +5,7 @@ use App\Models\LocalPersistentVolume; use App\Support\ValidationPatterns; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; +use Livewire\Attributes\On; use Livewire\Component; class Show extends Component @@ -32,6 +33,8 @@ class Show extends Component public bool $isPreviewSuffixEnabled = true; + public bool $hasEnabledBackup = false; + protected $validationAttributes = [ 'name' => 'name', 'mountPath' => 'mount', @@ -81,10 +84,19 @@ private function syncData(bool $toModel = false): void } } - public function mount() + public function mount(): void { $this->syncData(false); $this->isReadOnly = $this->storage->shouldBeReadOnlyInUI(); + $this->refreshBackupStatus(); + } + + #[On('refreshVolumeBackups')] + public function refreshBackupStatus(): void + { + $this->hasEnabledBackup = $this->storage->scheduledBackups() + ->where('enabled', true) + ->exists(); } public function instantSave(): void @@ -115,6 +127,12 @@ public function delete($password, $selectedActions = []) return 'The provided password is incorrect.'; } + if ($this->storage->scheduledBackups()->exists()) { + $this->dispatch('error', 'Delete this volume backup schedule and its archives before deleting the volume.'); + + return false; + } + $this->storage->delete(); $this->dispatch('refreshStorages'); diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php new file mode 100644 index 000000000..63441ba34 --- /dev/null +++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php @@ -0,0 +1,394 @@ + ['required', 'string'], + 'enabled' => ['required', 'boolean'], + 'saveToS3' => ['required', 'boolean'], + 'disableLocalBackup' => ['required', 'boolean'], + 'pauseDuringBackup' => ['required', 'boolean'], + 's3StorageId' => ['nullable', 'integer'], + 'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'], + 'retentionDaysLocally' => ['required', 'integer', 'min:0'], + 'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'], + 'retentionAmountS3' => ['required', 'integer', 'min:0', 'max:10000'], + 'retentionDaysS3' => ['required', 'integer', 'min:0'], + 'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'], + 'timeout' => ['required', 'integer', 'min:60', 'max:36000'], + ]; + } + + public function mount(): void + { + $this->authorize('view', $this->resource); + $this->availableS3Storages = S3Storage::ownedByCurrentTeam() + ->where('is_usable', true) + ->get(); + $this->backup = $this->storage->scheduledBackups()->first(); + $server = $this->backup?->server() ?? data_get($this->resource, 'destination.server'); + $this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone'); + + if ($this->backup) { + $this->frequency = $this->backup->frequency; + $this->enabled = $this->backup->enabled; + $this->saveToS3 = $this->backup->save_s3; + $this->disableLocalBackup = $this->backup->disable_local_backup; + $this->pauseDuringBackup = $this->backup->pause_during_backup; + $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id; + $this->retentionAmountLocally = $this->backup->retention_amount_locally; + $this->retentionDaysLocally = $this->backup->retention_days_locally; + $this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally; + $this->retentionAmountS3 = $this->backup->retention_amount_s3; + $this->retentionDaysS3 = $this->backup->retention_days_s3; + $this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3; + $this->timeout = $this->backup->timeout; + } else { + $this->s3StorageId = $this->availableS3Storages->first()?->id; + } + } + + public function save(): void + { + $this->authorize('update', $this->resource); + + if (! $this->validateSettings()) { + return; + } + + $this->backup = $this->persistBackup($this->enabled); + $this->dispatch('success', 'Volume backup schedule saved.'); + } + + public function instantSave(): void + { + $this->save(); + } + + public function updatedS3StorageId(): void + { + if ($this->saveToS3) { + $this->save(); + } + } + + public function toggleEnabled(): void + { + $this->authorize('update', $this->resource); + + if (! $this->backup) { + if (! $this->validateSettings()) { + return; + } + + $this->enabled = true; + $this->backup = $this->persistBackup(true); + } else { + $this->enabled = ! $this->enabled; + $this->backup->update(['enabled' => $this->enabled]); + } + + $this->dispatch('success', $this->enabled ? 'Volume backups enabled.' : 'Volume backups disabled.'); + } + + public function backupNow(): void + { + $this->authorize('update', $this->resource); + + if (! $this->validateSettings()) { + return; + } + + if (! $this->backup) { + $this->enabled = false; + } + + $this->backup = $this->persistBackup($this->enabled); + VolumeBackupJob::dispatch($this->backup); + $this->dispatch('success', 'Volume backup queued.'); + } + + public function delete(?string $password = null, array $selectedActions = []) + { + $this->authorize('update', $this->resource); + + if (! $password || ! verifyPasswordConfirmation($password, $this)) { + return 'The provided password is incorrect.'; + } + + if (! $this->backup) { + return false; + } + + $lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300); + + if (! $lock->get()) { + $this->dispatch('error', 'Wait for the queued or running volume backup to finish before deleting this schedule.'); + + return false; + } + + try { + if ($this->backup->executions() + ->where(fn ($query) => $query + ->where('status', 'running') + ->orWhere('pause_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->exists()) { + $this->dispatch('error', 'Wait for the running volume backup and container recovery to finish before deleting this schedule.'); + + return false; + } + + $localFilenames = $this->backup->executions() + ->where('local_storage_deleted', false) + ->pluck('filename') + ->filter() + ->all(); + $server = $this->backup->server(); + + if ($localFilenames !== []) { + if (! $server) { + throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.'); + } + + deleteBackupsLocally($localFilenames, $server, throwError: true); + } + + $s3Filenames = $this->backup->executions() + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false) + ->pluck('filename') + ->filter() + ->all(); + + if ($s3Filenames !== []) { + if (! $this->backup->s3) { + throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.'); + } + + deleteBackupsS3($s3Filenames, $this->backup->s3); + } + + $this->backup->delete(); + $this->backup = null; + $this->dispatch('success', 'Volume backup schedule and archives deleted.'); + + return true; + } catch (Throwable $exception) { + $this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage()); + + return false; + } finally { + $lock->release(); + } + } + + public function cleanupFailed(): void + { + $this->authorize('update', $this->resource); + + $deletedCount = $this->backup?->executions() + ->where('status', 'failed') + ->where('pause_recovery_pending', false) + ->where('s3_cleanup_pending', false) + ->where(fn ($query) => $query + ->whereNull('filename') + ->orWhere('local_storage_deleted', true)) + ->delete() ?? 0; + + $this->dispatch( + $deletedCount > 0 ? 'success' : 'info', + $deletedCount > 0 ? 'Failed backup entries cleaned up.' : 'No safely removable failed backup entries found.', + ); + } + + public function cleanupDeleted(): void + { + $this->authorize('update', $this->resource); + + $deletedCount = $this->backup?->executions() + ->where('local_storage_deleted', true) + ->where(fn ($query) => $query + ->where('s3_storage_deleted', true) + ->orWhereNull('s3_uploaded') + ->orWhere('s3_uploaded', false)) + ->delete() ?? 0; + + $this->dispatch( + $deletedCount > 0 ? 'success' : 'info', + $deletedCount > 0 ? "Cleaned up {$deletedCount} deleted backup entries." : 'No deleted backup entries found.', + ); + } + + public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string + { + $this->authorize('update', $this->resource); + + if (! verifyPasswordConfirmation($password, $this)) { + return 'The provided password is incorrect.'; + } + + $execution = $this->backup?->executions()->whereKey($executionId)->first(); + if (! $execution) { + $this->dispatch('error', 'Backup execution not found.'); + + return false; + } + + if ($execution->status === 'running' || $execution->pause_recovery_pending || $execution->s3_cleanup_pending) { + $this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.'); + + return false; + } + + try { + $server = $this->backup->server(); + if (! $execution->local_storage_deleted && filled($execution->filename)) { + if (! $server) { + throw new \RuntimeException('The server is unavailable.'); + } + + deleteBackupsLocally($execution->filename, $server, throwError: true); + } + + if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) { + if (! $this->backup->s3) { + throw new \RuntimeException('The S3 storage is unavailable.'); + } + + deleteBackupsS3($execution->filename, $this->backup->s3); + } + + $execution->delete(); + $this->delete_backup_s3 = false; + $this->dispatch('success', 'Backup deleted.'); + + return true; + } catch (Throwable $exception) { + $this->dispatch('error', 'Failed to delete backup: '.$exception->getMessage()); + + return false; + } + } + + public function render() + { + $executions = $this->backup?->executions()->paginate(10); + + return view('livewire.project.shared.storages.volume-backups', [ + 'executions' => $executions ?? collect(), + 'latestExecution' => $this->backup?->executions()->first(), + ]); + } + + private function validateSettings(): bool + { + $this->validate(); + + if (! validate_cron_expression($this->frequency)) { + $this->addError('frequency', 'The frequency must be a valid cron or human expression.'); + + return false; + } + + if ($this->saveToS3 && ! $this->hasValidS3Storage()) { + $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.'); + + return false; + } + + $this->disableLocalBackup = $this->saveToS3 && $this->disableLocalBackup; + + return true; + } + + private function persistBackup(bool $enabled): ScheduledVolumeBackup + { + return ScheduledVolumeBackup::query()->updateOrCreate( + ['local_persistent_volume_id' => $this->storage->id], + [ + 'team_id' => currentTeam()->id, + 'frequency' => $this->frequency, + 'enabled' => $enabled, + 'save_s3' => $this->saveToS3, + 'disable_local_backup' => $this->disableLocalBackup, + 'pause_during_backup' => $this->pauseDuringBackup, + 's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null, + 'retention_amount_locally' => $this->retentionAmountLocally, + 'retention_days_locally' => $this->retentionDaysLocally, + 'retention_max_storage_locally' => $this->retentionMaxStorageLocally, + 'retention_amount_s3' => $this->retentionAmountS3, + 'retention_days_s3' => $this->retentionDaysS3, + 'retention_max_storage_s3' => $this->retentionMaxStorageS3, + 'timeout' => $this->timeout, + ], + ); + } + + private function hasValidS3Storage(): bool + { + return $this->s3StorageId !== null + && S3Storage::query() + ->whereKey($this->s3StorageId) + ->where('team_id', currentTeam()->id) + ->where('is_usable', true) + ->exists(); + } +} diff --git a/app/Livewire/Storage/Show.php b/app/Livewire/Storage/Show.php index dd6640c23..914366d4f 100644 --- a/app/Livewire/Storage/Show.php +++ b/app/Livewire/Storage/Show.php @@ -4,6 +4,7 @@ use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledVolumeBackup; use Illuminate\Auth\Access\AuthorizationException; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; @@ -30,7 +31,8 @@ public function mount() return $this->redirectRoute('storage.index', navigate: true); } $this->currentRoute = request()->route()->getName(); - $this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count(); + $this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count() + + ScheduledVolumeBackup::where('s3_storage_id', $this->storage->id)->count(); } public function delete() diff --git a/app/Models/LocalPersistentVolume.php b/app/Models/LocalPersistentVolume.php index d44c86c0c..3b2864ce2 100644 --- a/app/Models/LocalPersistentVolume.php +++ b/app/Models/LocalPersistentVolume.php @@ -3,6 +3,7 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Relations\HasMany; use Symfony\Component\Yaml\Yaml; class LocalPersistentVolume extends BaseModel @@ -41,6 +42,11 @@ public function database() return $this->morphTo('resource'); } + public function scheduledBackups(): HasMany + { + return $this->hasMany(ScheduledVolumeBackup::class, 'local_persistent_volume_id'); + } + protected function customizeName($value) { return str($value)->trim()->value; diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index 70703cd52..f8b4bdd56 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -69,6 +69,16 @@ protected static function boot(): void 'save_s3' => false, 's3_storage_id' => null, ]); + $volumeBackupIds = ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->pluck('id'); + ScheduledVolumeBackupExecution::whereIn('scheduled_volume_backup_id', $volumeBackupIds) + ->update([ + 's3_storage_deleted' => true, + 's3_cleanup_pending' => false, + ]); + ScheduledVolumeBackup::whereKey($volumeBackupIds)->update([ + 'save_s3' => false, + 's3_storage_id' => null, + ]); }); } @@ -101,6 +111,11 @@ public function scheduledBackups() return $this->hasMany(ScheduledDatabaseBackup::class, 's3_storage_id'); } + public function scheduledVolumeBackups() + { + return $this->hasMany(ScheduledVolumeBackup::class, 's3_storage_id'); + } + public function awsUrl() { return "{$this->endpoint}/{$this->bucket}"; diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.php new file mode 100644 index 000000000..23e2369b1 --- /dev/null +++ b/app/Models/ScheduledVolumeBackup.php @@ -0,0 +1,82 @@ + 'boolean', + 'save_s3' => 'boolean', + 'disable_local_backup' => 'boolean', + 'pause_during_backup' => 'boolean', + 'retention_amount_locally' => 'integer', + 'retention_days_locally' => 'integer', + 'retention_max_storage_locally' => 'float', + 'retention_amount_s3' => 'integer', + 'retention_days_s3' => 'integer', + 'retention_max_storage_s3' => 'float', + 'timeout' => 'integer', + ]; + } + + public function volume(): BelongsTo + { + return $this->belongsTo(LocalPersistentVolume::class, 'local_persistent_volume_id'); + } + + public function team(): BelongsTo + { + return $this->belongsTo(Team::class); + } + + public function s3(): BelongsTo + { + return $this->belongsTo(S3Storage::class, 's3_storage_id'); + } + + public function executions(): HasMany + { + return $this->hasMany(ScheduledVolumeBackupExecution::class)->latest(); + } + + public function latestExecution(): HasOne + { + return $this->hasOne(ScheduledVolumeBackupExecution::class)->latestOfMany(); + } + + public function server(): ?Server + { + $resource = $this->volume?->resource; + + if ($resource instanceof ServiceApplication || $resource instanceof ServiceDatabase) { + return $resource->service?->server?->fresh(); + } + + return $resource?->destination?->server?->fresh(); + } +} diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php new file mode 100644 index 000000000..b15294461 --- /dev/null +++ b/app/Models/ScheduledVolumeBackupExecution.php @@ -0,0 +1,43 @@ + 'integer', + 'finished_at' => 'datetime', + 'pause_container_ids' => 'array', + 'pause_recovery_pending' => 'boolean', + 's3_cleanup_pending' => 'boolean', + 'local_storage_deleted' => 'boolean', + 's3_storage_deleted' => 'boolean', + 's3_uploaded' => 'boolean', + ]; + } + + public function scheduledVolumeBackup(): BelongsTo + { + return $this->belongsTo(ScheduledVolumeBackup::class); + } +} diff --git a/app/View/Components/Forms/Checkbox.php b/app/View/Components/Forms/Checkbox.php index e33e4b919..56bccb06f 100644 --- a/app/View/Components/Forms/Checkbox.php +++ b/app/View/Components/Forms/Checkbox.php @@ -25,6 +25,7 @@ public function __construct( public ?string $helper = null, public string|bool|null $checked = false, public string|bool $instantSave = false, + public bool $live = false, public bool $disabled = false, public string $defaultClass = 'dark:border-neutral-700 text-coolgray-400 dark:bg-coolgray-100 rounded-sm cursor-pointer dark:disabled:bg-base dark:disabled:cursor-not-allowed focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-coollabs dark:focus-visible:ring-warning focus-visible:ring-offset-2 dark:focus-visible:ring-offset-base', public ?string $canGate = null, diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php index 5f0b2e690..4aeda9dbf 100644 --- a/bootstrap/helpers/databases.php +++ b/bootstrap/helpers/databases.php @@ -179,7 +179,7 @@ function create_standalone_clickhouse($environment_id, StandaloneDocker|SwarmDoc return $database; } -function deleteBackupsLocally(string|array|null $filenames, Server $server): void +function deleteBackupsLocally(string|array|null $filenames, Server $server, bool $throwError = false): void { if (empty($filenames)) { return; @@ -187,8 +187,8 @@ function deleteBackupsLocally(string|array|null $filenames, Server $server): voi if (is_string($filenames)) { $filenames = [$filenames]; } - $quotedFiles = array_map(fn ($file) => "\"$file\"", $filenames); - instant_remote_process(['rm -f '.implode(' ', $quotedFiles)], $server, throwError: false); + $quotedFiles = array_map(fn ($file) => escapeshellarg($file), $filenames); + instant_remote_process(['rm -f '.implode(' ', $quotedFiles)], $server, throwError: $throwError); $foldersToCheck = collect($filenames)->map(fn ($file) => dirname($file))->unique(); $foldersToCheck->each(fn ($folder) => deleteEmptyBackupFolder($folder, $server)); @@ -214,7 +214,9 @@ function deleteBackupsS3(string|array|null $filenames, S3Storage $s3): void 'aws_url' => $s3->awsUrl(), ]); - $disk->delete($filenames); + if (! $disk->delete($filenames)) { + throw new RuntimeException('One or more S3 backup files could not be deleted.'); + } } function deleteEmptyBackupFolder($folderPath, Server $server): void diff --git a/database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php b/database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php new file mode 100644 index 000000000..6ebf1a80e --- /dev/null +++ b/database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php @@ -0,0 +1,39 @@ +id(); + $table->string('uuid')->unique(); + $table->foreignId('local_persistent_volume_id')->unique()->constrained()->restrictOnDelete(); + $table->foreignId('team_id')->constrained()->cascadeOnDelete(); + $table->foreignId('s3_storage_id')->nullable()->constrained()->nullOnDelete(); + $table->string('frequency'); + $table->boolean('enabled')->default(true); + $table->boolean('save_s3')->default(false); + $table->boolean('disable_local_backup')->default(false); + $table->boolean('pause_during_backup')->default(false); + $table->unsignedInteger('retention_amount_locally')->default(7); + $table->unsignedInteger('retention_amount_s3')->default(7); + $table->unsignedInteger('timeout')->default(3600); + $table->timestamps(); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('scheduled_volume_backups'); + } +}; diff --git a/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php new file mode 100644 index 000000000..20f5fc273 --- /dev/null +++ b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php @@ -0,0 +1,42 @@ +id(); + $table->string('uuid')->unique(); + $table->foreignId('scheduled_volume_backup_id')->constrained()->cascadeOnDelete(); + $table->enum('status', ['success', 'failed', 'running'])->default('running'); + $table->longText('message')->nullable(); + $table->unsignedBigInteger('size')->default(0); + $table->text('filename')->nullable(); + $table->json('pause_container_ids')->nullable(); + $table->boolean('pause_recovery_pending')->default(false); + $table->boolean('s3_cleanup_pending')->default(false); + $table->timestamp('finished_at')->nullable(); + $table->boolean('local_storage_deleted')->default(false); + $table->boolean('s3_storage_deleted')->default(false); + $table->boolean('s3_uploaded')->nullable(); + $table->timestamps(); + + $table->index(['scheduled_volume_backup_id', 'created_at'], 'scheduled_volume_executions_backup_created_index'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('scheduled_volume_backup_executions'); + } +}; diff --git a/database/migrations/2026_07_15_125721_add_retention_limits_to_scheduled_volume_backups_table.php b/database/migrations/2026_07_15_125721_add_retention_limits_to_scheduled_volume_backups_table.php new file mode 100644 index 000000000..1cf1fb7b1 --- /dev/null +++ b/database/migrations/2026_07_15_125721_add_retention_limits_to_scheduled_volume_backups_table.php @@ -0,0 +1,30 @@ +unsignedInteger('retention_days_locally')->default(0); + $table->decimal('retention_max_storage_locally', 17, 7)->default(0); + $table->unsignedInteger('retention_days_s3')->default(0); + $table->decimal('retention_max_storage_s3', 17, 7)->default(0); + }); + } + + public function down(): void + { + Schema::table('scheduled_volume_backups', function (Blueprint $table) { + $table->dropColumn([ + 'retention_days_locally', + 'retention_max_storage_locally', + 'retention_days_s3', + 'retention_max_storage_s3', + ]); + }); + } +}; diff --git a/resources/views/components/forms/checkbox.blade.php b/resources/views/components/forms/checkbox.blade.php index 29717b9b8..efd2d9e3b 100644 --- a/resources/views/components/forms/checkbox.blade.php +++ b/resources/views/components/forms/checkbox.blade.php @@ -39,7 +39,8 @@ value={{ $domValue }} id="{{ $htmlId }}" @if ($checked) checked @endif /> @else class([$defaultClass, 'shrink-0']) }} - wire:model={{ $value ?? $modelBinding }} id="{{ $htmlId }}" @if ($checked) checked @endif /> + @if ($live) wire:model.live={{ $value ?? $modelBinding }} @else wire:model={{ $value ?? $modelBinding }} @endif + id="{{ $htmlId }}" @if ($checked) checked @endif /> @endif @endif diff --git a/resources/views/components/forms/input.blade.php b/resources/views/components/forms/input.blade.php index c81f53ce9..e905aed24 100644 --- a/resources/views/components/forms/input.blade.php +++ b/resources/views/components/forms/input.blade.php @@ -10,6 +10,9 @@ @if ($helper) @endif + @isset($labelSuffix) + {{ $labelSuffix }} + @endisset @endif @if ($type === 'password') diff --git a/resources/views/livewire/project/application/backup/create.blade.php b/resources/views/livewire/project/application/backup/create.blade.php new file mode 100644 index 000000000..d2c739b07 --- /dev/null +++ b/resources/views/livewire/project/application/backup/create.blade.php @@ -0,0 +1,36 @@ +
+ @if ($volumes->isEmpty()) +
Add a persistent volume before configuring a backup.
+ @else +
+ + @foreach ($volumes as $volume) + + @endforeach + + +
+ +
+

S3

+ @if ($definedS3s->isEmpty()) +
No validated S3 storages found.
+ @else +
+ + @if ($saveToS3) + + @foreach ($definedS3s as $s3) + + @endforeach + + @endif +
+ @endif +
+ + Save + @endif +
diff --git a/resources/views/livewire/project/application/backup/index.blade.php b/resources/views/livewire/project/application/backup/index.blade.php new file mode 100644 index 000000000..87e473e29 --- /dev/null +++ b/resources/views/livewire/project/application/backup/index.blade.php @@ -0,0 +1,88 @@ +
+ + {{ data_get_str($application, 'name')->limit(10) }} > Backups | Coolify + +

Backups

+ + + +
+

Scheduled Backups

+ @can('update', $application) + + + + @endcan +
+ +
+ +
+ + +
diff --git a/resources/views/livewire/project/application/backup/show.blade.php b/resources/views/livewire/project/application/backup/show.blade.php new file mode 100644 index 000000000..9501d473c --- /dev/null +++ b/resources/views/livewire/project/application/backup/show.blade.php @@ -0,0 +1,11 @@ +
+ + {{ data_get_str($application, 'name')->limit(10) }} > Volume Backups | Coolify + +

Volume Backups

+ + + + +
diff --git a/resources/views/livewire/project/application/heading.blade.php b/resources/views/livewire/project/application/heading.blade.php index 443234869..691a90d5e 100644 --- a/resources/views/livewire/project/application/heading.blade.php +++ b/resources/views/livewire/project/application/heading.blade.php @@ -11,6 +11,11 @@ 'route' => 'project.application.deployment.index', 'active' => request()->routeIs('project.application.deployment.index', 'project.application.deployment.show'), ], + [ + 'label' => 'Backups', + 'route' => 'project.application.backup.index', + 'active' => request()->routeIs('project.application.backup.index', 'project.application.backup.show'), + ], [ 'label' => 'Logs', 'route' => 'project.application.logs', @@ -447,6 +452,10 @@ class="scrollbar hidden min-h-10 w-full flex-nowrap items-center gap-6 overflow- href="{{ route('project.application.deployment.index', $parameters) }}"> Deployments +
- - Logs - - @can('canAccessTerminal') - - Terminal - - @endcan @if ( $database->getMorphClass() === 'App\Models\StandalonePostgresql' || $database->getMorphClass() === 'App\Models\StandaloneMongodb' || @@ -210,6 +200,16 @@ class="scrollbar hidden min-h-10 w-full flex-nowrap items-center gap-6 overflow- Backups @endif + + Logs + + @can('canAccessTerminal') + + Terminal + + @endcan @if ($database->destination->server->isFunctional()) diff --git a/resources/views/livewire/project/database/scheduled-backups.blade.php b/resources/views/livewire/project/database/scheduled-backups.blade.php index b8241569c..c522cd666 100644 --- a/resources/views/livewire/project/database/scheduled-backups.blade.php +++ b/resources/views/livewire/project/database/scheduled-backups.blade.php @@ -1,4 +1,15 @@ -
+
@if ($database->is_migrated && blank($database->custom_type))
@@ -18,9 +29,16 @@
@else +
+ +
+
+ No scheduled backups match your search. +
@forelse($database->scheduledBackups as $backup) @if ($type == 'database') - $backup->latest_log && @@ -105,7 +123,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray-
@else -
data_get($backup, 'id') === data_get($selectedBackup, 'id'), diff --git a/resources/views/livewire/project/shared/storages/show.blade.php b/resources/views/livewire/project/shared/storages/show.blade.php index c7de8ad9e..b8e9e8293 100644 --- a/resources/views/livewire/project/shared/storages/show.blade.php +++ b/resources/views/livewire/project/shared/storages/show.blade.php @@ -12,10 +12,22 @@ $storage->resource_type === 'App\Models\ServiceApplication' || $storage->resource_type === 'App\Models\ServiceDatabase') + helper="Warning: Changing the volume name after the initial start could cause problems. Only use it when you know what are you doing."> + + @if ($hasEnabledBackup) + + @endif + + @else + helper="Warning: Changing the volume name after the initial start could cause problems. Only use it when you know what are you doing."> + + @if ($hasEnabledBackup) + + @endif + + @endif @if ($isService || $startedAt) @else
- + + + @if ($hasEnabledBackup) + + @endif + +
@@ -51,14 +69,26 @@ @can('update', $resource) @if ($isFirst)
- + + + @if ($hasEnabledBackup) + + @endif + +
@else
- + + + @if ($hasEnabledBackup) + + @endif + +
@@ -74,6 +104,13 @@ Update + @if ($resource instanceof \App\Models\Application) + + + + @endif - + + + @if ($hasEnabledBackup) + + @endif + + @else
- + + + @if ($hasEnabledBackup) + + @endif + +
@@ -101,4 +150,15 @@ @endcan @endif + @if ($isReadOnly && $resource instanceof \App\Models\Application) + @can('update', $resource) +
+ + + +
+ @endcan + @endif
diff --git a/resources/views/livewire/project/shared/storages/volume-backups.blade.php b/resources/views/livewire/project/shared/storages/volume-backups.blade.php new file mode 100644 index 000000000..8549a14c5 --- /dev/null +++ b/resources/views/livewire/project/shared/storages/volume-backups.blade.php @@ -0,0 +1,338 @@ +
+
+
+
+
+

Scheduled Backup

+
+ Save + @if (!$enabled) + Enable Backup + @else + Disable Backup + @endif + + Backup Now + @if ($backup) + + + Delete Backups and Schedule + + + @endif +
+
+

+ Persistent volume: + {{ $storage->name }} +

+
+ +
+ +
+ + @if ($availableS3Storages->isNotEmpty()) + + @elseif ($saveToS3) + + @else + + @endif + @if ($saveToS3) + + @else + + @endif +
+ +
+
+ S3 Storage + @if (!$saveToS3) + (currently disabled) + @else + + @endif +
+ + @if ($availableS3Storages->isEmpty()) + + @else + @foreach ($availableS3Storages as $s3Storage) + + @endforeach + @endif + +
+ +
+

Settings

+
+ Backups made while the application is writing to this volume may be inconsistent or corrupted. You can + pause containers during the archive step for a more consistent backup, but this briefly interrupts the + application. +
+
+ + + +
+ + +

Backup Retention Settings

+
+
    +
  • Setting a value to 0 means unlimited retention.
  • +
  • The retention rules work independently - whichever limit is reached first will trigger cleanup.
  • +
+
+ +
+
+

Local Backup Retention

+
+ + + +
+
+ @if ($saveToS3) +
+

S3 Storage Retention

+
+ + + +
+
+ @endif +
+
+
+ + @if ($backup) + @php + $executionCount = method_exists($executions, 'total') ? $executions->total() : $executions->count(); + $currentPage = method_exists($executions, 'currentPage') ? $executions->currentPage() : 1; + $lastPage = method_exists($executions, 'lastPage') ? $executions->lastPage() : 1; + @endphp +
+
+

Executions ({{ $executionCount }})

+ @if ($executionCount > 0) +
+ + + + + + + Page {{ $currentPage }} of {{ $lastPage }} + + + + + + +
+ @endif +
+ + Cleanup Failed Backups + + + + Cleanup Deleted + + +
+
+ +
+ @forelse ($executions as $execution) +
$execution->status === 'running', + 'border-error' => $execution->status === 'failed', + 'border-success' => $execution->status === 'success', + ])> + @if ($execution->status === 'running') +
+ @endif +
+ $execution->status === 'running', + 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200' => $execution->status === 'failed', + 'bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-200' => $execution->status === 'success' && $execution->s3_uploaded === false, + 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => $execution->status === 'success' && $execution->s3_uploaded !== false, + ])> + @if ($execution->status === 'running') + In Progress + @elseif ($execution->status === 'success' && $execution->s3_uploaded === false) + Success (S3 Warning) + @else + {{ ucfirst($execution->status) }} + @endif + +
+
+ @if ($execution->status === 'running') + + Running for {{ calculateDuration($execution->created_at, now()) }} + + @else + @php + $finishedAt = $execution->finished_at ?? $execution->updated_at; + @endphp + + {{ $finishedAt->diffForHumans() }} + ({{ calculateDuration($execution->created_at, $finishedAt) }}) + • {{ $finishedAt->format('M j, H:i') }} + + @endif + • Volume: {{ $storage->name }} + @if ($execution->size > 0) + • Size: {{ formatBytes($execution->size) }} + @endif +
+
+ Location: {{ $execution->filename ?? 'N/A' }} +
+
+
Backup Availability:
+ !$execution->local_storage_deleted, + 'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => $execution->local_storage_deleted, + ])> + + @if (!$execution->local_storage_deleted) + + + + @else + + + + @endif + Local Storage + + + @if ($execution->s3_uploaded !== null) + $execution->s3_uploaded === false && !$execution->s3_storage_deleted, + 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => $execution->s3_uploaded === true && !$execution->s3_storage_deleted, + 'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => $execution->s3_storage_deleted, + ])> + + @if ($execution->s3_uploaded === true && !$execution->s3_storage_deleted) + + + + @else + + + + @endif + S3 Storage + + + @endif +
+ @if ($execution->message) +
+
{{ $execution->message }}
+
+ @endif + @if ($execution->status !== 'running') +
+ @if ($execution->status === 'success' && !$execution->local_storage_deleted) + + Download + + @endif + @php + $executionCheckboxes = []; + $deleteActions = []; + if (!$execution->local_storage_deleted) { + $deleteActions[] = 'This backup will be permanently deleted from local storage.'; + } + if ($execution->s3_uploaded === true && !$execution->s3_storage_deleted) { + $executionCheckboxes[] = ['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently from S3 Storage']; + } + if (empty($deleteActions)) { + $deleteActions[] = 'This backup execution record will be deleted.'; + } + @endphp + + + Delete + + +
+ @endif +
+ @empty +
No executions found.
+ @endforelse +
+
+ @endif +
+ +@script + +@endscript diff --git a/resources/views/livewire/storage/show.blade.php b/resources/views/livewire/storage/show.blade.php index 0d580486e..f510da1ba 100644 --- a/resources/views/livewire/storage/show.blade.php +++ b/resources/views/livewire/storage/show.blade.php @@ -20,7 +20,7 @@ submitAction="delete({{ $storage->id }})" :actions="array_filter([ 'The selected storage location will be permanently deleted from Coolify.', $backupCount > 0 - ? $backupCount . ' backup schedule(s) will be updated to no longer save to S3 and will only store backups locally on the server.' + ? $backupCount . ' backup schedule(s) will stop saving to S3. Existing objects in this storage will not be deleted.' : null, ])" confirmationText="{{ $storage->name }}" confirmationLabel="Please confirm the execution of the actions by entering the Storage Name below" diff --git a/routes/web.php b/routes/web.php index f729af28e..0470ffb99 100644 --- a/routes/web.php +++ b/routes/web.php @@ -18,6 +18,8 @@ use App\Livewire\Notifications\Webhook as NotificationWebhook; use App\Livewire\Profile\Appearance as ProfileAppearance; use App\Livewire\Profile\Index as ProfileIndex; +use App\Livewire\Project\Application\Backup\Index as ApplicationBackupIndex; +use App\Livewire\Project\Application\Backup\Show as ApplicationBackupShow; use App\Livewire\Project\Application\Configuration as ApplicationConfiguration; use App\Livewire\Project\Application\Deployment\Index as DeploymentIndex; use App\Livewire\Project\Application\Deployment\Show as DeploymentShow; @@ -91,6 +93,7 @@ use App\Livewire\Team\Member\Index as TeamMemberIndex; use App\Livewire\Terminal\Index as TerminalIndex; use App\Models\ScheduledDatabaseBackupExecution; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\ServiceDatabase; use App\Providers\RouteServiceProvider; @@ -224,6 +227,8 @@ Route::get('/advanced', ApplicationConfiguration::class)->name('project.application.advanced'); Route::get('/environment-variables', ApplicationConfiguration::class)->name('project.application.environment-variables'); Route::get('/persistent-storage', ApplicationConfiguration::class)->name('project.application.persistent-storage'); + Route::get('/backups', ApplicationBackupIndex::class)->name('project.application.backup.index'); + Route::get('/backups/{backup_uuid}', ApplicationBackupShow::class)->name('project.application.backup.show'); Route::get('/source', ApplicationConfiguration::class)->name('project.application.source'); Route::get('/servers', ApplicationConfiguration::class)->name('project.application.servers'); Route::get('/scheduled-tasks', ApplicationConfiguration::class)->name('project.application.scheduled-tasks.show'); @@ -411,6 +416,68 @@ } })->name('download.backup'); + Route::get('/download/volume-backup/{executionId}', function () { + try { + $user = auth()->user(); + $team = $user->currentTeam(); + if (is_null($team)) { + return response()->json(['message' => 'Team not found.'], 404); + } + if ($user->isAdminFromSession() === false) { + return response()->json(['message' => 'Only team admins/owners can download backups.'], 403); + } + + $execution = ScheduledVolumeBackupExecution::query() + ->with('scheduledVolumeBackup.volume.resource') + ->findOrFail(request()->route('executionId')); + if ($team->id !== 0 && $team->id !== $execution->scheduledVolumeBackup->team_id) { + return response()->json(['message' => 'Permission denied.'], 403); + } + if ($execution->local_storage_deleted || blank($execution->filename)) { + return response()->json(['message' => 'Backup not found locally on the server.'], 404); + } + + $server = $execution->scheduledVolumeBackup->server(); + if (! $server) { + return response()->json(['message' => 'Server not found.'], 404); + } + + $filename = $execution->filename; + $disk = Storage::build([ + 'driver' => 'sftp', + 'host' => $server->ip, + 'port' => (int) $server->port, + 'username' => $server->user, + 'privateKey' => $server->privateKey->getKeyLocation(), + 'root' => '/', + ]); + if (! $disk->exists($filename)) { + return response()->json(['message' => 'Backup not found locally on the server.'], 404); + } + + return new StreamedResponse(function () use ($disk, $filename) { + if (ob_get_level()) { + ob_end_clean(); + } + $stream = $disk->readStream($filename); + if ($stream === false || is_null($stream)) { + abort(500, 'Failed to open stream for the requested file.'); + } + while (! feof($stream)) { + echo fread($stream, 2048); + flush(); + } + + fclose($stream); + }, 200, [ + 'Content-Type' => 'application/gzip', + 'Content-Disposition' => 'attachment; filename="'.basename($filename).'"', + ]); + } catch (Throwable) { + return response()->json(['message' => 'Failed to download backup.'], 500); + } + })->name('download.volume-backup'); + }); Route::any('/{any}', function () { diff --git a/tests/Feature/BackupEditValidationTest.php b/tests/Feature/BackupEditValidationTest.php index fe396b5da..b0041e226 100644 --- a/tests/Feature/BackupEditValidationTest.php +++ b/tests/Feature/BackupEditValidationTest.php @@ -72,6 +72,35 @@ function createS3StorageForBackupEditValidationTest(Team|int $team, string $name session(['currentTeam' => $this->team]); }); +it('renders a highlighted enable backup button and a regular disable backup button', function () { + $view = file_get_contents(resource_path('views/livewire/project/database/backup-edit.blade.php')); + + expect($view) + ->toContain('wire:target="toggleEnabled" isHighlighted>Enable Backup') + ->toContain('wire:target="toggleEnabled">Disable Backup') + ->not->toContain('label="Backup Enabled"'); +}); + +it('enables and disables a scheduled database backup from the title action', function () { + $backup = createBackupForEditValidationTest($this->team, [ + 'enabled' => false, + 'save_s3' => false, + ]); + + $component = Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) + ->assertSet('backupEnabled', false) + ->assertSee('Enable Backup') + ->call('toggleEnabled') + ->assertSet('backupEnabled', true) + ->assertSee('Disable Backup'); + + expect($backup->refresh()->enabled)->toBeTruthy(); + + $component->call('toggleEnabled')->assertSet('backupEnabled', false); + + expect($backup->refresh()->enabled)->toBeFalsy(); +}); + it('disables S3 backup when saved without a selected S3 storage', function () { $backup = createBackupForEditValidationTest($this->team); @@ -177,6 +206,30 @@ function createS3StorageForBackupEditValidationTest(Team|int $team, string $name ->assertSee('No S3 storage available'); }); +it('allows S3 backups to be disabled when no usable storage remains', function () { + $backup = createBackupForEditValidationTest($this->team, [ + 'save_s3' => true, + 's3_storage_id' => null, + ]); + + $component = Livewire::test(BackupEdit::class, [ + 'backup' => $backup->fresh(), + 'availableS3Storages' => collect(), + ])->assertSet('saveS3', true); + + preg_match('/]*wire:model=(?:"saveS3"|saveS3))[^>]*>/', $component->html(), $matches); + expect($matches[0] ?? null)->not->toBeNull() + ->and(preg_match('/\sdisabled(?:\s|\/>)/', $matches[0]))->toBe(0); + + $component->set('saveS3', false)->call('instantSave')->assertDispatched('success'); + + expect($backup->refresh()->save_s3)->toBeFalsy() + ->and($backup->s3_storage_id)->toBeNull(); + + preg_match('/]*wire:model=(?:"saveS3"|saveS3))[^>]*>/', $component->html(), $matches); + expect(preg_match('/\sdisabled(?:\s|\/>)/', $matches[0]))->toBe(1); +}); + it('shows when S3 backups are currently disabled', function () { createS3StorageForBackupEditValidationTest($this->team); $backup = createBackupForEditValidationTest($this->team, [ diff --git a/tests/Feature/BackupSearchTest.php b/tests/Feature/BackupSearchTest.php new file mode 100644 index 000000000..c6e91abb3 --- /dev/null +++ b/tests/Feature/BackupSearchTest.php @@ -0,0 +1,121 @@ + 0]); + + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->user->teams()->attach($this->team, ['role' => 'owner']); + $this->actingAs($this->user); + session(['currentTeam' => $this->team]); +}); + +it('renders frontend-only application backup search data for volume names and frequencies', function () { + $application = createBackupSearchApplication($this->team); + $dailyVolume = createBackupSearchVolume($application, 'app-data'); + $weeklyVolume = createBackupSearchVolume($application, 'Cache-Data'); + + ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $dailyVolume->id, + 'team_id' => $this->team->id, + 'frequency' => 'daily', + ]); + ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $weeklyVolume->id, + 'team_id' => $this->team->id, + 'frequency' => '0 4 * * 0', + ]); + + $parameters = [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + ]; + + $this->get(route('project.application.backup.index', [...$parameters, 'search' => 'cache'])) + ->assertOk() + ->assertSee('Cache-Data') + ->assertSee('Volume: app-data') + ->assertSee('x-model="search"', false) + ->assertSee('x-show=', false) + ->assertSee('No scheduled backups match your search.'); +}); + +it('renders frontend-only database backup search data for database names and frequencies', function () { + $server = Server::factory()->create(['team_id' => $this->team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $database = StandalonePostgresql::create([ + 'name' => 'Orders-Primary', + 'image' => 'postgres:16-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => '0 3 * * 1', + ]); + + $parameters = [ + 'project_uuid' => $project->uuid, + 'environment_uuid' => $environment->uuid, + 'database_uuid' => $database->uuid, + ]; + + $this->get(route('project.database.backup.index', [...$parameters, 'search' => '0 3'])) + ->assertOk() + ->assertSee('

0 3 * * 1

', false) + ->assertSee('

daily

', false) + ->assertSee('x-model="search"', false) + ->assertSee('x-show=', false) + ->assertSee('No scheduled backups match your search.'); +}); + +function createBackupSearchApplication(Team $team): Application +{ + $server = Server::factory()->create(['team_id' => $team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + + return $application; +} + +function createBackupSearchVolume(Application $application, string $name): LocalPersistentVolume +{ + return LocalPersistentVolume::create([ + 'name' => $name, + 'mount_path' => '/'.str($name)->lower(), + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); +} diff --git a/tests/Feature/MobileResourceMenuTest.php b/tests/Feature/MobileResourceMenuTest.php index 7f4d6fe9d..47ef03510 100644 --- a/tests/Feature/MobileResourceMenuTest.php +++ b/tests/Feature/MobileResourceMenuTest.php @@ -131,6 +131,19 @@ function mobileActionsAreBeforeSelect(string $heading, string $actionsId, string return $actionsPosition !== false && $selectPosition !== false && $actionsPosition < $selectPosition; } +it('places database backups immediately after configuration in navigation menus', function () { + $databaseHeading = file_get_contents(resource_path('views/livewire/project/database/heading.blade.php')); + $mobileMenuItems = str($databaseHeading)->between('$databasePageItems = [', '$databaseConfigurationItems = [')->toString(); + $desktopMenuItems = str($databaseHeading)->between('class="scrollbar hidden', '')->toString(); + + foreach ([$mobileMenuItems, $desktopMenuItems] as $menuItems) { + expect(strpos($menuItems, 'Configuration')) + ->toBeLessThan(strpos($menuItems, 'Backups')) + ->and(strpos($menuItems, 'Backups'))->toBeLessThan(strpos($menuItems, 'Logs')) + ->and(strpos($menuItems, 'Logs'))->toBeLessThan(strpos($menuItems, 'Terminal')); + } +}); + it('keeps configuration sidebars hidden until desktop breakpoint', function () { expect(file_get_contents(resource_path('views/livewire/project/database/configuration.blade.php'))) ->toContain('sub-menu-wrapper hidden md:flex'); diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php new file mode 100644 index 000000000..94bde7b32 --- /dev/null +++ b/tests/Feature/VolumeBackupTest.php @@ -0,0 +1,1092 @@ +toBeTrue() + ->and(class_exists(ScheduledVolumeBackupExecution::class))->toBeTrue() + ->and(class_exists(VolumeBackupJob::class))->toBeTrue() + ->and(class_exists(VolumeBackups::class))->toBeTrue() + ->and(method_exists(LocalPersistentVolume::class, 'scheduledBackups'))->toBeTrue(); +}); + +it('provides application backup index and detail routes', function () { + expect(Route::has('project.application.backup.index'))->toBeTrue() + ->and(Route::has('project.application.backup.show'))->toBeTrue() + ->and(Route::has('download.volume-backup'))->toBeTrue(); +}); + +it('creates a scheduled backup with a preselected volume from the shared modal', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + Livewire::test(CreateScheduledVolumeBackup::class, [ + 'application' => $application, + 'selectedVolumeId' => $volume->id, + ]) + ->assertSet('volumeId', $volume->id) + ->assertSee($volume->name) + ->set('frequency', 'daily') + ->call('submit') + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + + expect($backup->local_persistent_volume_id)->toBe($volume->id) + ->and($backup->frequency)->toBe('daily') + ->and($backup->enabled)->toBeTrue(); +}); + +it('shows volume backups on the application backups pages', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $parameters = [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + ]; + + $this->get(route('project.application.backup.index', $parameters)) + ->assertOk() + ->assertSee('Scheduled Backups') + ->assertSee($volume->name); + + $this->get(route('project.application.backup.show', [...$parameters, 'backup_uuid' => $backup->uuid])) + ->assertOk() + ->assertSee('

Volume Backups

', false) + ->assertSee($volume->name); +}); + +it('shows the configure backup modal trigger instead of inline backup settings on a volume', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + Livewire::test(Show::class, [ + 'storage' => $volume, + 'resource' => $application, + ]) + ->assertSee('Configure Backup') + ->assertDontSee('Backups made while the application is writing'); +}); + +it('only shows the backup enabled badge for an enabled volume backup', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => false, + ]); + + $component = Livewire::test(Show::class, [ + 'storage' => $volume, + 'resource' => $application, + ])->assertDontSee('Backup enabled'); + + $backup->update(['enabled' => true]); + + $component + ->dispatch('refreshVolumeBackups') + ->assertSeeInOrder(['Volume Name', 'Backup enabled']); + + Livewire::test(Show::class, [ + 'storage' => $volume, + 'resource' => $application, + 'isFirst' => false, + ])->assertSeeInOrder(['Volume Name', 'Backup enabled']); +}); + +it('stores volume backup schedules and executions', function () { + expect(Schema::hasColumns('scheduled_volume_backups', [ + 'uuid', + 'local_persistent_volume_id', + 'team_id', + 's3_storage_id', + 'frequency', + 'enabled', + 'save_s3', + 'disable_local_backup', + 'pause_during_backup', + 'retention_amount_locally', + 'retention_days_locally', + 'retention_max_storage_locally', + 'retention_amount_s3', + 'retention_days_s3', + 'retention_max_storage_s3', + 'timeout', + ]))->toBeTrue() + ->and(Schema::hasColumns('scheduled_volume_backup_executions', [ + 'uuid', + 'scheduled_volume_backup_id', + 'status', + 'message', + 'size', + 'filename', + 'pause_container_ids', + 'pause_recovery_pending', + 's3_cleanup_pending', + 'finished_at', + 'local_storage_deleted', + 's3_storage_deleted', + 's3_uploaded', + ]))->toBeTrue(); +}); + +it('exposes actions to manage and run volume backups', function () { + expect(method_exists(VolumeBackups::class, 'save'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'backupNow'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'toggleEnabled'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'delete'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'cleanupFailed'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'cleanupDeleted'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'deleteBackup'))->toBeTrue(); +}); + +it('renders volume backup executions like database backup executions', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $server->settings->update(['server_timezone' => 'Europe/Budapest']); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'size' => 1024, + 'finished_at' => now(), + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSet('timezone', 'Europe/Budapest') + ->assertSeeInOrder([ + 'Scheduled Backup', + 'Save', + 'Disable Backup', + 'Backup Now', + 'Delete Backups and Schedule', + 'Persistent volume:', + 'Pause containers while creating the archive', + 'S3 Enabled', + 'Disable Local Backup', + 'S3 Storage', + 'Settings', + 'Frequency', + 'Timezone', + 'Timeout', + 'Backup Retention Settings', + 'Local Backup Retention', + 'Executions', + ]) + ->assertSee('Persistent volume:') + ->assertSee('app-data') + ->assertSee('Scheduled Backup') + ->assertSee('Save') + ->assertSee('Disable Backup') + ->assertSee('Backup Now') + ->assertSee('Delete Backups and Schedule') + ->assertSee('S3 Enabled') + ->assertSee('Disable Local Backup') + ->assertSee('S3 Storage') + ->assertSee('(currently disabled)') + ->assertSee('Timezone') + ->assertSee('Setting a value to 0 means unlimited retention.') + ->assertSee('Days to keep backups') + ->assertSee('Maximum storage (GB)') + ->assertSee('Executions') + ->assertSee('Page 1 of 1') + ->assertSee('Cleanup Failed Backups') + ->assertSee('Cleanup Deleted') + ->assertSee('Backup Availability:') + ->assertSee('Local Storage') + ->assertSee('Location: /data/coolify/backups/volumes/test/archive.tar.gz') + ->assertSee('Download') + ->assertSee('Delete') + ->assertDontSee('border border-neutral-200', false); +}); + +it('cleans up failed and fully deleted volume backup execution records', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $failed = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + ]); + $deleted = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'local_storage_deleted' => true, + 's3_storage_deleted' => true, + 's3_uploaded' => true, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('cleanupFailed') + ->assertDispatched('success') + ->call('cleanupDeleted') + ->assertDispatched('success'); + + expect($failed->fresh())->toBeNull() + ->and($deleted->fresh())->toBeNull(); +}); + +it('deletes an individual volume backup archive and execution', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + Process::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'finished_at' => now(), + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('deleteBackup', $execution->id, 'password') + ->assertDispatched('success'); + + expect($execution->fresh())->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'archive.tar.gz')); +}); + +it('prevents another team from downloading a volume backup', function () { + $backupTeam = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($backupTeam); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $backupTeam->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + ]); + $otherTeam = Team::factory()->create(); + signInForVolumeBackups($this, $otherTeam); + + $this->get(route('download.volume-backup', $execution->id))->assertForbidden(); +}); + +it('enables and disables volume backups from the title action', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSet('enabled', false) + ->assertSee('Enable Backup') + ->call('toggleEnabled') + ->assertSet('enabled', true) + ->assertSee('Disable Backup'); + + expect(ScheduledVolumeBackup::query()->sole()->enabled)->toBeTrue(); + + $component->call('toggleEnabled')->assertSet('enabled', false); + + expect(ScheduledVolumeBackup::query()->sole()->enabled)->toBeFalse(); +}); + +function createVolumeBackupApplication(Team $team): array +{ + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => $privateKey->id, + 'ip' => '203.0.113.10', + ]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + + return [$application, $volume, $server]; +} + +function signInForVolumeBackups($testCase, Team $team): User +{ + $user = User::factory()->create(); + $user->teams()->attach($team, ['role' => 'owner']); + $testCase->actingAs($user); + session(['currentTeam' => $team]); + + return $user; +} + +it('creates a local scheduled backup for a persistent volume', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSee('Scheduled Backup') + ->assertSee('Persistent volume:') + ->assertSee('inconsistent or corrupted') + ->set('frequency', 'daily') + ->set('retentionAmountLocally', 5) + ->set('retentionDaysLocally', 14) + ->set('retentionMaxStorageLocally', 1.5) + ->set('pauseDuringBackup', true) + ->call('save') + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + + expect($backup->local_persistent_volume_id)->toBe($volume->id) + ->and($backup->team_id)->toBe($team->id) + ->and($backup->frequency)->toBe('daily') + ->and($backup->retention_amount_locally)->toBe(5) + ->and($backup->retention_days_locally)->toBe(14) + ->and($backup->retention_max_storage_locally)->toBe(1.5) + ->and($backup->pause_during_backup)->toBeTrue() + ->and($backup->save_s3)->toBeFalse(); +}); + +it('only accepts a usable S3 storage owned by the current team', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $foreignStorage = S3Storage::create([ + 'name' => 'Foreign S3', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => Team::factory()->create()->id, + 'is_usable' => true, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->set('frequency', 'daily') + ->set('saveToS3', true) + ->set('s3StorageId', $foreignStorage->id) + ->call('save') + ->assertHasErrors('s3StorageId'); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('saves the database-style S3 backup controls immediately', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + + $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSet('s3StorageId', $s3Storage->id) + ->set('saveToS3', true) + ->call('instantSave') + ->set('disableLocalBackup', true) + ->set('retentionAmountS3', 10) + ->set('retentionDaysS3', 30) + ->set('retentionMaxStorageS3', 5.5) + ->call('instantSave'); + + $backup = ScheduledVolumeBackup::query()->sole(); + expect($backup->save_s3)->toBeTrue() + ->and($backup->s3_storage_id)->toBe($s3Storage->id) + ->and($backup->disable_local_backup)->toBeTrue() + ->and($backup->retention_amount_s3)->toBe(10) + ->and($backup->retention_days_s3)->toBe(30) + ->and($backup->retention_max_storage_s3)->toBe(5.5); + + $component->set('saveToS3', false)->call('instantSave'); + + expect($backup->fresh()->save_s3)->toBeFalse() + ->and($backup->fresh()->s3_storage_id)->toBeNull() + ->and($backup->fresh()->disable_local_backup)->toBeFalse(); +}); + +it('allows volume S3 backups to be disabled when no usable storage remains', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => null, + ]); + + $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSet('saveToS3', true); + + preg_match('/]*wire:model=(?:"saveToS3"|saveToS3))[^>]*>/', $component->html(), $matches); + expect($matches[0] ?? null)->not->toBeNull() + ->and(preg_match('/\sdisabled(?:\s|\/>)/', $matches[0]))->toBe(0); + + $component->set('saveToS3', false)->call('instantSave')->assertDispatched('success'); + + expect($backup->refresh()->save_s3)->toBeFalse() + ->and($backup->s3_storage_id)->toBeNull(); + + preg_match('/]*wire:model=(?:"saveToS3"|saveToS3))[^>]*>/', $component->html(), $matches); + expect(preg_match('/\sdisabled(?:\s|\/>)/', $matches[0]))->toBe(1); +}); + +it('disables S3 volume backups when the storage is deleted', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/s3.tar.gz', + 's3_uploaded' => true, + 's3_cleanup_pending' => true, + ]); + + $s3Storage->delete(); + + expect($backup->fresh()->save_s3)->toBeFalse() + ->and($backup->fresh()->s3_storage_id)->toBeNull() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue() + ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse(); +}); + +it('queues a manual backup before a schedule has been saved', function () { + Queue::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('backupNow') + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + + expect($backup->enabled)->toBeFalse(); + Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup)); +}); + +it('deletes local archives before deleting a volume backup schedule', function () { + Process::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'size' => 128, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('success'); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'archive.tar.gz')); +}); + +it('refuses to delete a schedule while its backup is running', function () { + Process::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'running', + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('error'); + + expect($backup->fresh())->not->toBeNull(); + Process::assertNothingRan(); +}); + +it('uses the backup operation lock while deleting a schedule', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $lock = Cache::lock(VolumeBackupJob::lockKey($backup->id), 60); + $lock->get(); + + try { + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('error'); + + expect($backup->fresh())->not->toBeNull(); + } finally { + $lock->release(); + } +}); + +it('prevents a volume with tracked backup archives from being deleted', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + + expect(fn () => $volume->delete())->toThrow(QueryException::class); + expect($volume->fresh())->not->toBeNull(); +}); + +it('marks a running execution failed even when the job instance lost its execution state', function () { + Process::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'running', + 'filename' => '/data/coolify/backups/volumes/test/timed-out.tar.gz', + ]); + + (new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out')); + + expect($execution->fresh()->status)->toBe('failed') + ->and($execution->fresh()->message)->toBe('Worker timed out') + ->and($execution->fresh()->finished_at)->not->toBeNull() + ->and($execution->fresh()->filename)->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'timed-out.tar.gz')); +}); + +it('archives a named volume on its server', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + 'retention_amount_locally' => 7, + 'retention_amount_s3' => 7, + 'timeout' => 3600, + ]); + + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + $job = new VolumeBackupJob($backup); + $middleware = $job->middleware(); + $job->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + + expect($middleware)->toHaveCount(1) + ->and($middleware[0])->toBeInstanceOf(WithoutOverlapping::class) + ->and($job->queue)->toBe(crons_queue()) + ->and($execution->status)->toBe('success') + ->and($execution->size)->toBe(128) + ->and($execution->filename)->toEndWith('.tar.gz'); + + Process::assertRan(fn ($process) => str_contains($process->command, 'docker volume inspect') + && str_contains($process->command, 'docker run --rm --name ') + && str_contains($process->command, 'app-data:/volume:ro') + && str_contains($process->command, 'tar -czf -') + && str_contains($process->command, '> ') + && str_contains($process->command, '.tar.gz') + && ! str_contains($process->command, ':/backup')); +}); + +it('removes local volume backups older than the configured retention days', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'retention_amount_locally' => 0, + 'retention_days_locally' => 7, + 'retention_max_storage_locally' => 0, + ]); + $expiredExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/expired.tar.gz', + 'size' => 64, + ]); + ScheduledVolumeBackupExecution::query()->whereKey($expiredExecution)->update(['created_at' => now()->subDays(8)]); + $recentExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/recent.tar.gz', + 'size' => 64, + ]); + ScheduledVolumeBackupExecution::query()->whereKey($recentExecution)->update(['created_at' => now()->subDay()]); + + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + expect($expiredExecution->fresh())->toBeNull() + ->and($recentExecution->fresh())->not->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'expired.tar.gz')); +}); + +it('removes oldest local volume backups over the configured storage limit', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0.00000015, + ]); + $oldExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/over-limit.tar.gz', + 'size' => 64, + ]); + ScheduledVolumeBackupExecution::query()->whereKey($oldExecution)->update(['created_at' => now()->subDay()]); + + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + expect($oldExecution->fresh())->toBeNull() + ->and($backup->executions()->where('status', 'success')->count())->toBe(1); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'over-limit.tar.gz')); +}); + +it('keeps a successful backup successful when retention cleanup fails', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'disable_local_backup' => true, + 'retention_amount_locally' => 1, + ]); + $oldExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/old.tar.gz', + 'size' => 64, + 'created_at' => now()->subDay(), + ]); + + Process::fake([ + '*rm -f*' => Process::result(errorOutput: 'permission denied', exitCode: 1), + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $latestExecution = $backup->executions()->first(); + + expect($latestExecution->status)->toBe('success') + ->and($latestExecution->filename)->not->toBeNull() + ->and($oldExecution->fresh()->local_storage_deleted)->toBeFalse(); +}); + +it('pauses and resumes containers that use the volume when requested', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'pause_during_backup' => true, + ]); + + Process::fake([ + '*docker ps -q*' => "abc123\ndef456", + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + Process::assertRan(fn ($process) => str_contains($process->command, 'trap cleanup EXIT HUP INT TERM') + && str_contains($process->command, 'docker pause') + && str_contains($process->command, 'docker unpause') + && str_contains($process->command, 'state_file=')); + Process::assertRan(fn ($process) => str_contains($process->command, '{{println .Source}}{{println .Name}}') + && str_contains($process->command, "grep -Fqx -- 'app-data'") + && str_contains($process->command, '{{.State.Paused}}') + && str_contains($process->command, 'continue')); +}); + +it('finds containers using a bind mounted host path before pausing', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $volume->update(['host_path' => '/srv/app-data']); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'pause_during_backup' => true, + ]); + + Process::fake([ + '*docker ps -q*' => 'abc123', + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + Process::assertRan(fn ($process) => str_contains($process->command, '{{println .Source}}{{println .Name}}') + && str_contains($process->command, "grep -Fqx -- '/srv/app-data'")); +}); + +it('retries container recovery when a timed out backup left a container paused', function () { + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'running', + 'pause_recovery_pending' => true, + ]); + Process::fake([ + '*cat *coolify-volume-backup-*' => 'abc123', + '*docker unpause*' => Process::result(errorOutput: 'daemon unavailable', exitCode: 1), + ]); + + (new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out')); + + expect($execution->fresh()->status)->toBe('failed') + ->and($execution->fresh()->message)->toContain('Container recovery failed') + ->and($execution->fresh()->pause_container_ids)->toBe(['abc123']) + ->and($execution->fresh()->pause_recovery_pending)->toBeTrue(); + Queue::assertPushed(VolumeBackupRecoveryJob::class); +}); + +it('resumes containers and removes a partial archive when archiving fails', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => 'daily', + 'pause_during_backup' => true, + ]); + Process::fake([ + '*docker ps -q*' => 'abc123', + '*trap cleanup EXIT HUP INT TERM*' => Process::result(errorOutput: 'archive failed', exitCode: 1), + '*' => '', + ]); + + expect(fn () => (new VolumeBackupJob($backup))->handle())->toThrow(RuntimeException::class); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + expect($execution->status)->toBe('failed') + ->and($execution->pause_container_ids)->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, '.tar.gz')); +}); + +it('throws when the S3 filesystem reports that deletion failed', function () { + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + $s3 = new S3Storage([ + 'key' => 'key', + 'secret' => 'secret', + 'region' => 'us-east-1', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + ]); + + expect(fn () => deleteBackupsS3('archive.tar.gz', $s3)) + ->toThrow(RuntimeException::class, 'could not be deleted'); +}); + +it('cleans an interrupted S3 upload and coordinates recovery with the backup lock', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('build')->once()->andReturn($disk); + $job = new VolumeBackupRecoveryJob($execution); + + expect($job->middleware()[0]->getLockKey($job))->toBe(VolumeBackupJob::lockKey($backup->id)); + $job->handle(); + + expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + +it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () { + Queue::fake(); + Process::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'running', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + + (new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out')); + + expect($execution->fresh()->filename)->toBe('/data/coolify/backups/volumes/test/interrupted.tar.gz') + ->and($execution->fresh()->s3_cleanup_pending)->toBeTrue(); + Queue::assertPushed(VolumeBackupRecoveryJob::class); +}); + +it('dispatches due scheduled volume backups', function () { + config(['constants.coolify.self_hosted' => true]); + Carbon::setTestNow('2026-07-15 12:00:00'); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $server->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + ]); + expect($server->fresh()->isFunctional())->toBeTrue(); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => '* * * * *', + 'enabled' => true, + ]); + Cache::forget("scheduled-volume-backup:{$backup->id}"); + expect($backup->server()?->isFunctional())->toBeTrue(); + + (new ScheduledJobManager)->handle(); + + Queue::assertPushed( + VolumeBackupJob::class, + fn (VolumeBackupJob $job) => $job->backup->is($backup), + ); +}); + +it('dispatches pending recovery without starting another volume backup', function () { + config(['constants.coolify.self_hosted' => true]); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $server->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + ]); + $backup = ScheduledVolumeBackup::create([ + 'local_persistent_volume_id' => $volume->id, + 'team_id' => $team->id, + 'frequency' => '* * * * *', + 'enabled' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + 'pause_recovery_pending' => true, + ]); + + (new ScheduledJobManager)->handle(); + + Queue::assertPushed( + VolumeBackupRecoveryJob::class, + fn (VolumeBackupRecoveryJob $job) => $job->execution->is($execution), + ); + Queue::assertNotPushed(VolumeBackupJob::class); +});