rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(backups): pin S3 per volume execution and harden deletes

Browse source 
Store s3_storage_id on scheduled volume backup executions so retention
and recovery use the S3 that received the upload. Extract
DeleteScheduledVolumeBackup for UI and resource deletion, and isolate
database backup retention failures so cleanup errors do not fail the job.
This commit is contained in:
Andras Bacsai 2026-07-16 14:30:58 +02:00
parent 8cfa41a7d9
commit ac36a59088
14 changed files with 556 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

71
app/Actions/Shared/DeleteScheduledVolumeBackup.php Normal file
View file

@ -0,0 +1,71 @@
<?php
namespace App\Actions\Shared;
use App\Jobs\VolumeBackupJob;
use App\Models\ScheduledVolumeBackup;
use App\Models\Server;
use Illuminate\Support\Facades\Cache;
use Lorisleiva\Actions\Concerns\AsAction;
class DeleteScheduledVolumeBackup
{
use AsAction;
public function handle(ScheduledVolumeBackup $backup, ?Server $server = null): void
{
$lock = Cache::lock(VolumeBackupJob::lockKey($backup->id), $backup->timeout + 300);
if (! $lock->get()) {
throw new \RuntimeException('Wait for the queued or running storage backup to finish before deleting this schedule.');
}
try {
if ($backup->executions()
->where(fn ($query) => $query
->where('status', 'running')
->orWhere('stop_recovery_pending', true)
->orWhere('s3_cleanup_pending', true))
->exists()) {
throw new \RuntimeException('Wait for the running storage backup and recovery operations to finish before deleting this schedule.');
}
$localFilenames = $backup->executions()
->where('local_storage_deleted', false)
->pluck('filename')
->filter()
->all();
if ($localFilenames !== []) {
$server ??= $backup->server();
if (! $server) {
throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.');
}
deleteBackupsLocally($localFilenames, $server, throwError: true);
}
$s3Executions = $backup->executions()
->with('s3')
->where('s3_uploaded', true)
->where('s3_storage_deleted', false)
->get();
foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {
$s3 = $executions->first()->s3;
if (! $s3) {
throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');
}
$filenames = $executions->pluck('filename')->filter()->all();
if ($filenames !== []) {
deleteBackupsS3($filenames, $s3);
}
}
$backup->delete();
} finally {
$lock->release();
}
}
}

15
app/Jobs/DatabaseBackupJob.php
View file

@ -497,7 +497,7 @@ public function handle(): void
}
}
if ($this->backup_log && $this->backup_log->status === 'success') {
removeOldBackups($this->backup);
$this->removeExpiredBackups();
}
} catch (Throwable $e) {
throw $e;
@ -513,6 +513,19 @@ public function handle(): void
}
}
private function removeExpiredBackups(): void
{
try {
removeOldBackups($this->backup);
} catch (Throwable $exception) {
Log::channel('scheduled-errors')->warning('Database backup retention cleanup failed', [
'backup_id' => $this->backup->id,
'execution_id' => $this->backup_log?->id,
'error' => $exception->getMessage(),
]);
}
}
private function backup_standalone_mongodb(string $databaseWithCollections): void
{
try {

33
app/Jobs/DeleteResourceJob.php
View file

@ -7,7 +7,10 @@
use App\Actions\Server\CleanupDocker;
use App\Actions\Service\DeleteService;
use App\Actions\Service\StopService;
use App\Actions\Shared\DeleteScheduledVolumeBackup;
use App\Enums\ApplicationDeploymentStatus;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\ApplicationPreview;
use App\Models\Service;
use App\Models\StandaloneClickhouse;
@ -42,6 +45,10 @@ public function __construct(
public function handle()
{
if (! $this->resource instanceof ApplicationPreview) {
$this->deleteScheduledVolumeBackups();
}
try {
// Handle ApplicationPreview instances separately
if ($this->resource instanceof ApplicationPreview) {
@ -113,6 +120,24 @@ public function handle()
}
}
private function deleteScheduledVolumeBackups(): void
{
$server = data_get($this->resource, 'server') ?? data_get($this->resource, 'destination.server');
$resources = $this->resource instanceof Service
? $this->resource->applications()->get()->concat($this->resource->databases()->get())
: collect([$this->resource]);
foreach ($resources as $resource) {
$storages = $resource->persistentStorages()->get()->concat($resource->fileStorages()->get());
foreach ($storages as $storage) {
foreach ($storage->scheduledBackups()->get() as $backup) {
DeleteScheduledVolumeBackup::run($backup, $server);
}
}
}
}
private function deleteApplicationPreview()
{
$application = $this->resource->application;
@ -125,11 +150,11 @@ private function deleteApplicationPreview()
}
// Cancel any active deployments for this PR (same logic as API cancel_deployment)
$activeDeployments = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id)
$activeDeployments = ApplicationDeploymentQueue::where('application_id', $application->id)
->where('pull_request_id', $pull_request_id)
->whereIn('status', [
\App\Enums\ApplicationDeploymentStatus::QUEUED->value,
\App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value,
ApplicationDeploymentStatus::QUEUED->value,
ApplicationDeploymentStatus::IN_PROGRESS->value,
])
->get();
@ -137,7 +162,7 @@ private function deleteApplicationPreview()
try {
// Mark deployment as cancelled
$activeDeployment->update([
'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
// Add cancellation log entry

22
app/Jobs/VolumeBackupJob.php
View file

@ -63,7 +63,9 @@ public function handle(): void
throw new \RuntimeException('The storage backup resource, team, or server no longer exists.');
}
$this->execution = $this->backup->executions()->create();
$this->execution = $this->backup->executions()->create([
's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null,
]);
BackupCreated::dispatch($team->id);
$backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid;
@ -352,6 +354,7 @@ private function removeExpiredBackups(Server $server): void
if ($this->backup->save_s3 && $this->backup->s3) {
$s3Executions = $this->backup->executions()
->with('s3')
->where('status', 'success')
->where('s3_uploaded', true)
->where('s3_storage_deleted', false)
@ -363,11 +366,18 @@ private function removeExpiredBackups(Server $server): void
$this->backup->retention_max_storage_s3,
);
$filenames = $s3Executions->pluck('filename')->filter()->all();
if ($filenames !== []) {
deleteBackupsS3($filenames, $this->backup->s3);
$this->backup->executions()->whereKey($s3Executions->pluck('id')->all())
->update(['s3_storage_deleted' => true]);
foreach ($s3Executions->groupBy('s3_storage_id') as $executions) {
$s3 = $executions->first()->s3;
if (! $s3) {
throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.');
}
$filenames = $executions->pluck('filename')->filter()->all();
if ($filenames !== []) {
deleteBackupsS3($filenames, $s3);
$this->backup->executions()->whereKey($executions->pluck('id')->all())
->update(['s3_storage_deleted' => true]);
}
}
}

4
app/Jobs/VolumeBackupRecoveryJob.php
View file

@ -96,8 +96,8 @@ private static function recoverContainers(ScheduledVolumeBackupExecution $execut
public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void
{
$execution->loadMissing('scheduledVolumeBackup.s3');
$s3 = $execution->scheduledVolumeBackup?->s3;
$execution->loadMissing('s3');
$s3 = $execution->s3;
if (! $s3 || blank($execution->filename)) {
throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.');

8
app/Livewire/Project/Database/ScheduledBackups.php
View file

@ -3,6 +3,7 @@
namespace App\Livewire\Project\Database;
use App\Models\ServiceDatabase;
use Illuminate\Contracts\View\View;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Livewire\Component;
@ -63,4 +64,11 @@ public function refreshScheduledBackups(?int $id = null): void
$this->database->refresh();
$this->dispatch('refreshScheduledBackups');
}
public function render(): View
{
$this->database->loadMissing('scheduledBackups.s3');
return view('livewire.project.database.scheduled-backups');
}
}

55
app/Livewire/Project/Shared/Storages/VolumeBackups.php
View file

@ -2,6 +2,7 @@
namespace App\Livewire\Project\Shared\Storages;
use App\Actions\Shared\DeleteScheduledVolumeBackup;
use App\Jobs\VolumeBackupJob;
use App\Models\LocalFileVolume;
use App\Models\LocalPersistentVolume;
@ -9,7 +10,6 @@
use App\Models\ScheduledVolumeBackup;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
use Illuminate\Support\Collection;
use Illuminate\Support\Facades\Cache;
use Livewire\Component;
use Livewire\WithPagination;
use Throwable;
@ -214,57 +214,8 @@ public function delete(?string $password = null, array $selectedActions = [])
return false;
}
$lock = Cache::lock(VolumeBackupJob::lockKey($this->backup->id), $this->backup->timeout + 300);
if (! $lock->get()) {
$this->dispatch('error', 'Wait for the queued or running storage backup to finish before deleting this schedule.');
return false;
}
try {
if ($this->backup->executions()
->where(fn ($query) => $query
->where('status', 'running')
->orWhere('stop_recovery_pending', true)
->orWhere('s3_cleanup_pending', true))
->exists()) {
$this->dispatch('error', 'Wait for the running storage backup and container recovery to finish before deleting this schedule.');
return false;
}
$localFilenames = $this->backup->executions()
->where('local_storage_deleted', false)
->pluck('filename')
->filter()
->all();
$server = $this->backup->server();
if ($localFilenames !== []) {
if (! $server) {
throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.');
}
deleteBackupsLocally($localFilenames, $server, throwError: true);
}
$s3Filenames = $this->backup->executions()
->where('s3_uploaded', true)
->where('s3_storage_deleted', false)
->pluck('filename')
->filter()
->all();
if ($s3Filenames !== []) {
if (! $this->backup->s3) {
throw new \RuntimeException('The S3 storage is unavailable, so remote backup archives cannot be deleted.');
}
deleteBackupsS3($s3Filenames, $this->backup->s3);
}
$this->backup->delete();
DeleteScheduledVolumeBackup::run($this->backup);
$this->backup = null;
$this->dispatch('success', 'Storage backup schedule and archives deleted.');
$this->redirectRoute('project.application.backup.index', [
@ -278,8 +229,6 @@ public function delete(?string $password = null, array $selectedActions = [])
$this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage());
return false;
} finally {
$lock->release();
}
}

5
app/Models/S3Storage.php
View file

@ -69,13 +69,12 @@ protected static function boot(): void
'save_s3' => false,
's3_storage_id' => null,
]);
$volumeBackupIds = ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->pluck('id');
ScheduledVolumeBackupExecution::whereIn('scheduled_volume_backup_id', $volumeBackupIds)
ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id)
->update([
's3_storage_deleted' => true,
's3_cleanup_pending' => false,
]);
ScheduledVolumeBackup::whereKey($volumeBackupIds)->update([
ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([
'save_s3' => false,
's3_storage_id' => null,
]);

6
app/Models/ScheduledVolumeBackupExecution.php
View file

@ -9,6 +9,7 @@ class ScheduledVolumeBackupExecution extends BaseModel
protected $fillable = [
'uuid',
'scheduled_volume_backup_id',
's3_storage_id',
'status',
'message',
'size',
@ -40,4 +41,9 @@ public function scheduledVolumeBackup(): BelongsTo
{
return $this->belongsTo(ScheduledVolumeBackup::class);
}
public function s3(): BelongsTo
{
return $this->belongsTo(S3Storage::class, 's3_storage_id');
}
}

1
database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php
View file

@ -15,6 +15,7 @@ public function up(): void
$table->id();
$table->string('uuid')->unique();
$table->foreignId('scheduled_volume_backup_id')->constrained()->cascadeOnDelete();
$table->foreignId('s3_storage_id')->nullable()->constrained()->nullOnDelete();
$table->enum('status', ['success', 'failed', 'running'])->default('running');
$table->longText('message')->nullable();
$table->unsignedBigInteger('size')->default(0);

17
resources/views/livewire/project/database/scheduled-backups.blade.php
View file

@ -3,11 +3,12 @@
backups: @js($database->scheduledBackups->map(fn ($backup) => [
'name' => strtolower($database->name),
'frequency' => strtolower($backup->frequency),
's3_storage' => strtolower($backup->s3?->name ?? ''),
])->values()),
hasMatches() {
const query = this.search.toLowerCase();
return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query));
return this.backups.some((backup) => backup.name.includes(query) || backup.frequency.includes(query) || backup.s3_storage.includes(query));
},
}">
<div class="flex flex-col gap-2">
@ -31,14 +32,14 @@
@else
<div class="max-w-md pb-4">
<x-forms.input id="null" type="search" x-model="search"
placeholder="Search by database name or frequency..." />
placeholder="Search by database name, frequency, or S3 storage..." />
</div>
<div x-cloak x-show="search !== '' && backups.length > 0 && !hasMatches()">
No scheduled backups match your search.
</div>
@forelse($database->scheduledBackups as $backup)
@if ($type == 'database')
<a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase())" @class([
<a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase()) || @js(strtolower($backup->s3?->name ?? '')).includes(search.toLowerCase())" @class([
'flex flex-col border-l-2 transition-colors p-4 cursor-pointer bg-white hover:bg-gray-100 dark:bg-coolgray-100 dark:hover:bg-coolgray-200 text-black dark:text-white',
'border-blue-500/50 border-dashed' =>
$backup->latest_log &&
@ -112,18 +113,18 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray-
@endif
@endif
@if ($backup->save_s3)
• S3: Enabled
• S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
@endif
@else
Last Run: Never • Total Executions: 0
@if ($backup->save_s3)
• S3: Enabled
• S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
@endif
@endif
</div>
</a>
@else
<a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase())" @class([
<a x-show="search === '' || @js(strtolower($database->name)).includes(search.toLowerCase()) || @js(strtolower($backup->frequency)).includes(search.toLowerCase()) || @js(strtolower($backup->s3?->name ?? '')).includes(search.toLowerCase())" @class([
'flex flex-col border-l-2 transition-colors p-4 cursor-pointer bg-white hover:bg-gray-100 dark:bg-coolgray-100 dark:hover:bg-coolgray-200 text-black dark:text-white',
'border-blue-500/50 border-dashed' =>
$backup->latest_log &&
@ -197,7 +198,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray-
@endif
@endif
@if ($backup->save_s3)
• S3: Enabled
• S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
@endif
<br>Total Executions: {{ $backup->executions()->count() }}
@php
@ -217,7 +218,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray-
@else
Last Run: Never • Total Executions: 0
@if ($backup->save_s3)
• S3: Enabled
• S3: {{ $backup->s3?->name ?? 'Storage unavailable' }}
@endif
@endif
</div>

58
tests/Feature/BackupSearchTest.php
View file

@ -5,6 +5,7 @@
use App\Models\InstanceSettings;
use App\Models\LocalPersistentVolume;
use App\Models\Project;
use App\Models\S3Storage;
use App\Models\Server;
use App\Models\StandaloneDocker;
use App\Models\StandalonePostgresql;
@ -70,9 +71,21 @@
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
]);
$storage = S3Storage::create([
'name' => 'Archive-Bucket',
'region' => 'us-east-1',
'key' => 'test-key',
'secret' => 'test-secret',
'bucket' => 'test-bucket',
'endpoint' => 'https://s3.example.com',
'is_usable' => true,
'team_id' => $this->team->id,
]);
$database->scheduledBackups()->create([
'team_id' => $this->team->id,
'frequency' => 'daily',
'save_s3' => true,
's3_storage_id' => $storage->id,
]);
$database->scheduledBackups()->create([
'team_id' => $this->team->id,
@ -89,11 +102,56 @@
->assertOk()
->assertSee('<h3 class="font-semibold">0 3 * * 1</h3>', false)
->assertSee('<h3 class="font-semibold">daily</h3>', false)
->assertSee('S3: Archive-Bucket')
->assertSee('archive-bucket', false)
->assertSee('backup.s3_storage.includes(query)', false)
->assertSee('Search by database name, frequency, or S3 storage...')
->assertSee('x-model="search"', false)
->assertSee('x-show=', false)
->assertSee('No scheduled backups match your search.');
});
it('renders unavailable S3 storage only for S3-enabled database backups', function () {
$server = Server::factory()->create(['team_id' => $this->team->id]);
$destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail();
$project = Project::factory()->create(['team_id' => $this->team->id]);
$environment = Environment::factory()->create(['project_id' => $project->id]);
$database = StandalonePostgresql::create([
'name' => 'Orders-Primary',
'image' => 'postgres:16-alpine',
'postgres_user' => 'postgres',
'postgres_password' => 'password',
'postgres_db' => 'postgres',
'environment_id' => $environment->id,
'destination_id' => $destination->id,
'destination_type' => $destination->getMorphClass(),
]);
$database->scheduledBackups()->create([
'team_id' => $this->team->id,
'frequency' => 'daily',
'save_s3' => true,
's3_storage_id' => null,
]);
$database->scheduledBackups()->create([
'team_id' => $this->team->id,
'frequency' => 'weekly',
'save_s3' => false,
's3_storage_id' => null,
]);
$parameters = [
'project_uuid' => $project->uuid,
'environment_uuid' => $environment->uuid,
'database_uuid' => $database->uuid,
];
$response = $this->get(route('project.database.backup.index', $parameters))
->assertOk()
->assertSee('S3: Storage unavailable');
expect(substr_count($response->getContent(), 'S3:'))->toBe(1);
});
function createBackupSearchApplication(Team $team): Application
{
$server = Server::factory()->create(['team_id' => $team->id]);

52
tests/Feature/Jobs/DatabaseBackupJobTest.php
View file

@ -7,6 +7,7 @@
use App\Models\Team;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\Storage;
uses(RefreshDatabase::class);
@ -238,6 +239,57 @@
expect($log->message)->toContain('Some real failure');
});
test('retention cleanup failure does not fail a successful database backup', function () {
$team = Team::factory()->create();
$s3 = S3Storage::create([
'name' => 'Test S3',
'region' => 'us-east-1',
'key' => 'test-key',
'secret' => 'test-secret',
'bucket' => 'test-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
]);
$backup = ScheduledDatabaseBackup::create([
'frequency' => '0 0 * * *',
'save_s3' => true,
'disable_local_backup' => true,
's3_storage_id' => $s3->id,
'database_type' => 'App\Models\StandalonePostgresql',
'database_id' => 1,
'team_id' => $team->id,
'database_backup_retention_amount_s3' => 1,
]);
$oldExecution = ScheduledDatabaseBackupExecution::create([
'uuid' => 'old-backup',
'database_name' => 'database',
'filename' => '/backup/old.dmp',
'scheduled_database_backup_id' => $backup->id,
'status' => 'success',
's3_uploaded' => true,
'local_storage_deleted' => true,
'created_at' => now()->subDay(),
]);
ScheduledDatabaseBackupExecution::create([
'uuid' => 'new-backup',
'database_name' => 'database',
'filename' => '/backup/new.dmp',
'scheduled_database_backup_id' => $backup->id,
'status' => 'success',
's3_uploaded' => true,
'local_storage_deleted' => true,
]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse();
Storage::shouldReceive('build')->once()->andReturn($disk);
$job = new DatabaseBackupJob($backup);
expect(fn () => (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job))
->not->toThrow(Throwable::class);
expect($oldExecution->fresh()->s3_storage_deleted)->toBeFalse();
});
test('s3 storage has scheduled backups relationship', function () {
$team = Team::factory()->create();

286
tests/Feature/VolumeBackupTest.php
View file

@ -1,5 +1,6 @@
<?php
use App\Jobs\DeleteResourceJob;
use App\Jobs\ScheduledJobManager;
use App\Jobs\VolumeBackupJob;
use App\Jobs\VolumeBackupRecoveryJob;
@ -467,6 +468,11 @@
]))->toBeTrue();
});
it('records the S3 storage used by each volume backup execution', function () {
expect(Schema::hasColumn('scheduled_volume_backup_executions', 's3_storage_id'))->toBeTrue()
->and((new ScheduledVolumeBackupExecution)->s3())->not->toBeNull();
});
it('exposes actions to manage and run volume backups', function () {
expect(method_exists(VolumeBackups::class, 'save'))->toBeTrue()
->and(method_exists(VolumeBackups::class, 'backupNow'))->toBeTrue()
@ -998,6 +1004,7 @@ function signInForVolumeBackups($testCase, Team $team): User
]);
$execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $s3Storage->id,
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/s3.tar.gz',
's3_uploaded' => true,
@ -1012,6 +1019,50 @@ function signInForVolumeBackups($testCase, Team $team): User
->and($execution->fresh()->s3_cleanup_pending)->toBeFalse();
});
it('marks historical executions deleted when their recorded S3 storage is removed', function () {
$team = Team::factory()->create();
[$application, $volume] = createVolumeBackupApplication($team);
$originalStorage = S3Storage::create([
'name' => 'Original storage',
'region' => 'us-east-1',
'key' => 'original-key',
'secret' => 'secret',
'bucket' => 'original-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
]);
$newStorage = S3Storage::create([
'name' => 'New storage',
'region' => 'us-east-1',
'key' => 'new-key',
'secret' => 'secret',
'bucket' => 'new-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
]);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
's3_storage_id' => $newStorage->id,
'frequency' => 'daily',
'save_s3' => true,
]);
$execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $originalStorage->id,
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz',
's3_uploaded' => true,
's3_cleanup_pending' => true,
]);
$originalStorage->delete();
expect($backup->fresh()->s3_storage_id)->toBe($newStorage->id)
->and($execution->fresh()->s3_storage_id)->toBeNull()
->and($execution->fresh()->s3_storage_deleted)->toBeTrue()
->and($execution->fresh()->s3_cleanup_pending)->toBeFalse();
});
it('queues a manual backup before a schedule has been saved', function () {
Queue::fake();
$team = Team::factory()->create();
@ -1100,6 +1151,61 @@ function signInForVolumeBackups($testCase, Team $team): User
&& str_contains($process->command, 'archive.tar.gz'));
});
it('deletes S3 archives from the storage recorded on each execution', function () {
$team = Team::factory()->create();
signInForVolumeBackups($this, $team);
[$application, $volume] = createVolumeBackupApplication($team);
$originalStorage = S3Storage::create([
'name' => 'Original storage',
'region' => 'us-east-1',
'key' => 'original-key',
'secret' => 'secret',
'bucket' => 'original-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$newStorage = S3Storage::create([
'name' => 'New storage',
'region' => 'us-east-1',
'key' => 'new-key',
'secret' => 'secret',
'bucket' => 'new-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
's3_storage_id' => $newStorage->id,
'frequency' => 'daily',
'save_s3' => true,
]);
ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $originalStorage->id,
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz',
'local_storage_deleted' => true,
's3_uploaded' => true,
]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')
->once()
->with(['/data/coolify/backups/volumes/test/historical.tar.gz'])
->andReturnTrue();
Storage::shouldReceive('build')
->once()
->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key'))
->andReturn($disk);
Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application])
->call('delete', 'password')
->assertDispatched('success');
expect($backup->fresh())->toBeNull();
});
it('refuses to delete a schedule while its backup is running', function () {
Process::fake();
$team = Team::factory()->create();
@ -1156,6 +1262,31 @@ function signInForVolumeBackups($testCase, Team $team): User
expect($volume->fresh())->not->toBeNull();
});
it('deletes disabled volume backup archives before deleting their application', function () {
Process::fake();
Queue::fake();
$team = Team::factory()->create();
[$application, $volume] = createVolumeBackupApplication($team);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
'frequency' => 'daily',
'enabled' => false,
]);
$execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/application-delete.tar.gz',
]);
$application->delete();
(new DeleteResourceJob($application))->handle();
expect($backup->fresh())->toBeNull()
->and($execution->fresh())->toBeNull();
Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f')
&& str_contains($process->command, 'application-delete.tar.gz'));
});
it('marks a running execution failed even when the job instance lost its execution state', function () {
Process::fake();
$team = Team::factory()->create();
@ -1222,6 +1353,46 @@ function signInForVolumeBackups($testCase, Team $team): User
&& ! str_contains($process->command, ':/backup'));
});
it('keeps the upload destination on the volume backup execution', function () {
config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
$team = Team::factory()->create();
[$application, $volume] = createVolumeBackupApplication($team);
$s3Storage = S3Storage::create([
'name' => 'Execution destination',
'region' => 'us-east-1',
'key' => 'key',
'secret' => 'secret',
'bucket' => 'bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
'frequency' => 'daily',
'save_s3' => true,
's3_storage_id' => $s3Storage->id,
]);
$sshDisk = Storage::fake('ssh-keys');
$disk = Mockery::mock();
$disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
$disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
Process::fake([
'*du -b*' => '128',
'*' => '',
]);
(new VolumeBackupJob($backup))->handle();
$execution = ScheduledVolumeBackupExecution::query()->sole();
expect($execution->s3_storage_id)->toBe($s3Storage->id)
->and($execution->s3->is($s3Storage))->toBeTrue();
});
it('removes local volume backups older than the configured retention days', function () {
config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
@ -1295,6 +1466,70 @@ function signInForVolumeBackups($testCase, Team $team): User
&& str_contains($process->command, 'over-limit.tar.gz'));
});
it('removes retained S3 archives from the storage recorded on each execution', function () {
$team = Team::factory()->create();
[$application, $volume, $server] = createVolumeBackupApplication($team);
$originalStorage = S3Storage::create([
'name' => 'Original storage',
'region' => 'us-east-1',
'key' => 'original-key',
'secret' => 'secret',
'bucket' => 'original-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$newStorage = S3Storage::create([
'name' => 'New storage',
'region' => 'us-east-1',
'key' => 'new-key',
'secret' => 'secret',
'bucket' => 'new-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
's3_storage_id' => $newStorage->id,
'frequency' => 'daily',
'save_s3' => true,
'retention_amount_locally' => 0,
'retention_days_locally' => 0,
'retention_max_storage_locally' => 0,
'retention_amount_s3' => 1,
'retention_days_s3' => 0,
'retention_max_storage_s3' => 0,
]);
$oldExecution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $originalStorage->id,
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/old.tar.gz',
's3_uploaded' => true,
'created_at' => now()->subDay(),
]);
ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $newStorage->id,
'status' => 'success',
'filename' => '/data/coolify/backups/volumes/test/new.tar.gz',
's3_uploaded' => true,
]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue();
Storage::shouldReceive('build')
->once()
->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key'))
->andReturn($disk);
$job = new VolumeBackupJob($backup);
$method = (new ReflectionClass($job))->getMethod('removeExpiredBackups');
$method->invoke($job, $server);
expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue();
});
it('keeps a successful backup successful when retention cleanup fails', function () {
config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
@ -1511,6 +1746,7 @@ function signInForVolumeBackups($testCase, Team $team): User
]);
$execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $s3Storage->id,
'status' => 'failed',
'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
's3_cleanup_pending' => true,
@ -1527,6 +1763,55 @@ function signInForVolumeBackups($testCase, Team $team): User
->and($execution->fresh()->s3_storage_deleted)->toBeTrue();
});
it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () {
$team = Team::factory()->create();
[$application, $volume] = createVolumeBackupApplication($team);
$originalStorage = S3Storage::create([
'name' => 'Original storage',
'region' => 'us-east-1',
'key' => 'original-key',
'secret' => 'secret',
'bucket' => 'original-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$newStorage = S3Storage::create([
'name' => 'New storage',
'region' => 'us-east-1',
'key' => 'new-key',
'secret' => 'secret',
'bucket' => 'new-bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
's3_storage_id' => $newStorage->id,
'frequency' => 'daily',
'save_s3' => true,
]);
$execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $originalStorage->id,
'status' => 'failed',
'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
's3_cleanup_pending' => true,
]);
$disk = Mockery::mock();
$disk->shouldReceive('delete')->once()->andReturnTrue();
Storage::shouldReceive('build')
->once()
->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key'))
->andReturn($disk);
VolumeBackupRecoveryJob::cleanupS3Upload($execution);
expect($execution->fresh()->s3_cleanup_pending)->toBeFalse()
->and($execution->fresh()->s3_storage_deleted)->toBeTrue();
});
it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () {
Queue::fake();
Process::fake();
@ -1550,6 +1835,7 @@ function signInForVolumeBackups($testCase, Team $team): User
]);
$execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id,
's3_storage_id' => $s3Storage->id,
'status' => 'running',
'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz',
's3_cleanup_pending' => true,