rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(backups): stop containers during volume backup creation

Browse source 
Track and recover stopped containers across interrupted backup executions.
This commit is contained in:
Andras Bacsai 2026-07-15 15:59:35 +02:00
parent 63961e0799
commit 4eec1ac547
11 changed files with 131 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
app/Jobs/ScheduledJobManager.php
View file

@ -112,7 +112,7 @@ public function handle(): void
} }
try { try {
$this->recoverPausedVolumeBackupContainers(); $this->recoverStoppedVolumeBackupContainers();
$this->processScheduledVolumeBackups(); $this->processScheduledVolumeBackups();
} catch (\Exception $e) { } catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [ Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [
@ -365,11 +365,11 @@ private function processScheduledVolumeBackups(): void
}); });
} }
private function recoverPausedVolumeBackupContainers(): void private function recoverStoppedVolumeBackupContainers(): void
{ {
ScheduledVolumeBackupExecution::query() ScheduledVolumeBackupExecution::query()
->where(fn (Builder $query) => $query ->where(fn (Builder $query) => $query
->where('pause_recovery_pending', true) ->where('stop_recovery_pending', true)
->orWhere('s3_cleanup_pending', true)) ->orWhere('s3_cleanup_pending', true))
->chunkById(self::CHUNK_SIZE, function ($executions): void { ->chunkById(self::CHUNK_SIZE, function ($executions): void {
foreach ($executions as $execution) { foreach ($executions as $execution) {
@ -385,7 +385,7 @@ private function processScheduledVolumeBackup(ScheduledVolumeBackup $backup): vo
if ($backup->executions() if ($backup->executions()
->where(fn (Builder $query) => $query ->where(fn (Builder $query) => $query
->where('pause_recovery_pending', true) ->where('stop_recovery_pending', true)
->orWhere('s3_cleanup_pending', true)) ->orWhere('s3_cleanup_pending', true))
->exists()) { ->exists()) {
$this->skippedCount++; $this->skippedCount++;

23
app/Jobs/VolumeBackupJob.php
View file

@ -83,11 +83,11 @@ public function handle(): void
.' '.escapeshellarg($image) .' '.escapeshellarg($image)
.' tar -czf - -C /volume . > '.escapeshellarg($backupLocation); .' tar -czf - -C /volume . > '.escapeshellarg($backupLocation);
if ($this->backup->pause_during_backup) { if ($this->backup->stop_during_backup) {
$containers = $this->containersUsingVolume($source, $server); $containers = $this->containersUsingVolume($source, $server);
if ($containers !== []) { if ($containers !== []) {
$this->execution->update(['pause_recovery_pending' => true]); $this->execution->update(['stop_recovery_pending' => true]);
$archiveCommand = $this->archiveWithPausedContainers( $archiveCommand = $this->archiveWithStoppedContainers(
$archiveCommand, $archiveCommand,
$containers, $containers,
VolumeBackupRecoveryJob::stateFile($this->execution), VolumeBackupRecoveryJob::stateFile($this->execution),
@ -101,8 +101,8 @@ public function handle(): void
$archiveCommand, $archiveCommand,
], $server, timeout: $this->timeout, disableMultiplexing: true); ], $server, timeout: $this->timeout, disableMultiplexing: true);
$this->execution->update([ $this->execution->update([
'pause_container_ids' => null, 'stop_container_ids' => null,
'pause_recovery_pending' => false, 'stop_recovery_pending' => false,
]); ]);
$size = (int) instant_remote_process( $size = (int) instant_remote_process(
@ -253,7 +253,7 @@ private function containersUsingVolume(string $source, Server $server): array
/** /**
* @param array<int, string> $containers * @param array<int, string> $containers
*/ */
private function archiveWithPausedContainers(string $archiveCommand, array $containers, string $stateFile): string private function archiveWithStoppedContainers(string $archiveCommand, array $containers, string $stateFile): string
{ {
if ($containers === []) { if ($containers === []) {
return $archiveCommand; return $archiveCommand;
@ -263,9 +263,12 @@ private function archiveWithPausedContainers(string $archiveCommand, array $cont
$script = "set -eu\n" $script = "set -eu\n"
.'state_file='.escapeshellarg($stateFile)."\n" .'state_file='.escapeshellarg($stateFile)."\n"
.": > \"\$state_file\"\n" .": > \"\$state_file\"\n"
."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker unpause \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n" ."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker start \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n"
."trap cleanup EXIT HUP INT TERM\n" ."trap cleanup EXIT\n"
."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker pause \"\$container\" >/dev/null; done\n" ."trap 'exit 129' HUP\n"
."trap 'exit 130' INT\n"
."trap 'exit 143' TERM\n"
."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker stop \"\$container\" >/dev/null; done\n"
.$archiveCommand; .$archiveCommand;
return 'sh -c '.escapeshellarg($script); return 'sh -c '.escapeshellarg($script);
@ -273,7 +276,7 @@ private function archiveWithPausedContainers(string $archiveCommand, array $cont
private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string
{ {
if (! $execution->pause_recovery_pending && ! $execution->s3_cleanup_pending) { if (! $execution->stop_recovery_pending && ! $execution->s3_cleanup_pending) {
return ''; return '';
} }

14
app/Jobs/VolumeBackupRecoveryJob.php
View file

@ -48,7 +48,7 @@ public static function recover(ScheduledVolumeBackupExecution $execution): void
{ {
$execution->loadMissing('scheduledVolumeBackup.volume.resource'); $execution->loadMissing('scheduledVolumeBackup.volume.resource');
if ($execution->pause_recovery_pending) { if ($execution->stop_recovery_pending) {
self::recoverContainers($execution); self::recoverContainers($execution);
} }
@ -75,22 +75,22 @@ private static function recoverContainers(ScheduledVolumeBackupExecution $execut
->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1) ->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1)
->values() ->values()
->all(); ->all();
$execution->update(['pause_container_ids' => $containers]); $execution->update(['stop_container_ids' => $containers]);
$remainingFile = $stateFile.'.remaining'; $remainingFile = $stateFile.'.remaining';
$script = 'status=0; : > '.escapeshellarg($remainingFile).'; ' $script = 'status=0; : > '.escapeshellarg($remainingFile).'; '
.'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do ' .'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do '
.'[ -z "$container" ] && continue; paused=$(docker inspect --format \'{{.State.Paused}}\' "$container" 2>/dev/null) ' .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>/dev/null) '
.'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; ' .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; '
.'if [ "$paused" = true ] && ! docker unpause "$container" >/dev/null; then echo "$container" >> ' .'if [ "$running" != true ] && ! docker start "$container" >/dev/null; then echo "$container" >> '
.escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; ' .escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; '
.'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile) .'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile)
.'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status'; .'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status';
instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true); instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true);
$execution->update([ $execution->update([
'pause_container_ids' => null, 'stop_container_ids' => null,
'pause_recovery_pending' => false, 'stop_recovery_pending' => false,
]); ]);
} }
@ -112,6 +112,6 @@ public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution
public static function stateFile(ScheduledVolumeBackupExecution $execution): string public static function stateFile(ScheduledVolumeBackupExecution $execution): string
{ {
return '/tmp/coolify-volume-backup-'.$execution->uuid.'.paused'; return '/tmp/coolify-volume-backup-'.$execution->uuid.'.stopped';
} }
} }

14
app/Livewire/Project/Shared/Storages/VolumeBackups.php
View file

@ -32,7 +32,7 @@ class VolumeBackups extends Component
public bool $disableLocalBackup = false; public bool $disableLocalBackup = false;
public bool $pauseDuringBackup = false; public bool $stopDuringBackup = false;
public ?int $s3StorageId = null; public ?int $s3StorageId = null;
@ -63,7 +63,7 @@ protected function rules(): array
'enabled' => ['required', 'boolean'], 'enabled' => ['required', 'boolean'],
'saveToS3' => ['required', 'boolean'], 'saveToS3' => ['required', 'boolean'],
'disableLocalBackup' => ['required', 'boolean'], 'disableLocalBackup' => ['required', 'boolean'],
'pauseDuringBackup' => ['required', 'boolean'], 'stopDuringBackup' => ['required', 'boolean'],
's3StorageId' => ['nullable', 'integer'], 's3StorageId' => ['nullable', 'integer'],
'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'], 'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'],
'retentionDaysLocally' => ['required', 'integer', 'min:0'], 'retentionDaysLocally' => ['required', 'integer', 'min:0'],
@ -90,7 +90,7 @@ public function mount(): void
$this->enabled = $this->backup->enabled; $this->enabled = $this->backup->enabled;
$this->saveToS3 = $this->backup->save_s3; $this->saveToS3 = $this->backup->save_s3;
$this->disableLocalBackup = $this->backup->disable_local_backup; $this->disableLocalBackup = $this->backup->disable_local_backup;
$this->pauseDuringBackup = $this->backup->pause_during_backup; $this->stopDuringBackup = $this->backup->stop_during_backup;
$this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id; $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id;
$this->retentionAmountLocally = $this->backup->retention_amount_locally; $this->retentionAmountLocally = $this->backup->retention_amount_locally;
$this->retentionDaysLocally = $this->backup->retention_days_locally; $this->retentionDaysLocally = $this->backup->retention_days_locally;
@ -188,7 +188,7 @@ public function delete(?string $password = null, array $selectedActions = [])
if ($this->backup->executions() if ($this->backup->executions()
->where(fn ($query) => $query ->where(fn ($query) => $query
->where('status', 'running') ->where('status', 'running')
->orWhere('pause_recovery_pending', true) ->orWhere('stop_recovery_pending', true)
->orWhere('s3_cleanup_pending', true)) ->orWhere('s3_cleanup_pending', true))
->exists()) { ->exists()) {
$this->dispatch('error', 'Wait for the running volume backup and container recovery to finish before deleting this schedule.'); $this->dispatch('error', 'Wait for the running volume backup and container recovery to finish before deleting this schedule.');
@ -246,7 +246,7 @@ public function cleanupFailed(): void
$deletedCount = $this->backup?->executions() $deletedCount = $this->backup?->executions()
->where('status', 'failed') ->where('status', 'failed')
->where('pause_recovery_pending', false) ->where('stop_recovery_pending', false)
->where('s3_cleanup_pending', false) ->where('s3_cleanup_pending', false)
->where(fn ($query) => $query ->where(fn ($query) => $query
->whereNull('filename') ->whereNull('filename')
@ -292,7 +292,7 @@ public function deleteBackup(int $executionId, string $password, array $selected
return false; return false;
} }
if ($execution->status === 'running' || $execution->pause_recovery_pending || $execution->s3_cleanup_pending) { if ($execution->status === 'running' || $execution->stop_recovery_pending || $execution->s3_cleanup_pending) {
$this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.'); $this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.');
return false; return false;
@ -369,7 +369,7 @@ private function persistBackup(bool $enabled): ScheduledVolumeBackup
'enabled' => $enabled, 'enabled' => $enabled,
'save_s3' => $this->saveToS3, 'save_s3' => $this->saveToS3,
'disable_local_backup' => $this->disableLocalBackup, 'disable_local_backup' => $this->disableLocalBackup,
'pause_during_backup' => $this->pauseDuringBackup, 'stop_during_backup' => $this->stopDuringBackup,
's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null, 's3_storage_id' => $this->saveToS3 ? $this->s3StorageId : null,
'retention_amount_locally' => $this->retentionAmountLocally, 'retention_amount_locally' => $this->retentionAmountLocally,
'retention_days_locally' => $this->retentionDaysLocally, 'retention_days_locally' => $this->retentionDaysLocally,

4
app/Models/ScheduledVolumeBackup.php
View file

@ -17,7 +17,7 @@ class ScheduledVolumeBackup extends BaseModel
'enabled', 'enabled',
'save_s3', 'save_s3',
'disable_local_backup', 'disable_local_backup',
'pause_during_backup', 'stop_during_backup',
'retention_amount_locally', 'retention_amount_locally',
'retention_days_locally', 'retention_days_locally',
'retention_max_storage_locally', 'retention_max_storage_locally',
@ -33,7 +33,7 @@ protected function casts(): array
'enabled' => 'boolean', 'enabled' => 'boolean',
'save_s3' => 'boolean', 'save_s3' => 'boolean',
'disable_local_backup' => 'boolean', 'disable_local_backup' => 'boolean',
'pause_during_backup' => 'boolean', 'stop_during_backup' => 'boolean',
'retention_amount_locally' => 'integer', 'retention_amount_locally' => 'integer',
'retention_days_locally' => 'integer', 'retention_days_locally' => 'integer',
'retention_max_storage_locally' => 'float', 'retention_max_storage_locally' => 'float',

8
app/Models/ScheduledVolumeBackupExecution.php
View file

@ -13,8 +13,8 @@ class ScheduledVolumeBackupExecution extends BaseModel
'message', 'message',
'size', 'size',
'filename', 'filename',
'pause_container_ids', 'stop_container_ids',
'pause_recovery_pending', 'stop_recovery_pending',
's3_cleanup_pending', 's3_cleanup_pending',
'finished_at', 'finished_at',
'local_storage_deleted', 'local_storage_deleted',
@ -27,8 +27,8 @@ protected function casts(): array
return [ return [
'size' => 'integer', 'size' => 'integer',
'finished_at' => 'datetime', 'finished_at' => 'datetime',
'pause_container_ids' => 'array', 'stop_container_ids' => 'array',
'pause_recovery_pending' => 'boolean', 'stop_recovery_pending' => 'boolean',
's3_cleanup_pending' => 'boolean', 's3_cleanup_pending' => 'boolean',
'local_storage_deleted' => 'boolean', 'local_storage_deleted' => 'boolean',
's3_storage_deleted' => 'boolean', 's3_storage_deleted' => 'boolean',

2
database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php
View file

@ -21,7 +21,7 @@ public function up(): void
$table->boolean('enabled')->default(true); $table->boolean('enabled')->default(true);
$table->boolean('save_s3')->default(false); $table->boolean('save_s3')->default(false);
$table->boolean('disable_local_backup')->default(false); $table->boolean('disable_local_backup')->default(false);
$table->boolean('pause_during_backup')->default(false); $table->boolean('stop_during_backup')->default(false);
$table->unsignedInteger('retention_amount_locally')->default(7); $table->unsignedInteger('retention_amount_locally')->default(7);
$table->unsignedInteger('retention_amount_s3')->default(7); $table->unsignedInteger('retention_amount_s3')->default(7);
$table->unsignedInteger('timeout')->default(3600); $table->unsignedInteger('timeout')->default(3600);

4
database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php
View file

@ -19,8 +19,8 @@ public function up(): void
$table->longText('message')->nullable(); $table->longText('message')->nullable();
$table->unsignedBigInteger('size')->default(0); $table->unsignedBigInteger('size')->default(0);
$table->text('filename')->nullable(); $table->text('filename')->nullable();
$table->json('pause_container_ids')->nullable(); $table->json('stop_container_ids')->nullable();
$table->boolean('pause_recovery_pending')->default(false); $table->boolean('stop_recovery_pending')->default(false);
$table->boolean('s3_cleanup_pending')->default(false); $table->boolean('s3_cleanup_pending')->default(false);
$table->timestamp('finished_at')->nullable(); $table->timestamp('finished_at')->nullable();
$table->boolean('local_storage_deleted')->default(false); $table->boolean('local_storage_deleted')->default(false);

20
resources/views/livewire/project/shared/storages/show.blade.php
View file

@ -65,6 +65,15 @@
</div> </div>
@endcan @endcan
@endif @endif
@if ($resource instanceof \App\Models\Application)
@can('update', $resource)
<x-modal-input buttonTitle="Configure Backup" title="Configure Volume Backup" :wireIgnore="false">
<livewire:project.application.backup.create :application="$resource"
:selected-volume-id="$storage->id"
wire:key="configure-readonly-volume-backup-{{ $storage->id }}" />
</x-modal-input>
@endcan
@endif
@else @else
@can('update', $resource) @can('update', $resource)
@if ($isFirst) @if ($isFirst)
@ -150,15 +159,4 @@
@endcan @endcan
@endif @endif
</form> </form>
@if ($isReadOnly && $resource instanceof \App\Models\Application)
@can('update', $resource)
<div class="pt-2">
<x-modal-input buttonTitle="Configure Backup" title="Configure Volume Backup" :wireIgnore="false">
<livewire:project.application.backup.create :application="$resource"
:selected-volume-id="$storage->id"
wire:key="configure-readonly-volume-backup-{{ $storage->id }}" />
</x-modal-input>
</div>
@endcan
@endif
</div> </div>

16
resources/views/livewire/project/shared/storages/volume-backups.blade.php
View file

@ -33,12 +33,16 @@
<span class="font-medium text-neutral-800 dark:text-neutral-200">{{ $storage->name }}</span> <span class="font-medium text-neutral-800 dark:text-neutral-200">{{ $storage->name }}</span>
</p> </p>
</div> </div>
<div class="p-3 text-sm rounded bg-warning/10 text-warning">
Backups made while the application is writing to this volume may be inconsistent or corrupted. You can
gracefully stop containers during the archive step for a safer file-level backup, but this briefly
interrupts the application.
</div>
</div> </div>
<div class="w-full max-w-md"> <div class="w-full max-w-md">
<x-forms.checkbox id="pauseDuringBackup" label="Pause containers while creating the archive" <x-forms.checkbox instantSave id="stopDuringBackup" label="Stop containers while creating the archive"
helper="Off by default. Containers using this volume are resumed immediately after the archive is created." /> helper="Off by default. Containers using this volume are gracefully stopped and restarted immediately after the archive is created." />
@if ($availableS3Storages->isNotEmpty()) @if ($availableS3Storages->isNotEmpty())
<x-forms.checkbox instantSave id="saveToS3" label="S3 Enabled" /> <x-forms.checkbox instantSave id="saveToS3" label="S3 Enabled" />
@elseif ($saveToS3) @elseif ($saveToS3)
@ -80,11 +84,7 @@
<div class="flex flex-col gap-4"> <div class="flex flex-col gap-4">
<h3>Settings</h3> <h3>Settings</h3>
<div class="p-3 text-sm rounded bg-warning/10 text-warning">
Backups made while the application is writing to this volume may be inconsistent or corrupted. You can
pause containers during the archive step for a more consistent backup, but this briefly interrupts the
application.
</div>
<div class="grid grid-cols-1 gap-2 md:grid-cols-3"> <div class="grid grid-cols-1 gap-2 md:grid-cols-3">
<x-forms.input id="frequency" label="Frequency" required <x-forms.input id="frequency" label="Frequency" required
helper="Use every_minute, hourly, daily, weekly, monthly, yearly, or a cron expression." /> helper="Use every_minute, hourly, daily, weekly, monthly, yearly, or a cron expression." />

99
tests/Feature/VolumeBackupTest.php
View file

@ -96,18 +96,25 @@
->assertSee($volume->name); ->assertSee($volume->name);
}); });
it('shows the configure backup modal trigger instead of inline backup settings on a volume', function () { it('shows the configure backup modal trigger inside the volume card instead of inline backup settings', function () {
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
$team = Team::factory()->create(); $team = Team::factory()->create();
signInForVolumeBackups($this, $team); signInForVolumeBackups($this, $team);
[$application, $volume] = createVolumeBackupApplication($team); [$application, $volume] = createVolumeBackupApplication($team);
Livewire::test(Show::class, [ $component = Livewire::test(Show::class, [
'storage' => $volume, 'storage' => $volume,
'resource' => $application, 'resource' => $application,
]) ])
->set('isReadOnly', true)
->assertSee('Configure Backup') ->assertSee('Configure Backup')
->assertDontSee('Backups made while the application is writing'); ->assertDontSee('Backups made while the application is writing');
$html = $component->html();
expect(strpos($html, 'Configure Backup'))
->toBeGreaterThan(strpos($html, '<form'))
->toBeLessThan(strpos($html, '</form>'));
}); });
it('only shows the backup enabled badge for an enabled volume backup', function () { it('only shows the backup enabled badge for an enabled volume backup', function () {
@ -151,7 +158,7 @@
'enabled', 'enabled',
'save_s3', 'save_s3',
'disable_local_backup', 'disable_local_backup',
'pause_during_backup', 'stop_during_backup',
'retention_amount_locally', 'retention_amount_locally',
'retention_days_locally', 'retention_days_locally',
'retention_max_storage_locally', 'retention_max_storage_locally',
@ -167,8 +174,8 @@
'message', 'message',
'size', 'size',
'filename', 'filename',
'pause_container_ids', 'stop_container_ids',
'pause_recovery_pending', 'stop_recovery_pending',
's3_cleanup_pending', 's3_cleanup_pending',
'finished_at', 'finished_at',
'local_storage_deleted', 'local_storage_deleted',
@ -215,7 +222,7 @@
'Backup Now', 'Backup Now',
'Delete Backups and Schedule', 'Delete Backups and Schedule',
'Persistent volume:', 'Persistent volume:',
'Pause containers while creating the archive', 'Stop containers while creating the archive',
'S3 Enabled', 'S3 Enabled',
'Disable Local Backup', 'Disable Local Backup',
'S3 Storage', 'S3 Storage',
@ -397,11 +404,12 @@ function signInForVolumeBackups($testCase, Team $team): User
->assertSee('Scheduled Backup') ->assertSee('Scheduled Backup')
->assertSee('Persistent volume:') ->assertSee('Persistent volume:')
->assertSee('inconsistent or corrupted') ->assertSee('inconsistent or corrupted')
->assertSee('gracefully stop containers')
->set('frequency', 'daily') ->set('frequency', 'daily')
->set('retentionAmountLocally', 5) ->set('retentionAmountLocally', 5)
->set('retentionDaysLocally', 14) ->set('retentionDaysLocally', 14)
->set('retentionMaxStorageLocally', 1.5) ->set('retentionMaxStorageLocally', 1.5)
->set('pauseDuringBackup', true) ->set('stopDuringBackup', true)
->call('save') ->call('save')
->assertDispatched('success'); ->assertDispatched('success');
@ -413,7 +421,7 @@ function signInForVolumeBackups($testCase, Team $team): User
->and($backup->retention_amount_locally)->toBe(5) ->and($backup->retention_amount_locally)->toBe(5)
->and($backup->retention_days_locally)->toBe(14) ->and($backup->retention_days_locally)->toBe(14)
->and($backup->retention_max_storage_locally)->toBe(1.5) ->and($backup->retention_max_storage_locally)->toBe(1.5)
->and($backup->pause_during_backup)->toBeTrue() ->and($backup->stop_during_backup)->toBeTrue()
->and($backup->save_s3)->toBeFalse(); ->and($backup->save_s3)->toBeFalse();
}); });
@ -487,6 +495,45 @@ function signInForVolumeBackups($testCase, Team $team): User
->and($backup->fresh()->disable_local_backup)->toBeFalse(); ->and($backup->fresh()->disable_local_backup)->toBeFalse();
}); });
it('saves each editable volume backup checkbox immediately', function () {
$team = Team::factory()->create();
signInForVolumeBackups($this, $team);
[$application, $volume] = createVolumeBackupApplication($team);
$s3Storage = S3Storage::create([
'name' => 'Volume backups',
'region' => 'us-east-1',
'key' => 'key',
'secret' => 'secret',
'bucket' => 'bucket',
'endpoint' => 'https://s3.example.com',
'team_id' => $team->id,
'is_usable' => true,
]);
$backup = ScheduledVolumeBackup::create([
'local_persistent_volume_id' => $volume->id,
'team_id' => $team->id,
's3_storage_id' => $s3Storage->id,
'frequency' => 'daily',
'save_s3' => true,
]);
$component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]);
$html = $component->html();
foreach (['stopDuringBackup', 'saveToS3', 'disableLocalBackup'] as $property) {
preg_match('/<input\b(?=[^>]*wire:model=(?:"'.$property.'"|'.$property.'))[^>]*>/', $html, $matches);
expect($matches[0] ?? null)->not->toBeNull()
->and($matches[0])->toContain("wire:click='instantSave'");
}
$component->set('stopDuringBackup', true)->call('instantSave')->assertDispatched('success');
expect($backup->refresh()->stop_during_backup)->toBeTrue();
$component->set('stopDuringBackup', false)->call('instantSave')->assertDispatched('success');
expect($backup->refresh()->stop_during_backup)->toBeFalse();
});
it('allows volume S3 backups to be disabled when no usable storage remains', function () { it('allows volume S3 backups to be disabled when no usable storage remains', function () {
$team = Team::factory()->create(); $team = Team::factory()->create();
signInForVolumeBackups($this, $team); signInForVolumeBackups($this, $team);
@ -829,7 +876,7 @@ function signInForVolumeBackups($testCase, Team $team): User
->and($oldExecution->fresh()->local_storage_deleted)->toBeFalse(); ->and($oldExecution->fresh()->local_storage_deleted)->toBeFalse();
}); });
it('pauses and resumes containers that use the volume when requested', function () { it('stops and restarts containers that use the volume when requested', function () {
config(['broadcasting.default' => 'null']); config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
$team = Team::factory()->create(); $team = Team::factory()->create();
@ -838,7 +885,7 @@ function signInForVolumeBackups($testCase, Team $team): User
'local_persistent_volume_id' => $volume->id, 'local_persistent_volume_id' => $volume->id,
'team_id' => $team->id, 'team_id' => $team->id,
'frequency' => 'daily', 'frequency' => 'daily',
'pause_during_backup' => true, 'stop_during_backup' => true,
]); ]);
Process::fake([ Process::fake([
@ -849,9 +896,11 @@ function signInForVolumeBackups($testCase, Team $team): User
(new VolumeBackupJob($backup))->handle(); (new VolumeBackupJob($backup))->handle();
Process::assertRan(fn ($process) => str_contains($process->command, 'trap cleanup EXIT HUP INT TERM') Process::assertRan(fn ($process) => str_contains($process->command, 'trap cleanup EXIT')
&& str_contains($process->command, 'docker pause') && str_contains($process->command, 'exit 143')
&& str_contains($process->command, 'docker unpause') && str_contains($process->command, 'TERM')
&& str_contains($process->command, 'docker stop')
&& str_contains($process->command, 'docker start')
&& str_contains($process->command, 'state_file=')); && str_contains($process->command, 'state_file='));
Process::assertRan(fn ($process) => str_contains($process->command, '{{println .Source}}{{println .Name}}') Process::assertRan(fn ($process) => str_contains($process->command, '{{println .Source}}{{println .Name}}')
&& str_contains($process->command, "grep -Fqx -- 'app-data'") && str_contains($process->command, "grep -Fqx -- 'app-data'")
@ -859,7 +908,7 @@ function signInForVolumeBackups($testCase, Team $team): User
&& str_contains($process->command, 'continue')); && str_contains($process->command, 'continue'));
}); });
it('finds containers using a bind mounted host path before pausing', function () { it('finds containers using a bind mounted host path before stopping', function () {
config(['broadcasting.default' => 'null']); config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
$team = Team::factory()->create(); $team = Team::factory()->create();
@ -869,7 +918,7 @@ function signInForVolumeBackups($testCase, Team $team): User
'local_persistent_volume_id' => $volume->id, 'local_persistent_volume_id' => $volume->id,
'team_id' => $team->id, 'team_id' => $team->id,
'frequency' => 'daily', 'frequency' => 'daily',
'pause_during_backup' => true, 'stop_during_backup' => true,
]); ]);
Process::fake([ Process::fake([
@ -884,7 +933,7 @@ function signInForVolumeBackups($testCase, Team $team): User
&& str_contains($process->command, "grep -Fqx -- '/srv/app-data'")); && str_contains($process->command, "grep -Fqx -- '/srv/app-data'"));
}); });
it('retries container recovery when a timed out backup left a container paused', function () { it('retries container recovery when a timed out backup left a container stopped', function () {
Queue::fake(); Queue::fake();
$team = Team::factory()->create(); $team = Team::factory()->create();
[$application, $volume] = createVolumeBackupApplication($team); [$application, $volume] = createVolumeBackupApplication($team);
@ -896,23 +945,23 @@ function signInForVolumeBackups($testCase, Team $team): User
$execution = ScheduledVolumeBackupExecution::create([ $execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id, 'scheduled_volume_backup_id' => $backup->id,
'status' => 'running', 'status' => 'running',
'pause_recovery_pending' => true, 'stop_recovery_pending' => true,
]); ]);
Process::fake([ Process::fake([
'*cat *coolify-volume-backup-*' => 'abc123', '*cat *coolify-volume-backup-*' => 'abc123',
'*docker unpause*' => Process::result(errorOutput: 'daemon unavailable', exitCode: 1), '*docker start*' => Process::result(errorOutput: 'daemon unavailable', exitCode: 1),
]); ]);
(new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out')); (new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out'));
expect($execution->fresh()->status)->toBe('failed') expect($execution->fresh()->status)->toBe('failed')
->and($execution->fresh()->message)->toContain('Container recovery failed') ->and($execution->fresh()->message)->toContain('Container recovery failed')
->and($execution->fresh()->pause_container_ids)->toBe(['abc123']) ->and($execution->fresh()->stop_container_ids)->toBe(['abc123'])
->and($execution->fresh()->pause_recovery_pending)->toBeTrue(); ->and($execution->fresh()->stop_recovery_pending)->toBeTrue();
Queue::assertPushed(VolumeBackupRecoveryJob::class); Queue::assertPushed(VolumeBackupRecoveryJob::class);
}); });
it('resumes containers and removes a partial archive when archiving fails', function () { it('restarts containers and removes a partial archive when archiving fails', function () {
config(['broadcasting.default' => 'null']); config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
$team = Team::factory()->create(); $team = Team::factory()->create();
@ -921,11 +970,11 @@ function signInForVolumeBackups($testCase, Team $team): User
'local_persistent_volume_id' => $volume->id, 'local_persistent_volume_id' => $volume->id,
'team_id' => $team->id, 'team_id' => $team->id,
'frequency' => 'daily', 'frequency' => 'daily',
'pause_during_backup' => true, 'stop_during_backup' => true,
]); ]);
Process::fake([ Process::fake([
'*docker ps -q*' => 'abc123', '*docker ps -q*' => 'abc123',
'*trap cleanup EXIT HUP INT TERM*' => Process::result(errorOutput: 'archive failed', exitCode: 1), '*trap cleanup EXIT*' => Process::result(errorOutput: 'archive failed', exitCode: 1),
'*' => '', '*' => '',
]); ]);
@ -933,7 +982,7 @@ function signInForVolumeBackups($testCase, Team $team): User
$execution = ScheduledVolumeBackupExecution::query()->sole(); $execution = ScheduledVolumeBackupExecution::query()->sole();
expect($execution->status)->toBe('failed') expect($execution->status)->toBe('failed')
->and($execution->pause_container_ids)->toBeNull(); ->and($execution->stop_container_ids)->toBeNull();
Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f')
&& str_contains($process->command, '.tar.gz')); && str_contains($process->command, '.tar.gz'));
}); });
@ -1079,7 +1128,7 @@ function signInForVolumeBackups($testCase, Team $team): User
$execution = ScheduledVolumeBackupExecution::create([ $execution = ScheduledVolumeBackupExecution::create([
'scheduled_volume_backup_id' => $backup->id, 'scheduled_volume_backup_id' => $backup->id,
'status' => 'failed', 'status' => 'failed',
'pause_recovery_pending' => true, 'stop_recovery_pending' => true,
]); ]);
(new ScheduledJobManager)->handle(); (new ScheduledJobManager)->handle();