rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(backups): use parallel gzip for volume backups

Browse source 
Add pigz to the helper image, select it when available with gzip fallback, and log the selected compressor in development.
This commit is contained in:
Andras Bacsai 2026-08-14 00:00:23 +02:00
parent fd5eb3e0cd
commit b3aa1fd232
4 changed files with 77 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
app/Jobs/VolumeBackupJob.php
View file

@ -77,14 +77,19 @@ public function handle(): void
$source = $this->backup->sourcePath();
$containerName = 'volume-backup-'.$this->execution->uuid;
$image = coolifyHelperImage().':'.getHelperVersion();
$this->logCompressorInDevelopment($image, $server);
$verifySourceCommand = $target instanceof LocalPersistentVolume && blank($target->host_path)
? 'docker volume inspect '.escapeshellarg($source).' >/dev/null'
: 'test -d '.escapeshellarg($source);
$archiveScript = "compressor='gzip -3'; "
.'if command -v pigz >/dev/null 2>&1; then compressor="pigz -3 -p $(( ($(nproc) + 1) / 2 ))"; fi; '
.'tar -I "$compressor" -cf - -C /volume .';
$archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
.' -v '.escapeshellarg($source.':/volume:ro')
.' '.escapeshellarg($image)
." tar -I 'gzip -1' -cf - -C /volume . > ".escapeshellarg($backupLocation);
.' sh -c '.escapeshellarg($archiveScript)
.' > '.escapeshellarg($backupLocation);
if ($this->backup->stop_during_backup) {
$containers = $this->containersUsingVolume($source, $server);
@ -332,6 +337,28 @@ private function uploadToS3(string $backupLocation, string $backupDirectory, Ser
}
}
private function logCompressorInDevelopment(string $image, Server $server): void
{
if (! isDev()) {
return;
}
$script = "if command -v pigz >/dev/null 2>&1; then printf 'pigz -3 -p %s' \"$(( ($(nproc) + 1) / 2 ))\"; else printf 'gzip -3'; fi";
$compressor = instant_remote_process(
['docker run --rm '.escapeshellarg($image).' sh -c '.escapeshellarg($script)],
$server,
timeout: 60,
disableMultiplexing: true,
);
Log::info('Volume backup compressor selected', [
'backup_id' => $this->backup->id,
'execution_id' => $this->execution?->id,
'compressor' => $compressor,
'helper_image' => $image,
]);
}
private function removeExpiredBackups(Server $server): void
{
if ($this->hasRetentionLimits(

2
config/constants.php
View file

@ -3,7 +3,7 @@
return [
'coolify' => [
'version' => env('COOLIFY_VERSION') ?: '4.3.3',
'helper_version' => '1.0.14',
'helper_version' => '1.0.15',
'realtime_version' => '1.0.17',
'railpack_version' => '0.23.0',
'self_hosted' => env('SELF_HOSTED', true),

2
docker/coolify-helper/Dockerfile
View file

@ -36,7 +36,7 @@ USER root
WORKDIR /artifacts
ENV RAILPACK_VERSION=${RAILPACK_VERSION}
RUN apk upgrade --no-cache && \
apk add --no-cache bash curl git git-lfs openssh-client tar tini
apk add --no-cache bash curl git git-lfs openssh-client pigz tar tini
RUN mkdir -p ~/.docker/cli-plugins
# Install mise (musl build) at the path railpack expects (/tmp/railpack/mise/mise-VERSION).

48
tests/Feature/VolumeBackupTest.php
View file

@ -31,6 +31,7 @@
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Facades\Route;
@ -50,6 +51,12 @@
->and(method_exists(LocalFileVolume::class, 'scheduledBackups'))->toBeTrue();
});
it('includes parallel gzip support in the Coolify helper image', function () {
$dockerfile = file_get_contents(base_path('docker/coolify-helper/Dockerfile'));
expect($dockerfile)->toContain('pigz');
});
it('keeps the volume backup script inside the Livewire root element', function () {
$view = file_get_contents(resource_path('views/livewire/project/shared/storages/volume-backups.blade.php'));
@ -1599,12 +1606,51 @@ function signInForVolumeBackups($testCase, Team $team): User
Process::assertRan(fn ($process) => str_contains($process->command, 'docker volume inspect')
&& str_contains($process->command, 'docker run --rm --name ')
&& str_contains($process->command, 'app-data:/volume:ro')
&& str_contains($process->command, "tar -I 'gzip -1' -cf -")
&& str_contains($process->command, 'command -v pigz')
&& str_contains($process->command, 'pigz -3 -p')
&& str_contains($process->command, '$(nproc) + 1')
&& str_contains($process->command, 'gzip -3')
&& str_contains($process->command, 'tar -I "$compressor" -cf -')
&& str_contains($process->command, '> ')
&& str_contains($process->command, '.tar.gz')
&& ! str_contains($process->command, ':/backup'));
});
it('logs the selected volume backup compressor in development', function (string $detectedCompressor) {
config(['app.env' => 'local', 'broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
$team = Team::factory()->create();
[$application, $volume] = createVolumeBackupApplication($team);
$backup = $volume->scheduledBackups()->create([
'team_id' => $team->id,
'frequency' => 'daily',
'retention_amount_locally' => 7,
'retention_days_locally' => 0,
'retention_max_storage_locally' => 0,
'retention_amount_s3' => 7,
'retention_days_s3' => 0,
'retention_max_storage_s3' => 0,
]);
Process::fake([
'*command -v pigz*' => $detectedCompressor,
'*du -b*' => '128',
'*' => '',
]);
Log::spy();
(new VolumeBackupJob($backup))->handle();
Log::shouldHaveReceived('info')->once()->with(
'Volume backup compressor selected',
Mockery::on(fn (array $context): bool => $context['compressor'] === $detectedCompressor
&& $context['backup_id'] === $backup->id),
);
})->with([
'pigz' => 'pigz -3 -p 4',
'gzip fallback' => 'gzip -3',
]);
it('keeps the upload destination on the volume backup execution', function () {
config(['broadcasting.default' => 'null']);
InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));