From 6293cd418c483e389e22b43d824740f3607c9f66 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:10:10 +0200 Subject: [PATCH] feat(backups): stream S3-only volume archives directly (#11642) --- app/Jobs/VolumeBackupJob.php | 96 ++++++++++++++----- tests/Feature/VolumeBackupTest.php | 147 +++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+), 22 deletions(-) diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php index b567a71b7..0b13c6660 100644 --- a/app/Jobs/VolumeBackupJob.php +++ b/app/Jobs/VolumeBackupJob.php @@ -73,6 +73,7 @@ public function handle(): void $filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz'; $backupLocation = $backupDirectory.'/'.$filename; $this->execution->update(['filename' => $backupLocation]); + $streamToS3 = $this->backup->save_s3 && $this->backup->disable_local_backup; try { $source = $this->backup->sourcePath(); @@ -86,11 +87,17 @@ public function handle(): void $compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage); $archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume ."; - $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName) - .' -v '.escapeshellarg($source.':/volume:ro') - .' '.escapeshellarg($image) - .' sh -c '.escapeshellarg($archiveScript) - .' > '.escapeshellarg($backupLocation); + if ($streamToS3) { + $this->execution->update(['local_storage_deleted' => true]); + $archiveCommand = $this->streamToS3Command($archiveScript, $backupLocation, $source, $containerName, $image); + $this->execution->update(['s3_cleanup_pending' => true]); + } else { + $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName) + .' -v '.escapeshellarg($source.':/volume:ro') + .' '.escapeshellarg($image) + .' sh -c '.escapeshellarg($archiveScript) + .' > '.escapeshellarg($backupLocation); + } if ($this->backup->stop_during_backup) { $containers = $this->containersUsingVolume($source, $server); @@ -104,21 +111,23 @@ public function handle(): void } } - instant_remote_process([ + $archiveOutput = instant_remote_process(array_filter([ $verifySourceCommand, - 'mkdir -p '.escapeshellarg($backupDirectory), + $streamToS3 ? null : 'mkdir -p '.escapeshellarg($backupDirectory), $archiveCommand, - ], $server, timeout: $this->timeout, disableMultiplexing: true); + ]), $server, timeout: $this->timeout, disableMultiplexing: true); $this->execution->update([ 'stop_container_ids' => null, 'stop_recovery_pending' => false, ]); - $size = (int) instant_remote_process( - ['du -b '.escapeshellarg($backupLocation).' | cut -f1'], - $server, - disableMultiplexing: true, - ); + $size = $streamToS3 + ? (int) str($archiveOutput)->trim()->afterLast("\n")->toString() + : (int) instant_remote_process( + ['du -b '.escapeshellarg($backupLocation).' | cut -f1'], + $server, + disableMultiplexing: true, + ); if ($size <= 0) { throw new \RuntimeException('The storage backup archive is empty or was not created.'); @@ -127,9 +136,12 @@ public function handle(): void $warning = null; $s3Uploaded = null; $s3CleanupPending = false; - $localStorageDeleted = false; + $localStorageDeleted = $streamToS3; - if ($this->backup->save_s3) { + if ($streamToS3) { + $s3Uploaded = true; + $this->execution->update(['s3_cleanup_pending' => false]); + } elseif ($this->backup->save_s3) { $s3CleanupPending = true; $this->execution->update(['s3_cleanup_pending' => true]); @@ -181,13 +193,23 @@ public function handle(): void } } catch (Throwable $exception) { $recoveryError = $this->recoverIncompleteBackup($this->execution); - $archiveDeleted = false; + $archiveDeleted = $streamToS3; - try { - deleteBackupsLocally($backupLocation, $server, throwError: true); - $archiveDeleted = true; - } catch (Throwable $cleanupException) { - $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage(); + if ($streamToS3) { + $exception = new \RuntimeException( + 'S3-only streaming backup failed: '.$exception->getMessage() + .'. The S3 destination may not support streaming uploads. Enable local backups to use the local archive upload method.', + previous: $exception, + ); + } + + if (! $streamToS3) { + try { + deleteBackupsLocally($backupLocation, $server, throwError: true); + $archiveDeleted = true; + } catch (Throwable $cleanupException) { + $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage(); + } } $s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending; @@ -195,7 +217,9 @@ public function handle(): void $this->execution->update([ 'status' => 'failed', 'message' => $exception->getMessage().$recoveryError, - 'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation, + 'filename' => $streamToS3 + ? ($s3CleanupPending ? $backupLocation : null) + : ($archiveDeleted && ! $s3CleanupPending ? null : $backupLocation), 'local_storage_deleted' => $archiveDeleted, ]); @@ -338,6 +362,34 @@ private function uploadToS3(string $backupLocation, string $backupDirectory, Ser } } + private function streamToS3Command(string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string + { + $s3 = $this->backup->s3; + + if (! $s3) { + $this->backup->update(['save_s3' => false, 's3_storage_id' => null]); + + throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.'); + } + + $s3->testConnection(shouldSave: true); + $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts())) + ->map(fn (string $option): string => '--resolve '.escapeshellarg($option)) + ->implode(' '); + $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions; + $destination = 'temporary/'.$s3->bucket.$backupLocation; + $streamScript = 'set -o pipefail; mc alias set'.$resolveOptions.' temporary ' + .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret) + .' >/dev/null && ('.$archiveScript.' | mc pipe --quiet'.$resolveOptions.' '.escapeshellarg($destination).' >/dev/null)' + .' && mc stat --json'.$resolveOptions.' '.escapeshellarg($destination) + .' | sed -n '.escapeshellarg('s/.*"size":\([0-9][0-9]*\).*/\1/p'); + + return 'docker run --rm --name '.escapeshellarg($containerName) + .' -v '.escapeshellarg($source.':/volume:ro') + .' '.escapeshellarg($image) + .' sh -c '.escapeshellarg($streamScript); + } + private function logCompressorInDevelopment(string $image, Server $server, int $compressionCpuPercentage): void { if (! isDev()) { diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php index 9c6897632..36a04ceca 100644 --- a/tests/Feature/VolumeBackupTest.php +++ b/tests/Feature/VolumeBackupTest.php @@ -28,6 +28,7 @@ use App\Models\Team; use App\Models\User; use Illuminate\Database\Eloquent\Collection; +use Illuminate\Filesystem\FilesystemAdapter; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Http\RedirectResponse; use Illuminate\Queue\Middleware\WithoutOverlapping; @@ -1902,6 +1903,152 @@ function signInForVolumeBackups($testCase, Team $team): User ->and($execution->s3->is($s3Storage))->toBeTrue(); }); +it('streams S3-only volume backups without creating or copying a local archive', function () { + config(['broadcasting.default' => 'null']); + defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203); + Carbon::setTestNow('2026-08-15 12:00:00'); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Streaming destination', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.amazonaws.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $s3Storage->id, + ]); + $sshDisk = Storage::fake('ssh-keys'); + $disk = Mockery::mock(FilesystemAdapter::class); + $disk->shouldReceive('files')->once()->andReturn([]); + $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); + Storage::shouldReceive('build')->once()->andReturn($disk); + Process::fake([ + '*mc pipe*' => "Added `temporary` successfully.\n128 bytes -> `temporary/bucket/archive.tar.gz`\n128", + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + $expectedFilename = 'volume-app-data-1786795200.tar.gz'; + Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I') + && str_contains($process->command, 'mc pipe') + && str_contains($process->command, '>/dev/null && (compressor=') + && str_contains($process->command, '>/dev/null) && mc stat') + && str_contains($process->command, 'temporary/bucket/') + && str_contains($process->command, $expectedFilename) + && substr_count($process->command, '--resolve') === 3 + && substr_count($process->command, '>/dev/null') >= 2 + && ! str_contains($process->command, ' > ') + && ! str_contains($process->command, 'mc cp')); + expect($execution->status)->toBe('success') + ->and($execution->size)->toBe(128) + ->and($execution->s3_uploaded)->toBeTrue() + ->and($execution->local_storage_deleted)->toBeTrue(); +}); + +it('fails an unsupported S3 stream without falling back to a local archive', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Unsupported streaming destination', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.amazonaws.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $s3Storage->id, + ]); + $sshDisk = Storage::fake('ssh-keys'); + $disk = Mockery::mock(FilesystemAdapter::class); + $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); + Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk); + Process::fake([ + '*mc pipe*' => Process::result(errorOutput: 'streaming upload is unsupported', exitCode: 1), + '*' => '', + ]); + + expect(fn () => (new VolumeBackupJob($backup))->handle()) + ->toThrow(RuntimeException::class, 'Enable local backups to use the local archive upload method.'); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + expect($execution->status)->toBe('failed') + ->and($execution->message)->toContain('The S3 destination may not support streaming uploads.') + ->and($execution->message)->toContain('Enable local backups to use the local archive upload method.') + ->and($execution->filename)->toBeNull() + ->and($execution->local_storage_deleted)->toBeTrue(); + Process::assertRan(fn ($process) => str_contains($process->command, 'mc pipe') + && ! str_contains($process->command, 'mc cp') + && ! str_contains($process->command, ' > ')); + Process::assertNotRan(fn ($process) => str_contains($process->command, 'mkdir -p') + || (str_contains($process->command, 'rm -f') && str_contains($process->command, '.tar.gz'))); +}); + +it('keeps local-first archive creation and mc copy when retaining a local volume backup', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Copied destination', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.amazonaws.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'disable_local_backup' => false, + 's3_storage_id' => $s3Storage->id, + ]); + $sshDisk = Storage::fake('ssh-keys'); + $disk = Mockery::mock(FilesystemAdapter::class); + $disk->shouldReceive('files')->once()->andReturn([]); + $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); + Storage::shouldReceive('build')->once()->andReturn($disk); + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I') + && str_contains($process->command, '>') + && ! str_contains($process->command, 'mc pipe')); + Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp') + && ! str_contains($process->command, 'mc pipe')); +}); + it('removes local volume backups older than the configured retention days', function () { config(['broadcasting.default' => 'null']); InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));