feat(backups): stream S3-only volume archives directly (#11642)

app/Jobs/VolumeBackupJob.php

@@ -73,6 +73,7 @@ public function handle(): void
         $filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz';
         $backupLocation = $backupDirectory.'/'.$filename;
         $this->execution->update(['filename' => $backupLocation]);
+        $streamToS3 = $this->backup->save_s3 && $this->backup->disable_local_backup;
 
         try {
             $source = $this->backup->sourcePath();
@@ -86,11 +87,17 @@ public function handle(): void
 
             $compressorCommand = BackupCompression::compressorCommand($compressionCpuPercentage);
             $archiveScript = "compressor=\$({$compressorCommand}); tar -I \"\$compressor\" -cf - -C /volume .";
-            $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
-                .' -v '.escapeshellarg($source.':/volume:ro')
-                .' '.escapeshellarg($image)
-                .' sh -c '.escapeshellarg($archiveScript)
-                .' > '.escapeshellarg($backupLocation);
+            if ($streamToS3) {
+                $this->execution->update(['local_storage_deleted' => true]);
+                $archiveCommand = $this->streamToS3Command($archiveScript, $backupLocation, $source, $containerName, $image);
+                $this->execution->update(['s3_cleanup_pending' => true]);
+            } else {
+                $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName)
+                    .' -v '.escapeshellarg($source.':/volume:ro')
+                    .' '.escapeshellarg($image)
+                    .' sh -c '.escapeshellarg($archiveScript)
+                    .' > '.escapeshellarg($backupLocation);
+            }
 
             if ($this->backup->stop_during_backup) {
                 $containers = $this->containersUsingVolume($source, $server);
@@ -104,21 +111,23 @@ public function handle(): void
                 }
             }
 
-            instant_remote_process([
+            $archiveOutput = instant_remote_process(array_filter([
                 $verifySourceCommand,
-                'mkdir -p '.escapeshellarg($backupDirectory),
+                $streamToS3 ? null : 'mkdir -p '.escapeshellarg($backupDirectory),
                 $archiveCommand,
-            ], $server, timeout: $this->timeout, disableMultiplexing: true);
+            ]), $server, timeout: $this->timeout, disableMultiplexing: true);
             $this->execution->update([
                 'stop_container_ids' => null,
                 'stop_recovery_pending' => false,
             ]);
 
-            $size = (int) instant_remote_process(
-                ['du -b '.escapeshellarg($backupLocation).' | cut -f1'],
-                $server,
-                disableMultiplexing: true,
-            );
+            $size = $streamToS3
+                ? (int) str($archiveOutput)->trim()->afterLast("\n")->toString()
+                : (int) instant_remote_process(
+                    ['du -b '.escapeshellarg($backupLocation).' | cut -f1'],
+                    $server,
+                    disableMultiplexing: true,
+                );
 
             if ($size <= 0) {
                 throw new \RuntimeException('The storage backup archive is empty or was not created.');
@@ -127,9 +136,12 @@ public function handle(): void
             $warning = null;
             $s3Uploaded = null;
             $s3CleanupPending = false;
-            $localStorageDeleted = false;
+            $localStorageDeleted = $streamToS3;
 
-            if ($this->backup->save_s3) {
+            if ($streamToS3) {
+                $s3Uploaded = true;
+                $this->execution->update(['s3_cleanup_pending' => false]);
+            } elseif ($this->backup->save_s3) {
                 $s3CleanupPending = true;
                 $this->execution->update(['s3_cleanup_pending' => true]);
 
@@ -181,13 +193,23 @@ public function handle(): void
             }
         } catch (Throwable $exception) {
             $recoveryError = $this->recoverIncompleteBackup($this->execution);
-            $archiveDeleted = false;
+            $archiveDeleted = $streamToS3;
 
-            try {
-                deleteBackupsLocally($backupLocation, $server, throwError: true);
-                $archiveDeleted = true;
-            } catch (Throwable $cleanupException) {
-                $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
+            if ($streamToS3) {
+                $exception = new \RuntimeException(
+                    'S3-only streaming backup failed: '.$exception->getMessage()
+                    .'. The S3 destination may not support streaming uploads. Enable local backups to use the local archive upload method.',
+                    previous: $exception,
+                );
+            }
+
+            if (! $streamToS3) {
+                try {
+                    deleteBackupsLocally($backupLocation, $server, throwError: true);
+                    $archiveDeleted = true;
+                } catch (Throwable $cleanupException) {
+                    $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage();
+                }
             }
 
             $s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending;
@@ -195,7 +217,9 @@ public function handle(): void
             $this->execution->update([
                 'status' => 'failed',
                 'message' => $exception->getMessage().$recoveryError,
-                'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation,
+                'filename' => $streamToS3
+                    ? ($s3CleanupPending ? $backupLocation : null)
+                    : ($archiveDeleted && ! $s3CleanupPending ? null : $backupLocation),
                 'local_storage_deleted' => $archiveDeleted,
             ]);
 
@@ -338,6 +362,34 @@ private function uploadToS3(string $backupLocation, string $backupDirectory, Ser
         }
     }
 
+    private function streamToS3Command(string $archiveScript, string $backupLocation, string $source, string $containerName, string $image): string
+    {
+        $s3 = $this->backup->s3;
+
+        if (! $s3) {
+            $this->backup->update(['save_s3' => false, 's3_storage_id' => null]);
+
+            throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.');
+        }
+
+        $s3->testConnection(shouldSave: true);
+        $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint, $s3->trustedInternalHosts()))
+            ->map(fn (string $option): string => '--resolve '.escapeshellarg($option))
+            ->implode(' ');
+        $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
+        $destination = 'temporary/'.$s3->bucket.$backupLocation;
+        $streamScript = 'set -o pipefail; mc alias set'.$resolveOptions.' temporary '
+            .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret)
+            .' >/dev/null && ('.$archiveScript.' | mc pipe --quiet'.$resolveOptions.' '.escapeshellarg($destination).' >/dev/null)'
+            .' && mc stat --json'.$resolveOptions.' '.escapeshellarg($destination)
+            .' | sed -n '.escapeshellarg('s/.*"size":\([0-9][0-9]*\).*/\1/p');
+
+        return 'docker run --rm --name '.escapeshellarg($containerName)
+            .' -v '.escapeshellarg($source.':/volume:ro')
+            .' '.escapeshellarg($image)
+            .' sh -c '.escapeshellarg($streamScript);
+    }
+
     private function logCompressorInDevelopment(string $image, Server $server, int $compressionCpuPercentage): void
     {
         if (! isDev()) {

tests/Feature/VolumeBackupTest.php

@@ -28,6 +28,7 @@
 use App\Models\Team;
 use App\Models\User;
 use Illuminate\Database\Eloquent\Collection;
+use Illuminate\Filesystem\FilesystemAdapter;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Http\RedirectResponse;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
@@ -1902,6 +1903,152 @@ function signInForVolumeBackups($testCase, Team $team): User
         ->and($execution->s3->is($s3Storage))->toBeTrue();
 });
 
+it('streams S3-only volume backups without creating or copying a local archive', function () {
+    config(['broadcasting.default' => 'null']);
+    defined('CURLOPT_RESOLVE') || define('CURLOPT_RESOLVE', 10203);
+    Carbon::setTestNow('2026-08-15 12:00:00');
+    InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $s3Storage = S3Storage::create([
+        'name' => 'Streaming destination',
+        'region' => 'us-east-1',
+        'key' => 'key',
+        'secret' => 'secret',
+        'bucket' => 'bucket',
+        'endpoint' => 'https://s3.amazonaws.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        'disable_local_backup' => true,
+        's3_storage_id' => $s3Storage->id,
+    ]);
+    $sshDisk = Storage::fake('ssh-keys');
+    $disk = Mockery::mock(FilesystemAdapter::class);
+    $disk->shouldReceive('files')->once()->andReturn([]);
+    $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
+    Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+    Storage::shouldReceive('build')->once()->andReturn($disk);
+    Process::fake([
+        '*mc pipe*' => "Added `temporary` successfully.\n128 bytes -> `temporary/bucket/archive.tar.gz`\n128",
+        '*' => '',
+    ]);
+
+    (new VolumeBackupJob($backup))->handle();
+
+    $execution = ScheduledVolumeBackupExecution::query()->sole();
+    $expectedFilename = 'volume-app-data-1786795200.tar.gz';
+    Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I')
+        && str_contains($process->command, 'mc pipe')
+        && str_contains($process->command, '>/dev/null && (compressor=')
+        && str_contains($process->command, '>/dev/null) && mc stat')
+        && str_contains($process->command, 'temporary/bucket/')
+        && str_contains($process->command, $expectedFilename)
+        && substr_count($process->command, '--resolve') === 3
+        && substr_count($process->command, '>/dev/null') >= 2
+        && ! str_contains($process->command, ' > ')
+        && ! str_contains($process->command, 'mc cp'));
+    expect($execution->status)->toBe('success')
+        ->and($execution->size)->toBe(128)
+        ->and($execution->s3_uploaded)->toBeTrue()
+        ->and($execution->local_storage_deleted)->toBeTrue();
+});
+
+it('fails an unsupported S3 stream without falling back to a local archive', function () {
+    config(['broadcasting.default' => 'null']);
+    InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $s3Storage = S3Storage::create([
+        'name' => 'Unsupported streaming destination',
+        'region' => 'us-east-1',
+        'key' => 'key',
+        'secret' => 'secret',
+        'bucket' => 'bucket',
+        'endpoint' => 'https://s3.amazonaws.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        'disable_local_backup' => true,
+        's3_storage_id' => $s3Storage->id,
+    ]);
+    $sshDisk = Storage::fake('ssh-keys');
+    $disk = Mockery::mock(FilesystemAdapter::class);
+    $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]);
+    $disk->shouldReceive('delete')->once()->andReturnTrue();
+    Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+    Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk);
+    Process::fake([
+        '*mc pipe*' => Process::result(errorOutput: 'streaming upload is unsupported', exitCode: 1),
+        '*' => '',
+    ]);
+
+    expect(fn () => (new VolumeBackupJob($backup))->handle())
+        ->toThrow(RuntimeException::class, 'Enable local backups to use the local archive upload method.');
+
+    $execution = ScheduledVolumeBackupExecution::query()->sole();
+    expect($execution->status)->toBe('failed')
+        ->and($execution->message)->toContain('The S3 destination may not support streaming uploads.')
+        ->and($execution->message)->toContain('Enable local backups to use the local archive upload method.')
+        ->and($execution->filename)->toBeNull()
+        ->and($execution->local_storage_deleted)->toBeTrue();
+    Process::assertRan(fn ($process) => str_contains($process->command, 'mc pipe')
+        && ! str_contains($process->command, 'mc cp')
+        && ! str_contains($process->command, ' > '));
+    Process::assertNotRan(fn ($process) => str_contains($process->command, 'mkdir -p')
+        || (str_contains($process->command, 'rm -f') && str_contains($process->command, '.tar.gz')));
+});
+
+it('keeps local-first archive creation and mc copy when retaining a local volume backup', function () {
+    config(['broadcasting.default' => 'null']);
+    InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));
+    $team = Team::factory()->create();
+    [$application, $volume] = createVolumeBackupApplication($team);
+    $s3Storage = S3Storage::create([
+        'name' => 'Copied destination',
+        'region' => 'us-east-1',
+        'key' => 'key',
+        'secret' => 'secret',
+        'bucket' => 'bucket',
+        'endpoint' => 'https://s3.amazonaws.com',
+        'team_id' => $team->id,
+        'is_usable' => true,
+    ]);
+    $backup = $volume->scheduledBackups()->create([
+        'team_id' => $team->id,
+        'frequency' => 'daily',
+        'save_s3' => true,
+        'disable_local_backup' => false,
+        's3_storage_id' => $s3Storage->id,
+    ]);
+    $sshDisk = Storage::fake('ssh-keys');
+    $disk = Mockery::mock(FilesystemAdapter::class);
+    $disk->shouldReceive('files')->once()->andReturn([]);
+    $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue();
+    Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk);
+    Storage::shouldReceive('build')->once()->andReturn($disk);
+    Process::fake([
+        '*du -b*' => '128',
+        '*' => '',
+    ]);
+
+    (new VolumeBackupJob($backup))->handle();
+
+    Process::assertRan(fn ($process) => str_contains($process->command, 'tar -I')
+        && str_contains($process->command, '>')
+        && ! str_contains($process->command, 'mc pipe'));
+    Process::assertRan(fn ($process) => str_contains($process->command, 'mc cp')
+        && ! str_contains($process->command, 'mc pipe'));
+});
+
 it('removes local volume backups older than the configured retention days', function () {
     config(['broadcasting.default' => 'null']);
     InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0]));