rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(backups): forward configured timeouts to the ssh timeout wrapper (#11183)

Browse source
This commit is contained in:
Andras Bacsai 2026-08-12 23:40:55 +02:00 committed by GitHub
commit 24c32c3c39
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 106 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
app/Jobs/DatabaseBackupJob.php
View file

@ -785,7 +785,7 @@ private function upload_to_s3(): void
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set{$resolveOptions} temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}";
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp {$escapedBackupLocation} {$escapedS3Destination}";
instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
$this->s3_uploaded = true;
} catch (Throwable $e) {

2
bootstrap/helpers/remoteProcess.php
View file

@ -177,7 +177,7 @@ function instant_remote_process(Collection|array $command, Server $server, bool
return SshRetryHandler::retry(
function () use ($server, $command_string, $effectiveTimeout, $disableMultiplexing) {
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing);
$sshCommand = SshMultiplexingHelper::generateSshCommand($server, $command_string, $disableMultiplexing, (int) $effectiveTimeout);
$process = Process::timeout($effectiveTimeout)->run($sshCommand);
$output = trim($process->output());

2
config/constants.php
View file

@ -77,7 +77,7 @@
'mux_orphan_reap_enabled' => env('SSH_MUX_ORPHAN_REAP_ENABLED', false), // false = dry-run, only log orphans
'connection_timeout' => 10,
'server_interval' => 20,
'command_timeout' => 3600,
'command_timeout' => env('SSH_COMMAND_TIMEOUT', 3600),
'max_retries' => env('SSH_MAX_RETRIES', 3),
'retry_base_delay' => env('SSH_RETRY_BASE_DELAY', 2), // seconds
'retry_max_delay' => env('SSH_RETRY_MAX_DELAY', 30), // seconds

103
tests/Feature/RemoteProcessTimeoutTest.php Normal file
View file

@ -0,0 +1,103 @@
<?php
use App\Helpers\SshMultiplexingHelper;
use App\Models\PrivateKey;
use App\Models\Server;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Facades\Storage;
/**
* Tests that per-call SSH command timeouts (e.g. a scheduled backup's configured
* timeout) reach the shell-level `timeout N ssh` wrapper instead of being capped
* by the global constants.ssh.command_timeout default.
*
* @see https://github.com/coollabsio/coolify DatabaseBackupJob/VolumeBackupJob pass
* a per-backup timeout to instant_remote_process()
*/
uses(RefreshDatabase::class);
function makeTimeoutTestServer(): Server
{
$user = User::factory()->create();
$team = $user->teams()->first();
$privateKeyContent = '-----BEGIN OPENSSH PRIVATE KEY-----
'.
'b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
'.
'QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk
'.
'hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA
'.
'AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV
'.
'uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA==
'.
'-----END OPENSSH PRIVATE KEY-----';
$privateKey = PrivateKey::create([
'name' => 'timeout-test-key-'.uniqid(),
'private_key' => $privateKeyContent,
'team_id' => $team->id,
]);
Storage::fake('ssh-keys');
Storage::disk('ssh-keys')->put("ssh_key@{$privateKey->uuid}", $privateKeyContent);
$server = Server::factory()->create([
'team_id' => $team->id,
'private_key_id' => $privateKey->id,
]);
Storage::disk('ssh-keys')->put("ssh_key@{$server->privateKey->uuid}", $server->privateKey->private_key);
return $server;
}
it('wraps ssh commands with an explicitly passed command timeout', function () {
config(['constants.ssh.mux_enabled' => false]);
$server = makeTimeoutTestServer();
$command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok', commandTimeout: 7200);
expect($command)->toStartWith('timeout 7200 ssh ');
});
it('wraps ssh commands with the configured default timeout when none is passed', function () {
config([
'constants.ssh.mux_enabled' => false,
'constants.ssh.command_timeout' => 1234,
]);
$server = makeTimeoutTestServer();
$command = SshMultiplexingHelper::generateSshCommand($server, 'echo ok');
expect($command)->toStartWith('timeout 1234 ssh ');
});
it('forwards the per-call timeout of instant_remote_process to the ssh timeout wrapper', function () {
config(['constants.ssh.mux_enabled' => false]);
$server = makeTimeoutTestServer();
Process::fake();
instant_remote_process(['echo ok'], $server, timeout: 7200, disableMultiplexing: true);
Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 7200 ssh '));
});
it('uses the configured default timeout in instant_remote_process when no timeout is passed', function () {
config([
'constants.ssh.mux_enabled' => false,
'constants.ssh.command_timeout' => 1234,
]);
$server = makeTimeoutTestServer();
Process::fake();
instant_remote_process(['echo ok'], $server, disableMultiplexing: true);
Process::assertRan(fn ($process) => str_starts_with($process->command, 'timeout 1234 ssh '));
});