rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(ssh): enhance error handling in SSH command execution and improve connection validation logging

Browse source
This commit is contained in:
Andras Bacsai 2025-09-07 18:45:44 +02:00
parent 579cc25898
commit 4bd29bf966
4 changed files with 26 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
app/Models/Server.php
View file

@ -1082,6 +1082,7 @@ public function sendUnreachableNotification()
public function validateConnection(bool $justCheckingNewKey = false)
{
ray('validateConnection', $this->id);
$this->disableSshMux();
if ($this->skipServer()) {

4
app/Traits/ExecuteRemoteCommand.php
View file

@ -88,10 +88,6 @@ public function execute_remote_command(...$commands)
private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors)
{
$remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command);
// Randomly fail the command with a key exchange error for testing
// if (random_int(1, 20) === 1) { // 5% chance to fail
// throw new \RuntimeException('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer');
// }
$process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) {
$output = str($output)->trim();
if ($output->startsWith('╔')) {

32
app/Traits/SshRetryable.php
View file

@ -57,9 +57,9 @@ protected function isRetryableSshError(string $errorOutput): bool
*/
protected function calculateRetryDelay(int $attempt): int
{
$baseDelay = config('constants.ssh.retry_base_delay', 2);
$maxDelay = config('constants.ssh.retry_max_delay', 30);
$multiplier = config('constants.ssh.retry_multiplier', 2);
$baseDelay = config('constants.ssh.retry_base_delay');
$maxDelay = config('constants.ssh.retry_max_delay');
$multiplier = config('constants.ssh.retry_multiplier');
$delay = min($baseDelay * pow($multiplier, $attempt), $maxDelay);
@ -76,23 +76,17 @@ protected function calculateRetryDelay(int $attempt): int
*/
protected function executeWithSshRetry(callable $callback, array $context = [], bool $throwError = true)
{
$maxRetries = config('constants.ssh.max_retries', 3);
$maxRetries = config('constants.ssh.max_retries');
$lastError = null;
$lastErrorMessage = '';
// Randomly fail the command with a key exchange error for testing
// if (random_int(1, 10) === 1) { // 10% chance to fail
// ray('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer');
// throw new \RuntimeException('SSH key exchange failed: kex_exchange_identification: read: Connection reset by peer');
// }
for ($attempt = 0; $attempt < $maxRetries; $attempt++) {
try {
// Execute the callback
$result = $callback();
// If we get here, it succeeded
if ($attempt > 0) {
Log::info('SSH operation succeeded after retry', array_merge($context, [
'attempt' => $attempt + 1,
]));
}
return $result;
return $callback();
} catch (\Throwable $e) {
$lastError = $e;
$lastErrorMessage = $e->getMessage();
@ -125,6 +119,12 @@ protected function executeWithSshRetry(callable $callback, array $context = [],
}
if ($throwError && $lastError) {
// If the error message is empty, provide a more meaningful one
if (empty($lastErrorMessage) || trim($lastErrorMessage) === '') {
$contextInfo = isset($context['server']) ? " to server {$context['server']}" : '';
$attemptInfo = $attempt > 1 ? " after {$attempt} attempts" : '';
throw new \RuntimeException("SSH connection failed{$contextInfo}{$attemptInfo}", $lastError->getCode());
}
throw $lastError;
}

11
bootstrap/helpers/remoteProcess.php
View file

@ -159,11 +159,18 @@ function excludeCertainErrors(string $errorOutput, ?int $exitCode = null)
'Could not resolve hostname',
]);
$ignored = $ignoredErrors->contains(fn ($error) => Str::contains($errorOutput, $error));
// Ensure we always have a meaningful error message
$errorMessage = trim($errorOutput);
if (empty($errorMessage)) {
$errorMessage = "SSH command failed with exit code: $exitCode";
}
if ($ignored) {
// TODO: Create new exception and disable in sentry
throw new \RuntimeException($errorOutput, $exitCode);
throw new \RuntimeException($errorMessage, $exitCode);
}
throw new \RuntimeException($errorOutput, $exitCode);
throw new \RuntimeException($errorMessage, $exitCode);
}
function decode_remote_command_output(?ApplicationDeploymentQueue $application_deployment_queue = null): Collection