From a4e13f56c0576616bb63992e8ee3457b5a61faf2 Mon Sep 17 00:00:00 2001 From: Ahmed A Date: Tue, 9 Sep 2025 17:25:55 +0300 Subject: [PATCH 1/4] Adding support for using config values for process --- app/Traits/ExecuteRemoteCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index a228a5d10..a37a2c768 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -44,7 +44,7 @@ public function execute_remote_command(...$commands) } } $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); - $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { + $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { $output = str($output)->trim(); if ($output->startsWith('╔')) { $output = "\n".$output; From c2d6cd14452b7951f7d95a00181d789c2e061642 Mon Sep 17 00:00:00 2001 From: Ahmed A Date: Tue, 9 Sep 2025 17:28:58 +0300 Subject: [PATCH 2/4] spacing fix --- app/Traits/ExecuteRemoteCommand.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index a37a2c768..3b88c3f16 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -44,7 +44,7 @@ public function execute_remote_command(...$commands) } } $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); - $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { + $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { $output = str($output)->trim(); if ($output->startsWith('╔')) { $output = "\n".$output; From b3d8b999590dac0dc04dce72e9b8e1087ff511cf Mon Sep 17 00:00:00 2001 From: Ahmed A Date: Tue, 9 Sep 2025 17:25:55 +0300 Subject: [PATCH 3/4] Adding support for using config values for process --- app/Traits/ExecuteRemoteCommand.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 0e7961368..289084266 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -96,7 +96,7 @@ public function execute_remote_command(...$commands) private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors) { $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); - $process = Process::timeout(3600)->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { + $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { $output = str($output)->trim(); if ($output->startsWith('╔')) { $output = "\n".$output; @@ -210,4 +210,4 @@ private function addRetryLogEntry(int $attempt, int $maxRetries, int $delay, str $this->application_deployment_queue->save(); } -} +} \ No newline at end of file From 4d52a26ac6348a16f8be70a8cfcdb9ea185a5f02 Mon Sep 17 00:00:00 2001 From: Ahmed A Date: Sat, 13 Sep 2025 17:04:22 +0300 Subject: [PATCH 4/4] fix rebase --- app/Traits/ExecuteRemoteCommand.php | 60 ++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/app/Traits/ExecuteRemoteCommand.php b/app/Traits/ExecuteRemoteCommand.php index 40edd65dd..289084266 100644 --- a/app/Traits/ExecuteRemoteCommand.php +++ b/app/Traits/ExecuteRemoteCommand.php @@ -45,12 +45,62 @@ public function execute_remote_command(...$commands) $command = parseLineForSudo($command, $this->server); } } - $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); - $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { - $output = str($output)->trim(); - if ($output->startsWith('╔')) { - $output = "\n".$output; + + $maxRetries = config('constants.ssh.max_retries'); + $attempt = 0; + $lastError = null; + $commandExecuted = false; + + while ($attempt < $maxRetries && ! $commandExecuted) { + try { + $this->executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors); + $commandExecuted = true; + } catch (\RuntimeException $e) { + $lastError = $e; + $errorMessage = $e->getMessage(); + // Only retry if it's an SSH connection error and we haven't exhausted retries + if ($this->isRetryableSshError($errorMessage) && $attempt < $maxRetries - 1) { + $attempt++; + $delay = $this->calculateRetryDelay($attempt - 1); + + // Track SSH retry event in Sentry + $this->trackSshRetryEvent($attempt, $maxRetries, $delay, $errorMessage, [ + 'server' => $this->server->name ?? $this->server->ip ?? 'unknown', + 'command' => remove_iip($command), + 'trait' => 'ExecuteRemoteCommand', + ]); + + // Add log entry for the retry + if (isset($this->application_deployment_queue)) { + $this->addRetryLogEntry($attempt, $maxRetries, $delay, $errorMessage); + } + + sleep($delay); + } else { + // Not retryable or max retries reached + throw $e; + } } + } + + // If we exhausted all retries and still failed + if (! $commandExecuted && $lastError) { + throw $lastError; + } + }); + } + + /** + * Execute the actual command with process handling + */ + private function executeCommandWithProcess($command, $hidden, $customType, $append, $ignore_errors) + { + $remote_command = SshMultiplexingHelper::generateSshCommand($this->server, $command); + $process = Process::timeout(config('constants.ssh.command_timeout'))->idleTimeout(3600)->start($remote_command, function (string $type, string $output) use ($command, $hidden, $customType, $append) { + $output = str($output)->trim(); + if ($output->startsWith('╔')) { + $output = "\n".$output; + } // Sanitize output to ensure valid UTF-8 encoding before JSON encoding $sanitized_output = sanitize_utf8_text($output);