diff --git a/app/Actions/Database/StartDatabaseProxy.php b/app/Actions/Database/StartDatabaseProxy.php index c634f14ba..4331c6ae7 100644 --- a/app/Actions/Database/StartDatabaseProxy.php +++ b/app/Actions/Database/StartDatabaseProxy.php @@ -112,12 +112,52 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St $dockercompose_base64 = base64_encode(Yaml::dump($docker_compose, 4, 2)); $nginxconf_base64 = base64_encode($nginxconf); instant_remote_process(["docker rm -f $proxyContainerName"], $server, false); - instant_remote_process([ - "mkdir -p $configuration_dir", - "echo '{$nginxconf_base64}' | base64 -d | tee $configuration_dir/nginx.conf > /dev/null", - "echo '{$dockercompose_base64}' | base64 -d | tee $configuration_dir/docker-compose.yaml > /dev/null", - "docker compose --project-directory {$configuration_dir} pull", - "docker compose --project-directory {$configuration_dir} up -d", - ], $server); + + try { + instant_remote_process([ + "mkdir -p $configuration_dir", + "echo '{$nginxconf_base64}' | base64 -d | tee $configuration_dir/nginx.conf > /dev/null", + "echo '{$dockercompose_base64}' | base64 -d | tee $configuration_dir/docker-compose.yaml > /dev/null", + "docker compose --project-directory {$configuration_dir} pull", + "docker compose --project-directory {$configuration_dir} up -d", + ], $server); + } catch (\RuntimeException $e) { + if ($this->isNonTransientError($e->getMessage())) { + $database->update(['is_public' => false]); + + $team = data_get($database, 'environment.project.team') + ?? data_get($database, 'service.environment.project.team'); + + $team?->notify( + new \App\Notifications\Container\ContainerRestarted( + "TCP Proxy for {$database->name} database has been disabled due to error: {$e->getMessage()}", + $server, + ) + ); + + ray("Database proxy for {$database->name} disabled due to non-transient error: {$e->getMessage()}"); + + return; + } + + throw $e; + } + } + + private function isNonTransientError(string $message): bool + { + $nonTransientPatterns = [ + 'port is already allocated', + 'address already in use', + 'Bind for', + ]; + + foreach ($nonTransientPatterns as $pattern) { + if (str_contains($message, $pattern)) { + return true; + } + } + + return false; } } diff --git a/app/Console/Commands/Generate/OpenApi.php b/app/Console/Commands/Generate/OpenApi.php index 2b266c258..224c10792 100644 --- a/app/Console/Commands/Generate/OpenApi.php +++ b/app/Console/Commands/Generate/OpenApi.php @@ -32,7 +32,8 @@ public function handle() echo $process->output(); $yaml = file_get_contents('openapi.yaml'); - $json = json_encode(Yaml::parse($yaml), JSON_PRETTY_PRINT); + + $json = json_encode(Yaml::parse($yaml), JSON_PRETTY_PRINT)."\n"; file_put_contents('openapi.json', $json); echo "Converted OpenAPI YAML to JSON.\n"; } diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index c02a7e3c5..5d018cf19 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -207,6 +207,9 @@ public function handle() $serviceId = $labels->get('coolify.serviceId'); $subType = $labels->get('coolify.service.subType'); $subId = $labels->get('coolify.service.subId'); + if (empty($subId)) { + continue; + } if ($subType === 'application') { $this->foundServiceApplicationIds->push($subId); // Store container status for aggregation diff --git a/app/Jobs/ServerCheckJob.php b/app/Jobs/ServerCheckJob.php index 2ac92e72d..a18d45b9a 100644 --- a/app/Jobs/ServerCheckJob.php +++ b/app/Jobs/ServerCheckJob.php @@ -15,6 +15,7 @@ use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\Log; class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue { @@ -33,6 +34,19 @@ public function middleware(): array public function __construct(public Server $server) {} + public function failed(?\Throwable $exception): void + { + if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) { + Log::warning('ServerCheckJob timed out', [ + 'server_id' => $this->server->id, + 'server_name' => $this->server->name, + ]); + + // Delete the queue job so it doesn't appear in Horizon's failed list. + $this->job?->delete(); + } + } + public function handle() { try { diff --git a/app/Jobs/ServerConnectionCheckJob.php b/app/Jobs/ServerConnectionCheckJob.php index 9dbce4bfe..d4a499865 100644 --- a/app/Jobs/ServerConnectionCheckJob.php +++ b/app/Jobs/ServerConnectionCheckJob.php @@ -101,12 +101,31 @@ public function handle() 'is_usable' => false, ]); - throw $e; + return; + } + } + + public function failed(?\Throwable $exception): void + { + if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) { + Log::warning('ServerConnectionCheckJob timed out', [ + 'server_id' => $this->server->id, + 'server_name' => $this->server->name, + ]); + $this->server->settings->update([ + 'is_reachable' => false, + 'is_usable' => false, + ]); + + // Delete the queue job so it doesn't appear in Horizon's failed list. + $this->job?->delete(); } } private function checkHetznerStatus(): void { + $status = null; + try { $hetznerService = new \App\Services\HetznerService($this->server->cloudProviderToken->token); $serverData = $hetznerService->getServer($this->server->hetzner_server_id); diff --git a/app/Jobs/ServerStorageCheckJob.php b/app/Jobs/ServerStorageCheckJob.php index 9d45491c6..51426d880 100644 --- a/app/Jobs/ServerStorageCheckJob.php +++ b/app/Jobs/ServerStorageCheckJob.php @@ -10,6 +10,7 @@ use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\RateLimiter; use Laravel\Horizon\Contracts\Silenced; @@ -28,6 +29,19 @@ public function backoff(): int public function __construct(public Server $server, public int|string|null $percentage = null) {} + public function failed(?\Throwable $exception): void + { + if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) { + Log::warning('ServerStorageCheckJob timed out', [ + 'server_id' => $this->server->id, + 'server_name' => $this->server->name, + ]); + + // Delete the queue job so it doesn't appear in Horizon's failed list. + $this->job?->delete(); + } + } + public function handle() { try { diff --git a/app/Traits/SshRetryable.php b/app/Traits/SshRetryable.php index a26481056..2092dc5f3 100644 --- a/app/Traits/SshRetryable.php +++ b/app/Traits/SshRetryable.php @@ -95,9 +95,6 @@ protected function executeWithSshRetry(callable $callback, array $context = [], if ($this->isRetryableSshError($lastErrorMessage) && $attempt < $maxRetries - 1) { $delay = $this->calculateRetryDelay($attempt); - // Track SSH retry event in Sentry - $this->trackSshRetryEvent($attempt + 1, $maxRetries, $delay, $lastErrorMessage, $context); - // Add deployment log if available (for ExecuteRemoteCommand trait) if (isset($this->application_deployment_queue) && method_exists($this, 'addRetryLogEntry')) { $this->addRetryLogEntry($attempt + 1, $maxRetries, $delay, $lastErrorMessage); @@ -133,42 +130,4 @@ protected function executeWithSshRetry(callable $callback, array $context = [], return null; } - - /** - * Track SSH retry event in Sentry - */ - protected function trackSshRetryEvent(int $attempt, int $maxRetries, int $delay, string $errorMessage, array $context = []): void - { - // Only track in production/cloud instances - if (isDev() || ! config('constants.sentry.sentry_dsn')) { - return; - } - - try { - app('sentry')->captureMessage( - 'SSH connection retry triggered', - \Sentry\Severity::warning(), - [ - 'extra' => [ - 'attempt' => $attempt, - 'max_retries' => $maxRetries, - 'delay_seconds' => $delay, - 'error_message' => $errorMessage, - 'context' => $context, - 'retryable_error' => true, - ], - 'tags' => [ - 'component' => 'ssh_retry', - 'error_type' => 'connection_retry', - ], - ] - ); - } catch (\Throwable $e) { - // Don't let Sentry tracking errors break the SSH retry flow - Log::warning('Failed to track SSH retry event in Sentry', [ - 'error' => $e->getMessage(), - 'original_attempt' => $attempt, - ]); - } - } } diff --git a/openapi.json b/openapi.json index cbd79ca1d..f5da0883f 100644 --- a/openapi.json +++ b/openapi.json @@ -11300,4 +11300,4 @@ "description": "Teams" } ] -} \ No newline at end of file +} diff --git a/tests/Feature/PushServerUpdateJobTest.php b/tests/Feature/PushServerUpdateJobTest.php new file mode 100644 index 000000000..d508d58ab --- /dev/null +++ b/tests/Feature/PushServerUpdateJobTest.php @@ -0,0 +1,76 @@ +create(); + $service = Service::factory()->create([ + 'server_id' => $server->id, + ]); + $serviceApp = ServiceApplication::factory()->create([ + 'service_id' => $service->id, + ]); + + $data = [ + 'containers' => [ + [ + 'name' => 'test-container', + 'state' => 'running', + 'health_status' => 'healthy', + 'labels' => [ + 'coolify.managed' => true, + 'coolify.serviceId' => (string) $service->id, + 'coolify.service.subType' => 'application', + 'coolify.service.subId' => '', + ], + ], + ], + ]; + + $job = new PushServerUpdateJob($server, $data); + + // Run handle - should not throw a PDOException about empty bigint + $job->handle(); + + // The empty subId container should have been skipped + expect($job->foundServiceApplicationIds)->not->toContain(''); + expect($job->serviceContainerStatuses)->toBeEmpty(); +}); + +test('containers with valid service subId are processed', function () { + $server = Server::factory()->create(); + $service = Service::factory()->create([ + 'server_id' => $server->id, + ]); + $serviceApp = ServiceApplication::factory()->create([ + 'service_id' => $service->id, + ]); + + $data = [ + 'containers' => [ + [ + 'name' => 'test-container', + 'state' => 'running', + 'health_status' => 'healthy', + 'labels' => [ + 'coolify.managed' => true, + 'coolify.serviceId' => (string) $service->id, + 'coolify.service.subType' => 'application', + 'coolify.service.subId' => (string) $serviceApp->id, + 'com.docker.compose.service' => 'myapp', + ], + ], + ], + ]; + + $job = new PushServerUpdateJob($server, $data); + $job->handle(); + + expect($job->foundServiceApplicationIds)->toContain((string) $serviceApp->id); +}); diff --git a/tests/Feature/StartDatabaseProxyTest.php b/tests/Feature/StartDatabaseProxyTest.php new file mode 100644 index 000000000..c62569866 --- /dev/null +++ b/tests/Feature/StartDatabaseProxyTest.php @@ -0,0 +1,45 @@ +create(); + + $database = StandalonePostgresql::factory()->create([ + 'team_id' => $team->id, + 'is_public' => true, + 'public_port' => 5432, + ]); + + expect($database->is_public)->toBeTrue(); + + $action = new StartDatabaseProxy; + + // Use reflection to test the private method directly + $method = new ReflectionMethod($action, 'isNonTransientError'); + + expect($method->invoke($action, 'Bind for 0.0.0.0:5432 failed: port is already allocated'))->toBeTrue(); + expect($method->invoke($action, 'address already in use'))->toBeTrue(); + expect($method->invoke($action, 'some other error'))->toBeFalse(); +}); + +test('isNonTransientError detects port conflict patterns', function () { + $action = new StartDatabaseProxy; + $method = new ReflectionMethod($action, 'isNonTransientError'); + + expect($method->invoke($action, 'Bind for 0.0.0.0:5432 failed: port is already allocated'))->toBeTrue() + ->and($method->invoke($action, 'address already in use'))->toBeTrue() + ->and($method->invoke($action, 'Bind for 0.0.0.0:3306 failed: port is already allocated'))->toBeTrue() + ->and($method->invoke($action, 'network timeout'))->toBeFalse() + ->and($method->invoke($action, 'connection refused'))->toBeFalse(); +});