Merge branch 'next' into v4.x
This commit is contained in:
commit
331493a0b0
10 changed files with 222 additions and 51 deletions
|
|
@ -112,12 +112,52 @@ public function handle(StandaloneRedis|StandalonePostgresql|StandaloneMongodb|St
|
|||
$dockercompose_base64 = base64_encode(Yaml::dump($docker_compose, 4, 2));
|
||||
$nginxconf_base64 = base64_encode($nginxconf);
|
||||
instant_remote_process(["docker rm -f $proxyContainerName"], $server, false);
|
||||
instant_remote_process([
|
||||
"mkdir -p $configuration_dir",
|
||||
"echo '{$nginxconf_base64}' | base64 -d | tee $configuration_dir/nginx.conf > /dev/null",
|
||||
"echo '{$dockercompose_base64}' | base64 -d | tee $configuration_dir/docker-compose.yaml > /dev/null",
|
||||
"docker compose --project-directory {$configuration_dir} pull",
|
||||
"docker compose --project-directory {$configuration_dir} up -d",
|
||||
], $server);
|
||||
|
||||
try {
|
||||
instant_remote_process([
|
||||
"mkdir -p $configuration_dir",
|
||||
"echo '{$nginxconf_base64}' | base64 -d | tee $configuration_dir/nginx.conf > /dev/null",
|
||||
"echo '{$dockercompose_base64}' | base64 -d | tee $configuration_dir/docker-compose.yaml > /dev/null",
|
||||
"docker compose --project-directory {$configuration_dir} pull",
|
||||
"docker compose --project-directory {$configuration_dir} up -d",
|
||||
], $server);
|
||||
} catch (\RuntimeException $e) {
|
||||
if ($this->isNonTransientError($e->getMessage())) {
|
||||
$database->update(['is_public' => false]);
|
||||
|
||||
$team = data_get($database, 'environment.project.team')
|
||||
?? data_get($database, 'service.environment.project.team');
|
||||
|
||||
$team?->notify(
|
||||
new \App\Notifications\Container\ContainerRestarted(
|
||||
"TCP Proxy for {$database->name} database has been disabled due to error: {$e->getMessage()}",
|
||||
$server,
|
||||
)
|
||||
);
|
||||
|
||||
ray("Database proxy for {$database->name} disabled due to non-transient error: {$e->getMessage()}");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
throw $e;
|
||||
}
|
||||
}
|
||||
|
||||
private function isNonTransientError(string $message): bool
|
||||
{
|
||||
$nonTransientPatterns = [
|
||||
'port is already allocated',
|
||||
'address already in use',
|
||||
'Bind for',
|
||||
];
|
||||
|
||||
foreach ($nonTransientPatterns as $pattern) {
|
||||
if (str_contains($message, $pattern)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@ public function handle()
|
|||
echo $process->output();
|
||||
|
||||
$yaml = file_get_contents('openapi.yaml');
|
||||
$json = json_encode(Yaml::parse($yaml), JSON_PRETTY_PRINT);
|
||||
|
||||
$json = json_encode(Yaml::parse($yaml), JSON_PRETTY_PRINT)."\n";
|
||||
file_put_contents('openapi.json', $json);
|
||||
echo "Converted OpenAPI YAML to JSON.\n";
|
||||
}
|
||||
|
|
|
|||
|
|
@ -207,6 +207,9 @@ public function handle()
|
|||
$serviceId = $labels->get('coolify.serviceId');
|
||||
$subType = $labels->get('coolify.service.subType');
|
||||
$subId = $labels->get('coolify.service.subId');
|
||||
if (empty($subId)) {
|
||||
continue;
|
||||
}
|
||||
if ($subType === 'application') {
|
||||
$this->foundServiceApplicationIds->push($subId);
|
||||
// Store container status for aggregation
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class ServerCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||
{
|
||||
|
|
@ -33,6 +34,19 @@ public function middleware(): array
|
|||
|
||||
public function __construct(public Server $server) {}
|
||||
|
||||
public function failed(?\Throwable $exception): void
|
||||
{
|
||||
if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
|
||||
Log::warning('ServerCheckJob timed out', [
|
||||
'server_id' => $this->server->id,
|
||||
'server_name' => $this->server->name,
|
||||
]);
|
||||
|
||||
// Delete the queue job so it doesn't appear in Horizon's failed list.
|
||||
$this->job?->delete();
|
||||
}
|
||||
}
|
||||
|
||||
public function handle()
|
||||
{
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -101,12 +101,31 @@ public function handle()
|
|||
'is_usable' => false,
|
||||
]);
|
||||
|
||||
throw $e;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public function failed(?\Throwable $exception): void
|
||||
{
|
||||
if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
|
||||
Log::warning('ServerConnectionCheckJob timed out', [
|
||||
'server_id' => $this->server->id,
|
||||
'server_name' => $this->server->name,
|
||||
]);
|
||||
$this->server->settings->update([
|
||||
'is_reachable' => false,
|
||||
'is_usable' => false,
|
||||
]);
|
||||
|
||||
// Delete the queue job so it doesn't appear in Horizon's failed list.
|
||||
$this->job?->delete();
|
||||
}
|
||||
}
|
||||
|
||||
private function checkHetznerStatus(): void
|
||||
{
|
||||
$status = null;
|
||||
|
||||
try {
|
||||
$hetznerService = new \App\Services\HetznerService($this->server->cloudProviderToken->token);
|
||||
$serverData = $hetznerService->getServer($this->server->hetzner_server_id);
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
use Illuminate\Foundation\Bus\Dispatchable;
|
||||
use Illuminate\Queue\InteractsWithQueue;
|
||||
use Illuminate\Queue\SerializesModels;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\RateLimiter;
|
||||
use Laravel\Horizon\Contracts\Silenced;
|
||||
|
||||
|
|
@ -28,6 +29,19 @@ public function backoff(): int
|
|||
|
||||
public function __construct(public Server $server, public int|string|null $percentage = null) {}
|
||||
|
||||
public function failed(?\Throwable $exception): void
|
||||
{
|
||||
if ($exception instanceof \Illuminate\Queue\TimeoutExceededException) {
|
||||
Log::warning('ServerStorageCheckJob timed out', [
|
||||
'server_id' => $this->server->id,
|
||||
'server_name' => $this->server->name,
|
||||
]);
|
||||
|
||||
// Delete the queue job so it doesn't appear in Horizon's failed list.
|
||||
$this->job?->delete();
|
||||
}
|
||||
}
|
||||
|
||||
public function handle()
|
||||
{
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -95,9 +95,6 @@ protected function executeWithSshRetry(callable $callback, array $context = [],
|
|||
if ($this->isRetryableSshError($lastErrorMessage) && $attempt < $maxRetries - 1) {
|
||||
$delay = $this->calculateRetryDelay($attempt);
|
||||
|
||||
// Track SSH retry event in Sentry
|
||||
$this->trackSshRetryEvent($attempt + 1, $maxRetries, $delay, $lastErrorMessage, $context);
|
||||
|
||||
// Add deployment log if available (for ExecuteRemoteCommand trait)
|
||||
if (isset($this->application_deployment_queue) && method_exists($this, 'addRetryLogEntry')) {
|
||||
$this->addRetryLogEntry($attempt + 1, $maxRetries, $delay, $lastErrorMessage);
|
||||
|
|
@ -133,42 +130,4 @@ protected function executeWithSshRetry(callable $callback, array $context = [],
|
|||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Track SSH retry event in Sentry
|
||||
*/
|
||||
protected function trackSshRetryEvent(int $attempt, int $maxRetries, int $delay, string $errorMessage, array $context = []): void
|
||||
{
|
||||
// Only track in production/cloud instances
|
||||
if (isDev() || ! config('constants.sentry.sentry_dsn')) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
app('sentry')->captureMessage(
|
||||
'SSH connection retry triggered',
|
||||
\Sentry\Severity::warning(),
|
||||
[
|
||||
'extra' => [
|
||||
'attempt' => $attempt,
|
||||
'max_retries' => $maxRetries,
|
||||
'delay_seconds' => $delay,
|
||||
'error_message' => $errorMessage,
|
||||
'context' => $context,
|
||||
'retryable_error' => true,
|
||||
],
|
||||
'tags' => [
|
||||
'component' => 'ssh_retry',
|
||||
'error_type' => 'connection_retry',
|
||||
],
|
||||
]
|
||||
);
|
||||
} catch (\Throwable $e) {
|
||||
// Don't let Sentry tracking errors break the SSH retry flow
|
||||
Log::warning('Failed to track SSH retry event in Sentry', [
|
||||
'error' => $e->getMessage(),
|
||||
'original_attempt' => $attempt,
|
||||
]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11300,4 +11300,4 @@
|
|||
"description": "Teams"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
76
tests/Feature/PushServerUpdateJobTest.php
Normal file
76
tests/Feature/PushServerUpdateJobTest.php
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
<?php
|
||||
|
||||
use App\Jobs\PushServerUpdateJob;
|
||||
use App\Models\Server;
|
||||
use App\Models\Service;
|
||||
use App\Models\ServiceApplication;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
test('containers with empty service subId are skipped', function () {
|
||||
$server = Server::factory()->create();
|
||||
$service = Service::factory()->create([
|
||||
'server_id' => $server->id,
|
||||
]);
|
||||
$serviceApp = ServiceApplication::factory()->create([
|
||||
'service_id' => $service->id,
|
||||
]);
|
||||
|
||||
$data = [
|
||||
'containers' => [
|
||||
[
|
||||
'name' => 'test-container',
|
||||
'state' => 'running',
|
||||
'health_status' => 'healthy',
|
||||
'labels' => [
|
||||
'coolify.managed' => true,
|
||||
'coolify.serviceId' => (string) $service->id,
|
||||
'coolify.service.subType' => 'application',
|
||||
'coolify.service.subId' => '',
|
||||
],
|
||||
],
|
||||
],
|
||||
];
|
||||
|
||||
$job = new PushServerUpdateJob($server, $data);
|
||||
|
||||
// Run handle - should not throw a PDOException about empty bigint
|
||||
$job->handle();
|
||||
|
||||
// The empty subId container should have been skipped
|
||||
expect($job->foundServiceApplicationIds)->not->toContain('');
|
||||
expect($job->serviceContainerStatuses)->toBeEmpty();
|
||||
});
|
||||
|
||||
test('containers with valid service subId are processed', function () {
|
||||
$server = Server::factory()->create();
|
||||
$service = Service::factory()->create([
|
||||
'server_id' => $server->id,
|
||||
]);
|
||||
$serviceApp = ServiceApplication::factory()->create([
|
||||
'service_id' => $service->id,
|
||||
]);
|
||||
|
||||
$data = [
|
||||
'containers' => [
|
||||
[
|
||||
'name' => 'test-container',
|
||||
'state' => 'running',
|
||||
'health_status' => 'healthy',
|
||||
'labels' => [
|
||||
'coolify.managed' => true,
|
||||
'coolify.serviceId' => (string) $service->id,
|
||||
'coolify.service.subType' => 'application',
|
||||
'coolify.service.subId' => (string) $serviceApp->id,
|
||||
'com.docker.compose.service' => 'myapp',
|
||||
],
|
||||
],
|
||||
],
|
||||
];
|
||||
|
||||
$job = new PushServerUpdateJob($server, $data);
|
||||
$job->handle();
|
||||
|
||||
expect($job->foundServiceApplicationIds)->toContain((string) $serviceApp->id);
|
||||
});
|
||||
45
tests/Feature/StartDatabaseProxyTest.php
Normal file
45
tests/Feature/StartDatabaseProxyTest.php
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
<?php
|
||||
|
||||
use App\Actions\Database\StartDatabaseProxy;
|
||||
use App\Models\StandalonePostgresql;
|
||||
use App\Models\Team;
|
||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||
use Illuminate\Support\Facades\Notification;
|
||||
|
||||
uses(RefreshDatabase::class);
|
||||
|
||||
beforeEach(function () {
|
||||
Notification::fake();
|
||||
});
|
||||
|
||||
test('database proxy is disabled on port already allocated error', function () {
|
||||
$team = Team::factory()->create();
|
||||
|
||||
$database = StandalonePostgresql::factory()->create([
|
||||
'team_id' => $team->id,
|
||||
'is_public' => true,
|
||||
'public_port' => 5432,
|
||||
]);
|
||||
|
||||
expect($database->is_public)->toBeTrue();
|
||||
|
||||
$action = new StartDatabaseProxy;
|
||||
|
||||
// Use reflection to test the private method directly
|
||||
$method = new ReflectionMethod($action, 'isNonTransientError');
|
||||
|
||||
expect($method->invoke($action, 'Bind for 0.0.0.0:5432 failed: port is already allocated'))->toBeTrue();
|
||||
expect($method->invoke($action, 'address already in use'))->toBeTrue();
|
||||
expect($method->invoke($action, 'some other error'))->toBeFalse();
|
||||
});
|
||||
|
||||
test('isNonTransientError detects port conflict patterns', function () {
|
||||
$action = new StartDatabaseProxy;
|
||||
$method = new ReflectionMethod($action, 'isNonTransientError');
|
||||
|
||||
expect($method->invoke($action, 'Bind for 0.0.0.0:5432 failed: port is already allocated'))->toBeTrue()
|
||||
->and($method->invoke($action, 'address already in use'))->toBeTrue()
|
||||
->and($method->invoke($action, 'Bind for 0.0.0.0:3306 failed: port is already allocated'))->toBeTrue()
|
||||
->and($method->invoke($action, 'network timeout'))->toBeFalse()
|
||||
->and($method->invoke($action, 'connection refused'))->toBeFalse();
|
||||
});
|
||||
Loading…
Reference in a new issue