rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Run proxy restart as background job with real-time logs (#7475)

Browse source
This commit is contained in:
Andras Bacsai 2025-12-04 14:59:50 +01:00 committed by GitHub
commit 9e0fa03434
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 378 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
app/Events/ProxyStatusChangedUI.php
View file

@ -14,12 +14,15 @@ class ProxyStatusChangedUI implements ShouldBroadcast
public ?int $teamId = null;
public function __construct(?int $teamId = null)
public ?int $activityId = null;
public function __construct(?int $teamId = null, ?int $activityId = null)
{
if (is_null($teamId) && auth()->check() && auth()->user()->currentTeam()) {
$teamId = auth()->user()->currentTeam()->id;
}
$this->teamId = $teamId;
$this->activityId = $activityId;
}
public function broadcastOn(): array

129
app/Jobs/RestartProxyJob.php
View file

@ -2,9 +2,12 @@
namespace App\Jobs;
use App\Actions\Proxy\StartProxy;
use App\Actions\Proxy\StopProxy;
use App\Actions\Proxy\GetProxyConfiguration;
use App\Actions\Proxy\SaveProxyConfiguration;
use App\Enums\ProxyTypes;
use App\Events\ProxyStatusChangedUI;
use App\Models\Server;
use App\Services\ProxyDashboardCacheService;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
@ -19,11 +22,13 @@ class RestartProxyJob implements ShouldBeEncrypted, ShouldQueue
public $tries = 1;
public $timeout = 60;
public $timeout = 120;
public ?int $activity_id = null;
public function middleware(): array
{
return [(new WithoutOverlapping('restart-proxy-'.$this->server->uuid))->expireAfter(60)->dontRelease()];
return [(new WithoutOverlapping('restart-proxy-'.$this->server->uuid))->expireAfter(120)->dontRelease()];
}
public function __construct(public Server $server) {}
@ -31,15 +36,125 @@ public function __construct(public Server $server) {}
public function handle()
{
try {
StopProxy::run($this->server, restarting: true);
// Set status to restarting
$this->server->proxy->status = 'restarting';
$this->server->proxy->force_stop = false;
$this->server->save();
StartProxy::run($this->server, force: true, restarting: true);
// Build combined stop + start commands for a single activity
$commands = $this->buildRestartCommands();
// Create activity and dispatch immediately - returns Activity right away
// The remote_process runs asynchronously, so UI gets activity ID instantly
$activity = remote_process(
$commands,
$this->server,
callEventOnFinish: 'ProxyStatusChanged',
callEventData: $this->server->id
);
// Store activity ID and notify UI immediately with it
$this->activity_id = $activity->id;
ProxyStatusChangedUI::dispatch($this->server->team_id, $this->activity_id);
} catch (\Throwable $e) {
// Set error status
$this->server->proxy->status = 'error';
$this->server->save();
// Notify UI of error
ProxyStatusChangedUI::dispatch($this->server->team_id);
// Clear dashboard cache on error
ProxyDashboardCacheService::clearCache($this->server);
return handleError($e);
}
}
/**
* Build combined stop + start commands for proxy restart.
* This creates a single command sequence that shows all logs in one activity.
*/
private function buildRestartCommands(): array
{
$proxyType = $this->server->proxyType();
$containerName = $this->server->isSwarm() ? 'coolify-proxy_traefik' : 'coolify-proxy';
$proxy_path = $this->server->proxyPath();
$stopTimeout = 30;
// Get proxy configuration
$configuration = GetProxyConfiguration::run($this->server);
if (! $configuration) {
throw new \Exception('Configuration is not synced');
}
SaveProxyConfiguration::run($this->server, $configuration);
$docker_compose_yml_base64 = base64_encode($configuration);
$this->server->proxy->last_applied_settings = str($docker_compose_yml_base64)->pipe('md5')->value();
$this->server->save();
$commands = collect([]);
// === STOP PHASE ===
$commands = $commands->merge([
"echo 'Stopping proxy...'",
"docker stop -t=$stopTimeout $containerName 2>/dev/null || true",
"docker rm -f $containerName 2>/dev/null || true",
'# Wait for container to be fully removed',
'for i in {1..15}; do',
" if ! docker ps -a --format \"{{.Names}}\" | grep -q \"^$containerName$\"; then",
" echo 'Container removed successfully.'",
' break',
' fi',
' echo "Waiting for container to be removed... ($i/15)"',
' sleep 1',
' # Force remove on each iteration in case it got stuck',
" docker rm -f $containerName 2>/dev/null || true",
'done',
'# Final verification and force cleanup',
"if docker ps -a --format \"{{.Names}}\" | grep -q \"^$containerName$\"; then",
" echo 'Container still exists after wait, forcing removal...'",
" docker rm -f $containerName 2>/dev/null || true",
' sleep 2',
'fi',
"echo 'Proxy stopped successfully.'",
]);
// === START PHASE ===
if ($this->server->isSwarmManager()) {
$commands = $commands->merge([
"echo 'Starting proxy (Swarm mode)...'",
"mkdir -p $proxy_path/dynamic",
"cd $proxy_path",
"echo 'Creating required Docker Compose file.'",
"echo 'Starting coolify-proxy.'",
'docker stack deploy --detach=true -c docker-compose.yml coolify-proxy',
"echo 'Successfully started coolify-proxy.'",
]);
} else {
if (isDev() && $proxyType === ProxyTypes::CADDY->value) {
$proxy_path = '/data/coolify/proxy/caddy';
}
$caddyfile = 'import /dynamic/*.caddy';
$commands = $commands->merge([
"echo 'Starting proxy...'",
"mkdir -p $proxy_path/dynamic",
"cd $proxy_path",
"echo '$caddyfile' > $proxy_path/dynamic/Caddyfile",
"echo 'Creating required Docker Compose file.'",
"echo 'Pulling docker image.'",
'docker compose pull',
]);
// Ensure required networks exist BEFORE docker compose up
$commands = $commands->merge(ensureProxyNetworksExist($this->server));
$commands = $commands->merge([
"echo 'Starting coolify-proxy.'",
'docker compose up -d --wait --remove-orphans',
"echo 'Successfully started coolify-proxy.'",
]);
$commands = $commands->merge(connectProxyToNetworks($this->server));
}
return $commands->toArray();
}
}

57
app/Livewire/Server/Navbar.php
View file

@ -6,6 +6,7 @@
use App\Actions\Proxy\StartProxy;
use App\Actions\Proxy\StopProxy;
use App\Enums\ProxyTypes;
use App\Jobs\RestartProxyJob;
use App\Models\Server;
use App\Services\ProxyDashboardCacheService;
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
@ -27,6 +28,10 @@ class Navbar extends Component
public ?string $proxyStatus = 'unknown';
public ?string $lastNotifiedStatus = null;
public bool $restartInitiated = false;
public function getListeners()
{
$teamId = auth()->user()->currentTeam()->id;
@ -61,14 +66,19 @@ public function restart()
{
try {
$this->authorize('manageProxy', $this->server);
StopProxy::run($this->server, restarting: true);
$this->server->proxy->force_stop = false;
$this->server->save();
// Prevent duplicate restart calls
if ($this->restartInitiated) {
return;
}
$this->restartInitiated = true;
// Always use background job for all servers
RestartProxyJob::dispatch($this->server);
$activity = StartProxy::run($this->server, force: true, restarting: true);
$this->dispatch('activityMonitor', $activity->id);
} catch (\Throwable $e) {
$this->restartInitiated = false;
return handleError($e, $this);
}
}
@ -122,12 +132,27 @@ public function checkProxyStatus()
}
}
public function showNotification()
public function showNotification($event = null)
{
$previousStatus = $this->proxyStatus;
$this->server->refresh();
$this->proxyStatus = $this->server->proxy->status ?? 'unknown';
// If event contains activityId, open activity monitor
if ($event && isset($event['activityId'])) {
$this->dispatch('activityMonitor', $event['activityId']);
}
// Reset restart flag when proxy reaches a stable state
if (in_array($this->proxyStatus, ['running', 'exited', 'error'])) {
$this->restartInitiated = false;
}
// Skip notification if we already notified about this status (prevents duplicates)
if ($this->lastNotifiedStatus === $this->proxyStatus) {
return;
}
switch ($this->proxyStatus) {
case 'running':
$this->loadProxyConfiguration();
@ -135,6 +160,7 @@ public function showNotification()
// Don't show during normal start/restart flows (starting, restarting, stopping)
if (in_array($previousStatus, ['exited', 'stopped', 'unknown', null])) {
$this->dispatch('success', 'Proxy is running.');
$this->lastNotifiedStatus = $this->proxyStatus;
}
break;
case 'exited':
@ -142,19 +168,30 @@ public function showNotification()
// Don't show during normal stop/restart flows (stopping, restarting)
if (in_array($previousStatus, ['running'])) {
$this->dispatch('info', 'Proxy has exited.');
$this->lastNotifiedStatus = $this->proxyStatus;
}
break;
case 'stopping':
$this->dispatch('info', 'Proxy is stopping.');
// $this->dispatch('info', 'Proxy is stopping.');
$this->lastNotifiedStatus = $this->proxyStatus;
break;
case 'starting':
$this->dispatch('info', 'Proxy is starting.');
// $this->dispatch('info', 'Proxy is starting.');
$this->lastNotifiedStatus = $this->proxyStatus;
break;
case 'restarting':
// $this->dispatch('info', 'Proxy is restarting.');
$this->lastNotifiedStatus = $this->proxyStatus;
break;
case 'error':
$this->dispatch('error', 'Proxy restart failed. Check logs.');
$this->lastNotifiedStatus = $this->proxyStatus;
break;
case 'unknown':
$this->dispatch('info', 'Proxy status is unknown.');
// Don't notify for unknown status - too noisy
break;
default:
$this->dispatch('info', 'Proxy status updated.');
// Don't notify for other statuses
break;
}

8
resources/views/livewire/server/navbar.blade.php
View file

@ -2,6 +2,13 @@
<x-slide-over @startproxy.window="slideOverOpen = true" fullScreen closeWithX>
<x-slot:title>Proxy Startup Logs</x-slot:title>
<x-slot:content>
@if ($server->id === 0)
<div class="mb-4 p-3 text-sm bg-warning/10 border border-warning/30 rounded-lg text-warning">
<span class="font-semibold">Note:</span> This is the localhost server where Coolify runs.
During proxy restart, the connection may be temporarily lost.
If logs stop updating, please refresh the browser after a few minutes.
</div>
@endif
<livewire:activity-monitor header="Logs" fullHeight />
</x-slot:content>
</x-slide-over>
@ -174,6 +181,7 @@ class="flex items-center gap-6 overflow-x-scroll sm:overflow-x-hidden scrollbar
}
});
$wire.$on('restartEvent', () => {
if ($wire.restartInitiated) return;
window.dispatchEvent(new CustomEvent('startproxy'))
$wire.$call('restart');
});

139
tests/Feature/Proxy/RestartProxyTest.php Normal file
View file

@ -0,0 +1,139 @@
<?php
namespace Tests\Feature\Proxy;
use App\Jobs\RestartProxyJob;
use App\Models\Server;
use App\Models\Team;
use App\Models\User;
use Illuminate\Foundation\Testing\RefreshDatabase;
use Illuminate\Support\Facades\Queue;
use Livewire\Livewire;
use Tests\TestCase;
class RestartProxyTest extends TestCase
{
use RefreshDatabase;
protected User $user;
protected Team $team;
protected Server $server;
protected function setUp(): void
{
parent::setUp();
// Create test user and team
$this->user = User::factory()->create();
$this->team = Team::factory()->create(['name' => 'Test Team']);
$this->user->teams()->attach($this->team);
// Create test server
$this->server = Server::factory()->create([
'team_id' => $this->team->id,
'name' => 'Test Server',
'ip' => '192.168.1.100',
]);
// Authenticate user
$this->actingAs($this->user);
}
public function test_restart_dispatches_job_for_all_servers()
{
Queue::fake();
Livewire::test('server.navbar', ['server' => $this->server])
->call('restart');
// Assert job was dispatched
Queue::assertPushed(RestartProxyJob::class, function ($job) {
return $job->server->id === $this->server->id;
});
}
public function test_restart_dispatches_job_for_localhost_server()
{
Queue::fake();
// Create localhost server (id = 0)
$localhostServer = Server::factory()->create([
'id' => 0,
'team_id' => $this->team->id,
'name' => 'Localhost',
'ip' => 'host.docker.internal',
]);
Livewire::test('server.navbar', ['server' => $localhostServer])
->call('restart');
// Assert job was dispatched
Queue::assertPushed(RestartProxyJob::class, function ($job) use ($localhostServer) {
return $job->server->id === $localhostServer->id;
});
}
public function test_restart_shows_info_message()
{
Queue::fake();
Livewire::test('server.navbar', ['server' => $this->server])
->call('restart')
->assertDispatched('info', 'Proxy restart initiated. Monitor progress in activity logs.');
}
public function test_unauthorized_user_cannot_restart_proxy()
{
Queue::fake();
// Create another user without access
$unauthorizedUser = User::factory()->create();
$this->actingAs($unauthorizedUser);
Livewire::test('server.navbar', ['server' => $this->server])
->call('restart')
->assertForbidden();
// Assert job was NOT dispatched
Queue::assertNotPushed(RestartProxyJob::class);
}
public function test_restart_prevents_concurrent_jobs_via_without_overlapping()
{
Queue::fake();
// Dispatch job twice
Livewire::test('server.navbar', ['server' => $this->server])
->call('restart');
Livewire::test('server.navbar', ['server' => $this->server])
->call('restart');
// Assert job was pushed twice (WithoutOverlapping middleware will handle deduplication)
Queue::assertPushed(RestartProxyJob::class, 2);
// Get the jobs
$jobs = Queue::pushed(RestartProxyJob::class);
// Verify both jobs have WithoutOverlapping middleware
foreach ($jobs as $job) {
$middleware = $job['job']->middleware();
$this->assertCount(1, $middleware);
$this->assertInstanceOf(\Illuminate\Queue\Middleware\WithoutOverlapping::class, $middleware[0]);
}
}
public function test_restart_uses_server_team_id()
{
Queue::fake();
Livewire::test('server.navbar', ['server' => $this->server])
->call('restart');
Queue::assertPushed(RestartProxyJob::class, function ($job) {
return $job->server->team_id === $this->team->id;
});
}
}

58
tests/Unit/Jobs/RestartProxyJobTest.php Normal file
View file

@ -0,0 +1,58 @@
<?php
namespace Tests\Unit\Jobs;
use App\Jobs\RestartProxyJob;
use App\Models\Server;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Mockery;
use Tests\TestCase;
/**
* Unit tests for RestartProxyJob.
*
* These tests focus on testing the job's middleware configuration and constructor.
* Full integration tests for the job's handle() method are in tests/Feature/Proxy/
* because they require database and complex mocking of SchemalessAttributes.
*/
class RestartProxyJobTest extends TestCase
{
protected function tearDown(): void
{
Mockery::close();
parent::tearDown();
}
public function test_job_has_without_overlapping_middleware()
{
$server = Mockery::mock(Server::class);
$server->shouldReceive('getSchemalessAttributes')->andReturn([]);
$server->shouldReceive('getAttribute')->with('uuid')->andReturn('test-uuid');
$job = new RestartProxyJob($server);
$middleware = $job->middleware();
$this->assertCount(1, $middleware);
$this->assertInstanceOf(WithoutOverlapping::class, $middleware[0]);
}
public function test_job_has_correct_configuration()
{
$server = Mockery::mock(Server::class);
$job = new RestartProxyJob($server);
$this->assertEquals(1, $job->tries);
$this->assertEquals(120, $job->timeout);
$this->assertNull($job->activity_id);
}
public function test_job_stores_server()
{
$server = Mockery::mock(Server::class);
$job = new RestartProxyJob($server);
$this->assertSame($server, $job->server);
}
}