rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add deployment queue limit to prevent queue bombing

Browse source 
- Add configurable deployment_queue_limit server setting (default: 25)
- Check queue size before accepting new deployments
- Return 429 status for webhooks/API when queue is full (allows retry)
- Show error toast in UI when queue limit reached
- Add UI control in Server Advanced settings

Fixes #6708

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai 2025-12-04 13:52:27 +01:00
parent 76afc6841f
commit 6d16f52143
17 changed files with 129 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
app/Http/Controllers/Api/DeployController.php
View file

@ -388,7 +388,11 @@ private function by_uuids(string $uuid, int $teamId, bool $force = false, int $p
continue;
}
}
['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $this->deploy_resource($resource, $force, $pr);
$result = $this->deploy_resource($resource, $force, $pr);
if (isset($result['status']) && $result['status'] === 429) {
return response()->json(['message' => $result['message']], 429);
}
['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $result;
if ($deployment_uuid) {
$deployments->push(['message' => $return_message, 'resource_uuid' => $uuid, 'deployment_uuid' => $deployment_uuid->toString()]);
} else {
@ -430,7 +434,11 @@ public function by_tags(string $tags, int $team_id, bool $force = false)
continue;
}
foreach ($applications as $resource) {
['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $this->deploy_resource($resource, $force);
$result = $this->deploy_resource($resource, $force);
if (isset($result['status']) && $result['status'] === 429) {
return response()->json(['message' => $result['message']], 429);
}
['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $result;
if ($deployment_uuid) {
$deployments->push(['resource_uuid' => $resource->uuid, 'deployment_uuid' => $deployment_uuid->toString()]);
}
@ -474,8 +482,11 @@ public function deploy_resource($resource, bool $force = false, int $pr = 0): ar
deployment_uuid: $deployment_uuid,
force_rebuild: $force,
pull_request_id: $pr,
is_api: true,
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return ['message' => $result['message'], 'deployment_uuid' => null, 'status' => 429];
} elseif ($result['status'] === 'skipped') {
$message = $result['message'];
} else {
$message = "Application {$resource->name} deployment queued.";

8
app/Http/Controllers/Webhook/Bitbucket.php
View file

@ -107,7 +107,9 @@ public function manual(Request $request)
force_rebuild: false,
is_webhook: true
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',
@ -161,7 +163,9 @@ public function manual(Request $request)
is_webhook: true,
git_type: 'bitbucket'
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',

8
app/Http/Controllers/Webhook/Gitea.php
View file

@ -123,7 +123,9 @@ public function manual(Request $request)
commit: data_get($payload, 'after', 'HEAD'),
is_webhook: true,
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',
@ -193,7 +195,9 @@ public function manual(Request $request)
is_webhook: true,
git_type: 'gitea'
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',

17
app/Http/Controllers/Webhook/Github.php
View file

@ -136,7 +136,9 @@ public function manual(Request $request)
commit: data_get($payload, 'after', 'HEAD'),
is_webhook: true,
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',
@ -222,7 +224,9 @@ public function manual(Request $request)
is_webhook: true,
git_type: 'github'
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',
@ -427,12 +431,15 @@ public function normal(Request $request)
force_rebuild: false,
is_webhook: true,
);
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
}
$return_payloads->push([
'status' => $result['status'],
'message' => $result['message'],
'application_uuid' => $application->uuid,
'application_name' => $application->name,
'deployment_uuid' => $result['deployment_uuid'],
'deployment_uuid' => $result['deployment_uuid'] ?? null,
]);
} else {
$paths = str($application->watch_paths)->explode("\n");
@ -491,7 +498,9 @@ public function normal(Request $request)
is_webhook: true,
git_type: 'github'
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',

8
app/Http/Controllers/Webhook/Gitlab.php
View file

@ -149,7 +149,9 @@ public function manual(Request $request)
force_rebuild: false,
is_webhook: true,
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'status' => $result['status'],
'message' => $result['message'],
@ -220,7 +222,9 @@ public function manual(Request $request)
is_webhook: true,
git_type: 'gitlab'
);
if ($result['status'] === 'skipped') {
if ($result['status'] === 'queue_full') {
return response($result['message'], 429);
} elseif ($result['status'] === 'skipped') {
$return_payloads->push([
'application' => $application->name,
'status' => 'skipped',

2
app/Jobs/ApplicationDeploymentJob.php
View file

@ -1813,7 +1813,7 @@ private function health_check()
$this->application->update(['status' => 'running']);
$this->application_deployment_queue->addLogEntry('New container is healthy.');
break;
} elseif (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') {
} elseif (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') {
$this->newVersionIsHealthy = false;
$this->application_deployment_queue->addLogEntry('New container is unhealthy.', type: 'error');
$this->query_logs();

10
app/Livewire/Project/Application/Heading.php
View file

@ -100,6 +100,11 @@ public function deploy(bool $force_rebuild = false)
deployment_uuid: $this->deploymentUuid,
force_rebuild: $force_rebuild,
);
if ($result['status'] === 'queue_full') {
$this->dispatch('error', 'Deployment queue full', $result['message']);
return;
}
if ($result['status'] === 'skipped') {
$this->dispatch('error', 'Deployment skipped', $result['message']);
@ -151,6 +156,11 @@ public function restart()
deployment_uuid: $this->deploymentUuid,
restart_only: true,
);
if ($result['status'] === 'queue_full') {
$this->dispatch('error', 'Deployment queue full', $result['message']);
return;
}
if ($result['status'] === 'skipped') {
$this->dispatch('success', 'Deployment skipped', $result['message']);

5
app/Livewire/Project/Application/Previews.php
View file

@ -249,6 +249,11 @@ public function deploy(int $pull_request_id, ?string $pull_request_html_url = nu
pull_request_id: $pull_request_id,
git_type: $found->git_type ?? null,
);
if ($result['status'] === 'queue_full') {
$this->dispatch('error', 'Deployment queue full', $result['message']);
return;
}
if ($result['status'] === 'skipped') {
$this->dispatch('success', 'Deployment skipped', $result['message']);

8
app/Livewire/Project/Application/Rollback.php
View file

@ -30,7 +30,7 @@ public function rollbackImage($commit)
$deployment_uuid = new Cuid2;
queue_application_deployment(
$result = queue_application_deployment(
application: $this->application,
deployment_uuid: $deployment_uuid,
commit: $commit,
@ -38,6 +38,12 @@ public function rollbackImage($commit)
force_rebuild: false,
);
if ($result['status'] === 'queue_full') {
$this->dispatch('error', 'Deployment queue full', $result['message']);
return;
}
return redirect()->route('project.application.deployment.show', [
'project_uuid' => $this->parameters['project_uuid'],
'application_uuid' => $this->parameters['application_uuid'],

5
app/Livewire/Project/Shared/Destination.php
View file

@ -89,6 +89,11 @@ public function redeploy(int $network_id, int $server_id)
only_this_server: true,
no_questions_asked: true,
);
if ($result['status'] === 'queue_full') {
$this->dispatch('error', 'Deployment queue full', $result['message']);
return;
}
if ($result['status'] === 'skipped') {
$this->dispatch('success', 'Deployment skipped', $result['message']);

5
app/Livewire/Server/Advanced.php
View file

@ -24,6 +24,9 @@ class Advanced extends Component
#[Validate(['integer', 'min:1'])]
public int $dynamicTimeout = 1;
#[Validate(['integer', 'min:1'])]
public int $deploymentQueueLimit = 25;
public function mount(string $server_uuid)
{
try {
@ -43,12 +46,14 @@ public function syncData(bool $toModel = false)
$this->validate();
$this->server->settings->concurrent_builds = $this->concurrentBuilds;
$this->server->settings->dynamic_timeout = $this->dynamicTimeout;
$this->server->settings->deployment_queue_limit = $this->deploymentQueueLimit;
$this->server->settings->server_disk_usage_notification_threshold = $this->serverDiskUsageNotificationThreshold;
$this->server->settings->server_disk_usage_check_frequency = $this->serverDiskUsageCheckFrequency;
$this->server->settings->save();
} else {
$this->concurrentBuilds = $this->server->settings->concurrent_builds;
$this->dynamicTimeout = $this->server->settings->dynamic_timeout;
$this->deploymentQueueLimit = $this->server->settings->deployment_queue_limit;
$this->serverDiskUsageNotificationThreshold = $this->server->settings->server_disk_usage_notification_threshold;
$this->serverDiskUsageCheckFrequency = $this->server->settings->server_disk_usage_check_frequency;
}

1
app/Models/ServerSetting.php
View file

@ -13,6 +13,7 @@
properties: [
'id' => ['type' => 'integer'],
'concurrent_builds' => ['type' => 'integer'],
'deployment_queue_limit' => ['type' => 'integer'],
'dynamic_timeout' => ['type' => 'integer'],
'force_disabled' => ['type' => 'boolean'],
'force_server_cleanup' => ['type' => 'boolean'],

14
bootstrap/helpers/applications.php
View file

@ -28,6 +28,20 @@ function queue_application_deployment(Application $application, string $deployme
$destination_id = $destination->id;
}
// Check if the deployment queue is full for this server
$serverForQueueCheck = $server ?? Server::find($server_id);
$queue_limit = $serverForQueueCheck->settings->deployment_queue_limit ?? 25;
$queued_count = ApplicationDeploymentQueue::where('server_id', $server_id)
->where('status', ApplicationDeploymentStatus::QUEUED->value)
->count();
if ($queued_count >= $queue_limit) {
return [
'status' => 'queue_full',
'message' => 'Deployment queue is full. Please wait for existing deployments to complete.',
];
}
// Check if there's already a deployment in progress or queued for this application and commit
$existing_deployment = ApplicationDeploymentQueue::where('application_id', $application_id)
->where('commit', $commit)

28
database/migrations/2025_12_04_134435_add_deployment_queue_limit_to_server_settings.php Normal file
View file

@ -0,0 +1,28 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
/**
* Run the migrations.
*/
public function up(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->integer('deployment_queue_limit')->default(25)->after('concurrent_builds');
});
}
/**
* Reverse the migrations.
*/
public function down(): void
{
Schema::table('server_settings', function (Blueprint $table) {
$table->dropColumn('deployment_queue_limit');
});
}
};

3
openapi.json
View file

@ -9816,6 +9816,9 @@
"concurrent_builds": {
"type": "integer"
},
"deployment_queue_limit": {
"type": "integer"
},
"dynamic_timeout": {
"type": "integer"
},

2
openapi.yaml
View file

@ -6312,6 +6312,8 @@ components:
type: integer
concurrent_builds:
type: integer
deployment_queue_limit:
type: integer
dynamic_timeout:
type: integer
force_disabled:

3
resources/views/livewire/server/advanced.blade.php
View file

@ -36,6 +36,9 @@
<x-forms.input canGate="update" :canResource="$server" id="dynamicTimeout"
label="Deployment timeout (seconds)" required
helper="You can define the maximum duration for a deployment to run before timing it out." />
<x-forms.input canGate="update" :canResource="$server" id="deploymentQueueLimit"
label="Deployment queue limit" required
helper="Maximum number of queued deployments allowed. New deployments will be rejected with a 429 status when the limit is reached." />
</div>
</div>
</div>