From 6d16f521430c050539f1859919c72cd5e68f7ddc Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 4 Dec 2025 13:52:27 +0100 Subject: [PATCH 1/2] Add deployment queue limit to prevent queue bombing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add configurable deployment_queue_limit server setting (default: 25) - Check queue size before accepting new deployments - Return 429 status for webhooks/API when queue is full (allows retry) - Show error toast in UI when queue limit reached - Add UI control in Server Advanced settings Fixes #6708 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/Http/Controllers/Api/DeployController.php | 17 +++++++++-- app/Http/Controllers/Webhook/Bitbucket.php | 8 ++++-- app/Http/Controllers/Webhook/Gitea.php | 8 ++++-- app/Http/Controllers/Webhook/Github.php | 17 ++++++++--- app/Http/Controllers/Webhook/Gitlab.php | 8 ++++-- app/Jobs/ApplicationDeploymentJob.php | 2 +- app/Livewire/Project/Application/Heading.php | 10 +++++++ app/Livewire/Project/Application/Previews.php | 5 ++++ app/Livewire/Project/Application/Rollback.php | 8 +++++- app/Livewire/Project/Shared/Destination.php | 5 ++++ app/Livewire/Server/Advanced.php | 5 ++++ app/Models/ServerSetting.php | 1 + bootstrap/helpers/applications.php | 14 ++++++++++ ...loyment_queue_limit_to_server_settings.php | 28 +++++++++++++++++++ openapi.json | 3 ++ openapi.yaml | 2 ++ .../views/livewire/server/advanced.blade.php | 3 ++ 17 files changed, 129 insertions(+), 15 deletions(-) create mode 100644 database/migrations/2025_12_04_134435_add_deployment_queue_limit_to_server_settings.php diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.php index 16a7b6f71..26378c3bd 100644 --- a/app/Http/Controllers/Api/DeployController.php +++ b/app/Http/Controllers/Api/DeployController.php @@ -388,7 +388,11 @@ private function by_uuids(string $uuid, int $teamId, bool $force = false, int $p continue; } } - ['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $this->deploy_resource($resource, $force, $pr); + $result = $this->deploy_resource($resource, $force, $pr); + if (isset($result['status']) && $result['status'] === 429) { + return response()->json(['message' => $result['message']], 429); + } + ['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $result; if ($deployment_uuid) { $deployments->push(['message' => $return_message, 'resource_uuid' => $uuid, 'deployment_uuid' => $deployment_uuid->toString()]); } else { @@ -430,7 +434,11 @@ public function by_tags(string $tags, int $team_id, bool $force = false) continue; } foreach ($applications as $resource) { - ['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $this->deploy_resource($resource, $force); + $result = $this->deploy_resource($resource, $force); + if (isset($result['status']) && $result['status'] === 429) { + return response()->json(['message' => $result['message']], 429); + } + ['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $result; if ($deployment_uuid) { $deployments->push(['resource_uuid' => $resource->uuid, 'deployment_uuid' => $deployment_uuid->toString()]); } @@ -474,8 +482,11 @@ public function deploy_resource($resource, bool $force = false, int $pr = 0): ar deployment_uuid: $deployment_uuid, force_rebuild: $force, pull_request_id: $pr, + is_api: true, ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return ['message' => $result['message'], 'deployment_uuid' => null, 'status' => 429]; + } elseif ($result['status'] === 'skipped') { $message = $result['message']; } else { $message = "Application {$resource->name} deployment queued."; diff --git a/app/Http/Controllers/Webhook/Bitbucket.php b/app/Http/Controllers/Webhook/Bitbucket.php index 078494f82..5410564c8 100644 --- a/app/Http/Controllers/Webhook/Bitbucket.php +++ b/app/Http/Controllers/Webhook/Bitbucket.php @@ -107,7 +107,9 @@ public function manual(Request $request) force_rebuild: false, is_webhook: true ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', @@ -161,7 +163,9 @@ public function manual(Request $request) is_webhook: true, git_type: 'bitbucket' ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', diff --git a/app/Http/Controllers/Webhook/Gitea.php b/app/Http/Controllers/Webhook/Gitea.php index 3e0c5a0b6..8f9cdba0c 100644 --- a/app/Http/Controllers/Webhook/Gitea.php +++ b/app/Http/Controllers/Webhook/Gitea.php @@ -123,7 +123,9 @@ public function manual(Request $request) commit: data_get($payload, 'after', 'HEAD'), is_webhook: true, ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', @@ -193,7 +195,9 @@ public function manual(Request $request) is_webhook: true, git_type: 'gitea' ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', diff --git a/app/Http/Controllers/Webhook/Github.php b/app/Http/Controllers/Webhook/Github.php index a1fcaa7f5..e0ccf0850 100644 --- a/app/Http/Controllers/Webhook/Github.php +++ b/app/Http/Controllers/Webhook/Github.php @@ -136,7 +136,9 @@ public function manual(Request $request) commit: data_get($payload, 'after', 'HEAD'), is_webhook: true, ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', @@ -222,7 +224,9 @@ public function manual(Request $request) is_webhook: true, git_type: 'github' ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', @@ -427,12 +431,15 @@ public function normal(Request $request) force_rebuild: false, is_webhook: true, ); + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } $return_payloads->push([ 'status' => $result['status'], 'message' => $result['message'], 'application_uuid' => $application->uuid, 'application_name' => $application->name, - 'deployment_uuid' => $result['deployment_uuid'], + 'deployment_uuid' => $result['deployment_uuid'] ?? null, ]); } else { $paths = str($application->watch_paths)->explode("\n"); @@ -491,7 +498,9 @@ public function normal(Request $request) is_webhook: true, git_type: 'github' ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', diff --git a/app/Http/Controllers/Webhook/Gitlab.php b/app/Http/Controllers/Webhook/Gitlab.php index 3187663d4..004ab0e59 100644 --- a/app/Http/Controllers/Webhook/Gitlab.php +++ b/app/Http/Controllers/Webhook/Gitlab.php @@ -149,7 +149,9 @@ public function manual(Request $request) force_rebuild: false, is_webhook: true, ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'status' => $result['status'], 'message' => $result['message'], @@ -220,7 +222,9 @@ public function manual(Request $request) is_webhook: true, git_type: 'gitlab' ); - if ($result['status'] === 'skipped') { + if ($result['status'] === 'queue_full') { + return response($result['message'], 429); + } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, 'status' => 'skipped', diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index bcd7a729d..b6facba22 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -1813,7 +1813,7 @@ private function health_check() $this->application->update(['status' => 'running']); $this->application_deployment_queue->addLogEntry('New container is healthy.'); break; - } elseif (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') { + } elseif (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') { $this->newVersionIsHealthy = false; $this->application_deployment_queue->addLogEntry('New container is unhealthy.', type: 'error'); $this->query_logs(); diff --git a/app/Livewire/Project/Application/Heading.php b/app/Livewire/Project/Application/Heading.php index fc63c7f4b..523383e2b 100644 --- a/app/Livewire/Project/Application/Heading.php +++ b/app/Livewire/Project/Application/Heading.php @@ -100,6 +100,11 @@ public function deploy(bool $force_rebuild = false) deployment_uuid: $this->deploymentUuid, force_rebuild: $force_rebuild, ); + if ($result['status'] === 'queue_full') { + $this->dispatch('error', 'Deployment queue full', $result['message']); + + return; + } if ($result['status'] === 'skipped') { $this->dispatch('error', 'Deployment skipped', $result['message']); @@ -151,6 +156,11 @@ public function restart() deployment_uuid: $this->deploymentUuid, restart_only: true, ); + if ($result['status'] === 'queue_full') { + $this->dispatch('error', 'Deployment queue full', $result['message']); + + return; + } if ($result['status'] === 'skipped') { $this->dispatch('success', 'Deployment skipped', $result['message']); diff --git a/app/Livewire/Project/Application/Previews.php b/app/Livewire/Project/Application/Previews.php index 45371678b..41f352c14 100644 --- a/app/Livewire/Project/Application/Previews.php +++ b/app/Livewire/Project/Application/Previews.php @@ -249,6 +249,11 @@ public function deploy(int $pull_request_id, ?string $pull_request_html_url = nu pull_request_id: $pull_request_id, git_type: $found->git_type ?? null, ); + if ($result['status'] === 'queue_full') { + $this->dispatch('error', 'Deployment queue full', $result['message']); + + return; + } if ($result['status'] === 'skipped') { $this->dispatch('success', 'Deployment skipped', $result['message']); diff --git a/app/Livewire/Project/Application/Rollback.php b/app/Livewire/Project/Application/Rollback.php index da67a5707..c915ef212 100644 --- a/app/Livewire/Project/Application/Rollback.php +++ b/app/Livewire/Project/Application/Rollback.php @@ -30,7 +30,7 @@ public function rollbackImage($commit) $deployment_uuid = new Cuid2; - queue_application_deployment( + $result = queue_application_deployment( application: $this->application, deployment_uuid: $deployment_uuid, commit: $commit, @@ -38,6 +38,12 @@ public function rollbackImage($commit) force_rebuild: false, ); + if ($result['status'] === 'queue_full') { + $this->dispatch('error', 'Deployment queue full', $result['message']); + + return; + } + return redirect()->route('project.application.deployment.show', [ 'project_uuid' => $this->parameters['project_uuid'], 'application_uuid' => $this->parameters['application_uuid'], diff --git a/app/Livewire/Project/Shared/Destination.php b/app/Livewire/Project/Shared/Destination.php index 40291d2b0..28e3f23e7 100644 --- a/app/Livewire/Project/Shared/Destination.php +++ b/app/Livewire/Project/Shared/Destination.php @@ -89,6 +89,11 @@ public function redeploy(int $network_id, int $server_id) only_this_server: true, no_questions_asked: true, ); + if ($result['status'] === 'queue_full') { + $this->dispatch('error', 'Deployment queue full', $result['message']); + + return; + } if ($result['status'] === 'skipped') { $this->dispatch('success', 'Deployment skipped', $result['message']); diff --git a/app/Livewire/Server/Advanced.php b/app/Livewire/Server/Advanced.php index 8d17bb557..dba1b4903 100644 --- a/app/Livewire/Server/Advanced.php +++ b/app/Livewire/Server/Advanced.php @@ -24,6 +24,9 @@ class Advanced extends Component #[Validate(['integer', 'min:1'])] public int $dynamicTimeout = 1; + #[Validate(['integer', 'min:1'])] + public int $deploymentQueueLimit = 25; + public function mount(string $server_uuid) { try { @@ -43,12 +46,14 @@ public function syncData(bool $toModel = false) $this->validate(); $this->server->settings->concurrent_builds = $this->concurrentBuilds; $this->server->settings->dynamic_timeout = $this->dynamicTimeout; + $this->server->settings->deployment_queue_limit = $this->deploymentQueueLimit; $this->server->settings->server_disk_usage_notification_threshold = $this->serverDiskUsageNotificationThreshold; $this->server->settings->server_disk_usage_check_frequency = $this->serverDiskUsageCheckFrequency; $this->server->settings->save(); } else { $this->concurrentBuilds = $this->server->settings->concurrent_builds; $this->dynamicTimeout = $this->server->settings->dynamic_timeout; + $this->deploymentQueueLimit = $this->server->settings->deployment_queue_limit; $this->serverDiskUsageNotificationThreshold = $this->server->settings->server_disk_usage_notification_threshold; $this->serverDiskUsageCheckFrequency = $this->server->settings->server_disk_usage_check_frequency; } diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index 6da4dd4c6..af301d891 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -13,6 +13,7 @@ properties: [ 'id' => ['type' => 'integer'], 'concurrent_builds' => ['type' => 'integer'], + 'deployment_queue_limit' => ['type' => 'integer'], 'dynamic_timeout' => ['type' => 'integer'], 'force_disabled' => ['type' => 'boolean'], 'force_server_cleanup' => ['type' => 'boolean'], diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 7a36c4b63..03c53989c 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -28,6 +28,20 @@ function queue_application_deployment(Application $application, string $deployme $destination_id = $destination->id; } + // Check if the deployment queue is full for this server + $serverForQueueCheck = $server ?? Server::find($server_id); + $queue_limit = $serverForQueueCheck->settings->deployment_queue_limit ?? 25; + $queued_count = ApplicationDeploymentQueue::where('server_id', $server_id) + ->where('status', ApplicationDeploymentStatus::QUEUED->value) + ->count(); + + if ($queued_count >= $queue_limit) { + return [ + 'status' => 'queue_full', + 'message' => 'Deployment queue is full. Please wait for existing deployments to complete.', + ]; + } + // Check if there's already a deployment in progress or queued for this application and commit $existing_deployment = ApplicationDeploymentQueue::where('application_id', $application_id) ->where('commit', $commit) diff --git a/database/migrations/2025_12_04_134435_add_deployment_queue_limit_to_server_settings.php b/database/migrations/2025_12_04_134435_add_deployment_queue_limit_to_server_settings.php new file mode 100644 index 000000000..a1bcab5bb --- /dev/null +++ b/database/migrations/2025_12_04_134435_add_deployment_queue_limit_to_server_settings.php @@ -0,0 +1,28 @@ +integer('deployment_queue_limit')->default(25)->after('concurrent_builds'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('server_settings', function (Blueprint $table) { + $table->dropColumn('deployment_queue_limit'); + }); + } +}; diff --git a/openapi.json b/openapi.json index dd3c6783a..1c2a1e61e 100644 --- a/openapi.json +++ b/openapi.json @@ -9816,6 +9816,9 @@ "concurrent_builds": { "type": "integer" }, + "deployment_queue_limit": { + "type": "integer" + }, "dynamic_timeout": { "type": "integer" }, diff --git a/openapi.yaml b/openapi.yaml index 754b7ec6f..bbd9294c1 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -6312,6 +6312,8 @@ components: type: integer concurrent_builds: type: integer + deployment_queue_limit: + type: integer dynamic_timeout: type: integer force_disabled: diff --git a/resources/views/livewire/server/advanced.blade.php b/resources/views/livewire/server/advanced.blade.php index 6622961c5..33086aea1 100644 --- a/resources/views/livewire/server/advanced.blade.php +++ b/resources/views/livewire/server/advanced.blade.php @@ -36,6 +36,9 @@ + From d0195538093905bba9f35cfb4c2d638fdc7f7caf Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 11 Dec 2025 11:02:29 +0100 Subject: [PATCH 2/2] Add Retry-After header to 429 rate limit responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Retry-After: 60 header to all deployment queue full responses, helping webhook clients know when to retry their requests. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/Http/Controllers/Api/DeployController.php | 4 ++-- app/Http/Controllers/Webhook/Bitbucket.php | 4 ++-- app/Http/Controllers/Webhook/Gitea.php | 4 ++-- app/Http/Controllers/Webhook/Github.php | 8 ++++---- app/Http/Controllers/Webhook/Gitlab.php | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.php index 26378c3bd..136fcf557 100644 --- a/app/Http/Controllers/Api/DeployController.php +++ b/app/Http/Controllers/Api/DeployController.php @@ -390,7 +390,7 @@ private function by_uuids(string $uuid, int $teamId, bool $force = false, int $p } $result = $this->deploy_resource($resource, $force, $pr); if (isset($result['status']) && $result['status'] === 429) { - return response()->json(['message' => $result['message']], 429); + return response()->json(['message' => $result['message']], 429)->header('Retry-After', 60); } ['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $result; if ($deployment_uuid) { @@ -436,7 +436,7 @@ public function by_tags(string $tags, int $team_id, bool $force = false) foreach ($applications as $resource) { $result = $this->deploy_resource($resource, $force); if (isset($result['status']) && $result['status'] === 429) { - return response()->json(['message' => $result['message']], 429); + return response()->json(['message' => $result['message']], 429)->header('Retry-After', 60); } ['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $result; if ($deployment_uuid) { diff --git a/app/Http/Controllers/Webhook/Bitbucket.php b/app/Http/Controllers/Webhook/Bitbucket.php index 5410564c8..0efd7a537 100644 --- a/app/Http/Controllers/Webhook/Bitbucket.php +++ b/app/Http/Controllers/Webhook/Bitbucket.php @@ -108,7 +108,7 @@ public function manual(Request $request) is_webhook: true ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, @@ -164,7 +164,7 @@ public function manual(Request $request) git_type: 'bitbucket' ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, diff --git a/app/Http/Controllers/Webhook/Gitea.php b/app/Http/Controllers/Webhook/Gitea.php index 8f9cdba0c..f09b622bb 100644 --- a/app/Http/Controllers/Webhook/Gitea.php +++ b/app/Http/Controllers/Webhook/Gitea.php @@ -124,7 +124,7 @@ public function manual(Request $request) is_webhook: true, ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, @@ -196,7 +196,7 @@ public function manual(Request $request) git_type: 'gitea' ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, diff --git a/app/Http/Controllers/Webhook/Github.php b/app/Http/Controllers/Webhook/Github.php index e0ccf0850..bf8afac22 100644 --- a/app/Http/Controllers/Webhook/Github.php +++ b/app/Http/Controllers/Webhook/Github.php @@ -137,7 +137,7 @@ public function manual(Request $request) is_webhook: true, ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, @@ -225,7 +225,7 @@ public function manual(Request $request) git_type: 'github' ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, @@ -432,7 +432,7 @@ public function normal(Request $request) is_webhook: true, ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } $return_payloads->push([ 'status' => $result['status'], @@ -499,7 +499,7 @@ public function normal(Request $request) git_type: 'github' ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, diff --git a/app/Http/Controllers/Webhook/Gitlab.php b/app/Http/Controllers/Webhook/Gitlab.php index 004ab0e59..67d75bfdd 100644 --- a/app/Http/Controllers/Webhook/Gitlab.php +++ b/app/Http/Controllers/Webhook/Gitlab.php @@ -150,7 +150,7 @@ public function manual(Request $request) is_webhook: true, ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'status' => $result['status'], @@ -223,7 +223,7 @@ public function manual(Request $request) git_type: 'gitlab' ); if ($result['status'] === 'queue_full') { - return response($result['message'], 429); + return response($result['message'], 429)->header('Retry-After', 60); } elseif ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name,