rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Add comprehensive PR cleanup to GitLab, Bitbucket, and Gitea (#7537)

Browse source
This commit is contained in:
Andras Bacsai 2025-12-08 20:12:33 +01:00 committed by GitHub
commit e1d5da3622
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 424 additions and 98 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
app/Actions/Application/CleanupPreviewDeployment.php Normal file
View file

@ -0,0 +1,176 @@
<?php
namespace App\Actions\Application;
use App\Enums\ApplicationDeploymentStatus;
use App\Jobs\DeleteResourceJob;
use App\Models\Application;
use App\Models\ApplicationDeploymentQueue;
use App\Models\ApplicationPreview;
use Lorisleiva\Actions\Concerns\AsAction;
class CleanupPreviewDeployment
{
use AsAction;
public string $jobQueue = 'high';
/**
* Clean up a PR preview deployment completely.
*
* This handles:
* 1. Cancelling active deployments for the PR (QUEUED/IN_PROGRESS → CANCELLED_BY_USER)
* 2. Killing helper containers by deployment_uuid
* 3. Stopping/removing all running PR containers
* 4. Dispatching DeleteResourceJob for thorough cleanup (volumes, networks, database records)
*
* This unifies the cleanup logic from GitHub webhook handler to be used across all providers.
*/
public function handle(
Application $application,
int $pull_request_id,
?ApplicationPreview $preview = null
): array {
$result = [
'cancelled_deployments' => 0,
'killed_containers' => 0,
'status' => 'success',
];
$server = $application->destination->server;
if (! $server->isFunctional()) {
return [
...$result,
'status' => 'failed',
'message' => 'Server is not functional',
];
}
// Step 1: Cancel all active deployments for this PR and kill helper containers
$result['cancelled_deployments'] = $this->cancelActiveDeployments(
$application,
$pull_request_id,
$server
);
// Step 2: Stop and remove all running PR containers
$result['killed_containers'] = $this->stopRunningContainers(
$application,
$pull_request_id,
$server
);
// Step 3: Find or use provided preview, then dispatch cleanup job for thorough cleanup
if (! $preview) {
$preview = ApplicationPreview::where('application_id', $application->id)
->where('pull_request_id', $pull_request_id)
->first();
}
if ($preview) {
DeleteResourceJob::dispatch($preview);
}
return $result;
}
/**
* Cancel all active (QUEUED/IN_PROGRESS) deployments for this PR.
*/
private function cancelActiveDeployments(
Application $application,
int $pull_request_id,
$server
): int {
$activeDeployments = ApplicationDeploymentQueue::where('application_id', $application->id)
->where('pull_request_id', $pull_request_id)
->whereIn('status', [
ApplicationDeploymentStatus::QUEUED->value,
ApplicationDeploymentStatus::IN_PROGRESS->value,
])
->get();
$cancelled = 0;
foreach ($activeDeployments as $deployment) {
try {
// Mark deployment as cancelled
$deployment->update([
'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
// Add cancellation log entry
$deployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
// Try to kill helper container if it exists
$this->killHelperContainer($deployment->deployment_uuid, $server);
$cancelled++;
} catch (\Throwable $e) {
\Log::warning("Failed to cancel deployment {$deployment->id}: {$e->getMessage()}");
}
}
return $cancelled;
}
/**
* Kill the helper container used during deployment.
*/
private function killHelperContainer(string $deployment_uuid, $server): void
{
try {
$escapedUuid = escapeshellarg($deployment_uuid);
$checkCommand = "docker ps -a --filter name={$escapedUuid} --format '{{.Names}}'";
$containerExists = instant_remote_process([$checkCommand], $server);
if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
instant_remote_process(["docker rm -f {$escapedUuid}"], $server);
}
} catch (\Throwable $e) {
// Silently handle - container may already be gone
}
}
/**
* Stop and remove all running containers for this PR.
*/
private function stopRunningContainers(
Application $application,
int $pull_request_id,
$server
): int {
$killed = 0;
try {
if ($server->isSwarm()) {
$escapedStackName = escapeshellarg("{$application->uuid}-{$pull_request_id}");
instant_remote_process(["docker stack rm {$escapedStackName}"], $server);
$killed++;
} else {
$containers = getCurrentApplicationContainerStatus(
$server,
$application->id,
$pull_request_id
);
if ($containers->isNotEmpty()) {
foreach ($containers as $container) {
$containerName = data_get($container, 'Names');
if ($containerName) {
$escapedContainerName = escapeshellarg($containerName);
instant_remote_process(
["docker rm -f {$escapedContainerName}"],
$server
);
$killed++;
}
}
}
}
} catch (\Throwable $e) {
\Log::warning("Error stopping containers for PR #{$pull_request_id}: {$e->getMessage()}");
}
return $killed;
}
}

5
app/Console/Kernel.php
View file

@ -6,6 +6,7 @@
use App\Jobs\CheckHelperImageJob;
use App\Jobs\CheckTraefikVersionJob;
use App\Jobs\CleanupInstanceStuffsJob;
use App\Jobs\CleanupOrphanedPreviewContainersJob;
use App\Jobs\PullChangelog;
use App\Jobs\PullTemplatesFromCDN;
use App\Jobs\RegenerateSslCertJob;
@ -17,7 +18,6 @@
use App\Models\Team;
use Illuminate\Console\Scheduling\Schedule;
use Illuminate\Foundation\Console\Kernel as ConsoleKernel;
use Illuminate\Support\Facades\Log;
class Kernel extends ConsoleKernel
{
@ -87,6 +87,9 @@ protected function schedule(Schedule $schedule): void
$this->scheduleInstance->command('cleanup:database --yes')->daily();
$this->scheduleInstance->command('uploads:clear')->everyTwoMinutes();
// Cleanup orphaned PR preview containers daily
$this->scheduleInstance->job(new CleanupOrphanedPreviewContainersJob)->daily()->onOneServer();
}
}

8
app/Http/Controllers/Webhook/Bitbucket.php
View file

@ -2,6 +2,7 @@
namespace App\Http\Controllers\Webhook;
use App\Actions\Application\CleanupPreviewDeployment;
use App\Http\Controllers\Controller;
use App\Models\Application;
use App\Models\ApplicationPreview;
@ -167,9 +168,10 @@ public function manual(Request $request)
if ($x_bitbucket_event === 'pullrequest:rejected' || $x_bitbucket_event === 'pullrequest:fulfilled') {
$found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first();
if ($found) {
$found->delete();
$container_name = generateApplicationContainerName($application, $pull_request_id);
instant_remote_process(["docker rm -f $container_name"], $application->destination->server);
// Use comprehensive cleanup that cancels active deployments,
// kills helper containers, and removes all PR containers
CleanupPreviewDeployment::run($application, $pull_request_id, $found);
$return_payloads->push([
'application' => $application->name,
'status' => 'success',

8
app/Http/Controllers/Webhook/Gitea.php
View file

@ -2,6 +2,7 @@
namespace App\Http\Controllers\Webhook;
use App\Actions\Application\CleanupPreviewDeployment;
use App\Http\Controllers\Controller;
use App\Models\Application;
use App\Models\ApplicationPreview;
@ -192,9 +193,10 @@ public function manual(Request $request)
if ($action === 'closed') {
$found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first();
if ($found) {
$found->delete();
$container_name = generateApplicationContainerName($application, $pull_request_id);
instant_remote_process(["docker rm -f $container_name"], $application->destination->server);
// Use comprehensive cleanup that cancels active deployments,
// kills helper containers, and removes all PR containers
CleanupPreviewDeployment::run($application, $pull_request_id, $found);
$return_payloads->push([
'application' => $application->name,
'status' => 'success',

88
app/Http/Controllers/Webhook/Github.php
View file

@ -2,10 +2,10 @@
namespace App\Http\Controllers\Webhook;
use App\Actions\Application\CleanupPreviewDeployment;
use App\Enums\ProcessStatus;
use App\Http\Controllers\Controller;
use App\Jobs\ApplicationPullRequestUpdateJob;
use App\Jobs\DeleteResourceJob;
use App\Jobs\GithubAppPermissionJob;
use App\Models\Application;
use App\Models\ApplicationPreview;
@ -221,41 +221,10 @@ public function manual(Request $request)
if ($action === 'closed') {
$found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first();
if ($found) {
// Cancel any active deployments for this PR immediately
$activeDeployment = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id)
->where('pull_request_id', $pull_request_id)
->whereIn('status', [
\App\Enums\ApplicationDeploymentStatus::QUEUED->value,
\App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value,
])
->first();
// Use comprehensive cleanup that cancels active deployments,
// kills helper containers, and removes all PR containers
CleanupPreviewDeployment::run($application, $pull_request_id, $found);
if ($activeDeployment) {
try {
// Mark deployment as cancelled
$activeDeployment->update([
'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
// Add cancellation log entry
$activeDeployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
// Check if helper container exists and kill it
$deployment_uuid = $activeDeployment->deployment_uuid;
$server = $application->destination->server;
$checkCommand = "docker ps -a --filter name={$deployment_uuid} --format '{{.Names}}'";
$containerExists = instant_remote_process([$checkCommand], $server);
if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
instant_remote_process(["docker rm -f {$deployment_uuid}"], $server);
$activeDeployment->addLogEntry('Deployment container stopped.');
}
} catch (\Throwable $e) {
// Silently handle errors during deployment cancellation
}
}
DeleteResourceJob::dispatch($found);
$return_payloads->push([
'application' => $application->name,
'status' => 'success',
@ -466,53 +435,12 @@ public function normal(Request $request)
if ($action === 'closed' || $action === 'close') {
$found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first();
if ($found) {
// Cancel any active deployments for this PR immediately
$activeDeployment = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id)
->where('pull_request_id', $pull_request_id)
->whereIn('status', [
\App\Enums\ApplicationDeploymentStatus::QUEUED->value,
\App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value,
])
->first();
if ($activeDeployment) {
try {
// Mark deployment as cancelled
$activeDeployment->update([
'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value,
]);
// Add cancellation log entry
$activeDeployment->addLogEntry('Deployment cancelled: Pull request closed.', 'stderr');
// Check if helper container exists and kill it
$deployment_uuid = $activeDeployment->deployment_uuid;
$server = $application->destination->server;
$checkCommand = "docker ps -a --filter name={$deployment_uuid} --format '{{.Names}}'";
$containerExists = instant_remote_process([$checkCommand], $server);
if ($containerExists && str($containerExists)->trim()->isNotEmpty()) {
instant_remote_process(["docker rm -f {$deployment_uuid}"], $server);
$activeDeployment->addLogEntry('Deployment container stopped.');
}
} catch (\Throwable $e) {
// Silently handle errors during deployment cancellation
}
}
// Clean up any deployed containers
$containers = getCurrentApplicationContainerStatus($application->destination->server, $application->id, $pull_request_id);
if ($containers->isNotEmpty()) {
$containers->each(function ($container) use ($application) {
$container_name = data_get($container, 'Names');
instant_remote_process(["docker rm -f $container_name"], $application->destination->server);
});
}
// Delete the PR comment on GitHub (GitHub-specific feature)
ApplicationPullRequestUpdateJob::dispatchSync(application: $application, preview: $found, status: ProcessStatus::CLOSED);
DeleteResourceJob::dispatch($found);
// Use comprehensive cleanup that cancels active deployments,
// kills helper containers, and removes all PR containers
CleanupPreviewDeployment::run($application, $pull_request_id, $found);
$return_payloads->push([
'application' => $application->name,

23
app/Http/Controllers/Webhook/Gitlab.php
View file

@ -2,6 +2,7 @@
namespace App\Http\Controllers\Webhook;
use App\Actions\Application\CleanupPreviewDeployment;
use App\Http\Controllers\Controller;
use App\Models\Application;
use App\Models\ApplicationPreview;
@ -224,22 +225,22 @@ public function manual(Request $request)
} elseif ($action === 'closed' || $action === 'close' || $action === 'merge') {
$found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first();
if ($found) {
$found->delete();
$container_name = generateApplicationContainerName($application, $pull_request_id);
instant_remote_process(["docker rm -f $container_name"], $application->destination->server);
// Use comprehensive cleanup that cancels active deployments,
// kills helper containers, and removes all PR containers
CleanupPreviewDeployment::run($application, $pull_request_id, $found);
$return_payloads->push([
'application' => $application->name,
'status' => 'success',
'message' => 'Preview Deployment closed',
'message' => 'Preview deployment closed.',
]);
} else {
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'No preview deployment found.',
]);
return response($return_payloads);
}
$return_payloads->push([
'application' => $application->name,
'status' => 'failed',
'message' => 'No Preview Deployment found',
]);
} else {
$return_payloads->push([
'application' => $application->name,

214
app/Jobs/CleanupOrphanedPreviewContainersJob.php Normal file
View file

@ -0,0 +1,214 @@
<?php
namespace App\Jobs;
use App\Models\ApplicationPreview;
use App\Models\Server;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
/**
* Scheduled job to clean up orphaned PR preview containers.
*
* This job acts as a safety net for containers that weren't properly cleaned up
* when a PR was closed (e.g., due to webhook failures, race conditions, etc.).
*
* It scans all functional servers for containers with the `coolify.pullRequestId` label
* and removes any where the corresponding ApplicationPreview record no longer exists.
*/
class CleanupOrphanedPreviewContainersJob implements ShouldBeEncrypted, ShouldBeUnique, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $timeout = 600; // 10 minutes max
public function __construct() {}
public function middleware(): array
{
return [(new WithoutOverlapping('cleanup-orphaned-preview-containers'))->expireAfter(600)->dontRelease()];
}
public function handle(): void
{
try {
$servers = $this->getServersToCheck();
foreach ($servers as $server) {
$this->cleanupOrphanedContainersOnServer($server);
}
} catch (\Throwable $e) {
Log::error('CleanupOrphanedPreviewContainersJob failed: '.$e->getMessage());
send_internal_notification('CleanupOrphanedPreviewContainersJob failed with error: '.$e->getMessage());
}
}
/**
* Get all functional servers to check for orphaned containers.
*/
private function getServersToCheck(): \Illuminate\Support\Collection
{
$query = Server::whereRelation('settings', 'is_usable', true)
->whereRelation('settings', 'is_reachable', true)
->where('ip', '!=', '1.2.3.4');
if (isCloud()) {
$query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true);
}
return $query->get()->filter(fn ($server) => $server->isFunctional());
}
/**
* Find and clean up orphaned PR containers on a specific server.
*/
private function cleanupOrphanedContainersOnServer(Server $server): void
{
try {
$prContainers = $this->getPRContainersOnServer($server);
if ($prContainers->isEmpty()) {
return;
}
$orphanedCount = 0;
foreach ($prContainers as $container) {
if ($this->isOrphanedContainer($container)) {
$this->removeContainer($container, $server);
$orphanedCount++;
}
}
if ($orphanedCount > 0) {
Log::info("CleanupOrphanedPreviewContainersJob - Removed {$orphanedCount} orphaned PR containers", [
'server' => $server->name,
]);
}
} catch (\Throwable $e) {
Log::warning("CleanupOrphanedPreviewContainersJob - Error on server {$server->name}: {$e->getMessage()}");
}
}
/**
* Get all PR containers on a server (containers with pullRequestId > 0).
*/
private function getPRContainersOnServer(Server $server): \Illuminate\Support\Collection
{
try {
$output = instant_remote_process([
"docker ps -a --filter 'label=coolify.pullRequestId' --format '{{json .}}'",
], $server, false);
if (empty($output)) {
return collect();
}
return format_docker_command_output_to_json($output)
->filter(function ($container) {
// Only include PR containers (pullRequestId > 0)
$prId = $this->extractPullRequestId($container);
return $prId !== null && $prId > 0;
});
} catch (\Throwable $e) {
Log::debug("Failed to get PR containers on server {$server->name}: {$e->getMessage()}");
return collect();
}
}
/**
* Extract pull request ID from container labels.
*/
private function extractPullRequestId($container): ?int
{
$labels = data_get($container, 'Labels', '');
if (preg_match('/coolify\.pullRequestId=(\d+)/', $labels, $matches)) {
return (int) $matches[1];
}
return null;
}
/**
* Extract application ID from container labels.
*/
private function extractApplicationId($container): ?int
{
$labels = data_get($container, 'Labels', '');
if (preg_match('/coolify\.applicationId=(\d+)/', $labels, $matches)) {
return (int) $matches[1];
}
return null;
}
/**
* Check if a container is orphaned (no corresponding ApplicationPreview record).
*/
private function isOrphanedContainer($container): bool
{
$applicationId = $this->extractApplicationId($container);
$pullRequestId = $this->extractPullRequestId($container);
if ($applicationId === null || $pullRequestId === null) {
return false;
}
// Check if ApplicationPreview record exists (including soft-deleted)
$previewExists = ApplicationPreview::withTrashed()
->where('application_id', $applicationId)
->where('pull_request_id', $pullRequestId)
->exists();
// If preview exists (even soft-deleted), container should be handled by DeleteResourceJob
// If preview doesn't exist at all, it's truly orphaned
return ! $previewExists;
}
/**
* Remove an orphaned container from the server.
*/
private function removeContainer($container, Server $server): void
{
$containerName = data_get($container, 'Names');
if (empty($containerName)) {
Log::warning('CleanupOrphanedPreviewContainersJob - Cannot remove container: missing container name', [
'container_data' => $container,
'server' => $server->name,
]);
return;
}
$applicationId = $this->extractApplicationId($container);
$pullRequestId = $this->extractPullRequestId($container);
Log::info('CleanupOrphanedPreviewContainersJob - Removing orphaned container', [
'container' => $containerName,
'application_id' => $applicationId,
'pull_request_id' => $pullRequestId,
'server' => $server->name,
]);
$escapedContainerName = escapeshellarg($containerName);
try {
instant_remote_process(
["docker rm -f {$escapedContainerName}"],
$server,
false
);
} catch (\Throwable $e) {
Log::warning("Failed to remove orphaned container {$containerName}: {$e->getMessage()}");
}
}
}