diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 9fb5e8a19..8687104e0 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -6,6 +6,7 @@ use App\Jobs\CheckHelperImageJob; use App\Jobs\CheckTraefikVersionJob; use App\Jobs\CleanupInstanceStuffsJob; +use App\Jobs\CleanupOrphanedPreviewContainersJob; use App\Jobs\PullChangelog; use App\Jobs\PullTemplatesFromCDN; use App\Jobs\RegenerateSslCertJob; @@ -17,7 +18,6 @@ use App\Models\Team; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; -use Illuminate\Support\Facades\Log; class Kernel extends ConsoleKernel { @@ -87,6 +87,9 @@ protected function schedule(Schedule $schedule): void $this->scheduleInstance->command('cleanup:database --yes')->daily(); $this->scheduleInstance->command('uploads:clear')->everyTwoMinutes(); + + // Cleanup orphaned PR preview containers daily + $this->scheduleInstance->job(new CleanupOrphanedPreviewContainersJob)->daily()->onOneServer(); } } diff --git a/app/Jobs/CleanupOrphanedPreviewContainersJob.php b/app/Jobs/CleanupOrphanedPreviewContainersJob.php new file mode 100644 index 000000000..790ad1489 --- /dev/null +++ b/app/Jobs/CleanupOrphanedPreviewContainersJob.php @@ -0,0 +1,202 @@ +expireAfter(600)->dontRelease()]; + } + + public function handle(): void + { + try { + $servers = $this->getServersToCheck(); + + foreach ($servers as $server) { + $this->cleanupOrphanedContainersOnServer($server); + } + } catch (\Throwable $e) { + Log::error('CleanupOrphanedPreviewContainersJob failed: '.$e->getMessage()); + send_internal_notification('CleanupOrphanedPreviewContainersJob failed with error: '.$e->getMessage()); + } + } + + /** + * Get all functional servers to check for orphaned containers. + */ + private function getServersToCheck(): \Illuminate\Support\Collection + { + $query = Server::whereRelation('settings', 'is_usable', true) + ->whereRelation('settings', 'is_reachable', true) + ->where('ip', '!=', '1.2.3.4'); + + if (isCloud()) { + $query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true); + } + + return $query->get()->filter(fn ($server) => $server->isFunctional()); + } + + /** + * Find and clean up orphaned PR containers on a specific server. + */ + private function cleanupOrphanedContainersOnServer(Server $server): void + { + try { + $prContainers = $this->getPRContainersOnServer($server); + + if ($prContainers->isEmpty()) { + return; + } + + $orphanedCount = 0; + foreach ($prContainers as $container) { + if ($this->isOrphanedContainer($container)) { + $this->removeContainer($container, $server); + $orphanedCount++; + } + } + + if ($orphanedCount > 0) { + Log::info("CleanupOrphanedPreviewContainersJob - Removed {$orphanedCount} orphaned PR containers", [ + 'server' => $server->name, + ]); + } + } catch (\Throwable $e) { + Log::warning("CleanupOrphanedPreviewContainersJob - Error on server {$server->name}: {$e->getMessage()}"); + } + } + + /** + * Get all PR containers on a server (containers with pullRequestId > 0). + */ + private function getPRContainersOnServer(Server $server): \Illuminate\Support\Collection + { + try { + $output = instant_remote_process([ + "docker ps -a --filter 'label=coolify.pullRequestId' --format '{{json .}}'", + ], $server, false); + + if (empty($output)) { + return collect(); + } + + return format_docker_command_output_to_json($output) + ->filter(function ($container) { + // Only include PR containers (pullRequestId > 0) + $prId = $this->extractPullRequestId($container); + + return $prId !== null && $prId > 0; + }); + } catch (\Throwable $e) { + Log::debug("Failed to get PR containers on server {$server->name}: {$e->getMessage()}"); + + return collect(); + } + } + + /** + * Extract pull request ID from container labels. + */ + private function extractPullRequestId($container): ?int + { + $labels = data_get($container, 'Labels', ''); + if (preg_match('/coolify\.pullRequestId=(\d+)/', $labels, $matches)) { + return (int) $matches[1]; + } + + return null; + } + + /** + * Extract application ID from container labels. + */ + private function extractApplicationId($container): ?int + { + $labels = data_get($container, 'Labels', ''); + if (preg_match('/coolify\.applicationId=(\d+)/', $labels, $matches)) { + return (int) $matches[1]; + } + + return null; + } + + /** + * Check if a container is orphaned (no corresponding ApplicationPreview record). + */ + private function isOrphanedContainer($container): bool + { + $applicationId = $this->extractApplicationId($container); + $pullRequestId = $this->extractPullRequestId($container); + + if ($applicationId === null || $pullRequestId === null) { + return false; + } + + // Check if ApplicationPreview record exists (including soft-deleted) + $previewExists = ApplicationPreview::withTrashed() + ->where('application_id', $applicationId) + ->where('pull_request_id', $pullRequestId) + ->exists(); + + // If preview exists (even soft-deleted), container should be handled by DeleteResourceJob + // If preview doesn't exist at all, it's truly orphaned + return ! $previewExists; + } + + /** + * Remove an orphaned container from the server. + */ + private function removeContainer($container, Server $server): void + { + $containerName = data_get($container, 'Names'); + $applicationId = $this->extractApplicationId($container); + $pullRequestId = $this->extractPullRequestId($container); + + Log::info('CleanupOrphanedPreviewContainersJob - Removing orphaned container', [ + 'container' => $containerName, + 'application_id' => $applicationId, + 'pull_request_id' => $pullRequestId, + 'server' => $server->name, + ]); + + try { + instant_remote_process( + ["docker rm -f {$containerName}"], + $server, + false + ); + } catch (\Throwable $e) { + Log::warning("Failed to remove orphaned container {$containerName}: {$e->getMessage()}"); + } + } +}