From 747a48b9337947a85e5b4a4b123ea405d676b5b0 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 20 Nov 2025 08:34:42 +0100 Subject: [PATCH] debug: add detailed Sentinel container processing logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added comprehensive logging to track why applicationContainerStatuses collection is empty in PushServerUpdateJob. ## Logging Added ### 1. Raw Sentinel Data (line 113-118) **Logs**: Complete container data received from Sentinel **Purpose**: See exactly what Sentinel is sending **Data**: Container count and full container array with all labels ### 2. Container Processing Loop (line 157-163) **Logs**: Every container as it's being processed **Purpose**: Track which containers enter the processing loop **Data**: Container name, status, all labels, coolify.managed flag ### 3. Skipped Containers - Not Managed (line 165-171) **Logs**: Containers without coolify.managed label **Purpose**: Identify containers being filtered out early **Data**: Container name ### 4. Successful Container Addition (line 193-198) **Logs**: When container is successfully added to applicationContainerStatuses **Purpose**: Confirm containers ARE being processed **Data**: Application ID, container name, container status ### 5. Missing com.docker.compose.service Label (line 200-206) **Logs**: Containers skipped due to missing com.docker.compose.service **Purpose**: Identify the most likely root cause **Data**: Container name, application ID, all labels ## Why This Matters User reported applicationContainerStatuses is empty (`[]`) even though Sentinel is pushing updates. This logging will reveal: 1. Is Sentinel sending containers at all? 2. Are containers filtered by coolify.managed check? 3. Is com.docker.compose.service label missing? (most likely) 4. What labels IS Sentinel actually sending? ## Expected Findings Based on investigation, the issue is likely: - Sentinel is NOT sending com.docker.compose.service in labels - Or Sentinel uses a different label format/name - Containers pass all other checks but fail on line 190-206 ## Next Steps After logs appear, we'll see exactly which filter is blocking containers and can fix the root cause (likely need to extract com.docker.compose.service from container name or use a different label source). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/Jobs/PushServerUpdateJob.php | 76 ++++++++++++++++++++++++++------ routes/api.php | 2 + 2 files changed, 64 insertions(+), 14 deletions(-) diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 8b3f56e14..4056bc8fd 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -110,6 +110,13 @@ public function handle() $this->containers = collect(data_get($data, 'containers')); + Log::debug('[STATUS-DEBUG] Raw Sentinel data received', [ + 'source' => 'PushServerUpdateJob', + 'container_count' => $this->containers->count(), + 'containers' => $this->containers->toArray(), + ]); + ray('Raw Sentinel containers:', $this->containers->toArray()); + $filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage'); ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot); @@ -146,6 +153,24 @@ public function handle() $containerStatus = "$containerStatus ($containerHealth)"; $labels = collect(data_get($container, 'labels')); $coolify_managed = $labels->has('coolify.managed'); + + Log::debug('[STATUS-DEBUG] Processing container from Sentinel', [ + 'source' => 'PushServerUpdateJob (loop)', + 'container_name' => data_get($container, 'name'), + 'container_status' => $containerStatus, + 'labels' => $labels->toArray(), + 'has_coolify_managed' => $coolify_managed, + ]); + + if (! $coolify_managed) { + Log::debug('[STATUS-DEBUG] Container skipped - not coolify managed', [ + 'source' => 'PushServerUpdateJob', + 'container_name' => data_get($container, 'name'), + ]); + + continue; + } + if ($coolify_managed) { $name = data_get($container, 'name'); if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) { @@ -166,6 +191,19 @@ public function handle() $containerName = $labels->get('com.docker.compose.service'); if ($containerName) { $this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus); + Log::debug('[STATUS-DEBUG] Container added to applicationContainerStatuses', [ + 'source' => 'PushServerUpdateJob', + 'application_id' => $applicationId, + 'container_name' => $containerName, + 'container_status' => $containerStatus, + ]); + } else { + Log::debug('[STATUS-DEBUG] Container skipped - no com.docker.compose.service label', [ + 'source' => 'PushServerUpdateJob', + 'container_name' => data_get($container, 'name'), + 'application_id' => $applicationId, + 'labels' => $labels->toArray(), + ]); } } else { $previewKey = $applicationId.':'.$pullRequestId; @@ -224,12 +262,22 @@ public function handle() private function aggregateMultiContainerStatuses() { + Log::debug('[STATUS-DEBUG] Starting aggregation of multi-container application statuses', [ + 'source' => 'PushServerUpdateJob', + ]); + ray('Starting aggregation of multi-container application statuses'); + ray($this->applicationContainerStatuses->toArray()); if ($this->applicationContainerStatuses->isEmpty()) { return; } foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) { $application = $this->applications->where('id', $applicationId)->first(); + Log::debug('[STATUS-DEBUG] Processing application for aggregation', [ + 'source' => 'PushServerUpdateJob', + 'app_id' => $applicationId, + 'container_statuses' => $containerStatuses->toArray(), + ]); if (! $application) { continue; } @@ -297,22 +345,22 @@ private function aggregateMultiContainerStatuses() // All containers are exited $aggregatedStatus = 'exited (unhealthy)'; } - + Log::debug('[STATUS-DEBUG] Sentinel status change', [ + 'source' => 'PushServerUpdateJob', + 'app_id' => $application->id, + 'app_name' => $application->name, + 'old_status' => $application->status, + 'new_status' => $aggregatedStatus, + 'container_statuses' => $relevantStatuses->toArray(), + 'flags' => [ + 'hasRunning' => $hasRunning, + 'hasUnhealthy' => $hasUnhealthy, + 'hasUnknown' => $hasUnknown, + ], + ]); // Update application status with aggregated result if ($aggregatedStatus && $application->status !== $aggregatedStatus) { - Log::debug('[STATUS-DEBUG] Sentinel status change', [ - 'source' => 'PushServerUpdateJob', - 'app_id' => $application->id, - 'app_name' => $application->name, - 'old_status' => $application->status, - 'new_status' => $aggregatedStatus, - 'container_statuses' => $relevantStatuses->toArray(), - 'flags' => [ - 'hasRunning' => $hasRunning, - 'hasUnhealthy' => $hasUnhealthy, - 'hasUnknown' => $hasUnknown, - ], - ]); + $application->status = $aggregatedStatus; $application->save(); } diff --git a/routes/api.php b/routes/api.php index 366a97d74..cd6ffc8c8 100644 --- a/routes/api.php +++ b/routes/api.php @@ -14,6 +14,7 @@ use App\Http\Middleware\ApiAllowed; use App\Jobs\PushServerUpdateJob; use App\Models\Server; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Route; Route::get('/health', [OtherController::class, 'healthcheck']); @@ -158,6 +159,7 @@ 'prefix' => 'v1', ], function () { Route::post('/sentinel/push', function () { + Log::info('Received Sentinel push request', ['ip' => request()->ip(), 'user_agent' => request()->header('User-Agent')]); $token = request()->header('Authorization'); if (! $token) { return response()->json(['message' => 'Unauthorized'], 401);