debug: add detailed Sentinel container processing logging
Added comprehensive logging to track why applicationContainerStatuses collection is empty in PushServerUpdateJob. ## Logging Added ### 1. Raw Sentinel Data (line 113-118) **Logs**: Complete container data received from Sentinel **Purpose**: See exactly what Sentinel is sending **Data**: Container count and full container array with all labels ### 2. Container Processing Loop (line 157-163) **Logs**: Every container as it's being processed **Purpose**: Track which containers enter the processing loop **Data**: Container name, status, all labels, coolify.managed flag ### 3. Skipped Containers - Not Managed (line 165-171) **Logs**: Containers without coolify.managed label **Purpose**: Identify containers being filtered out early **Data**: Container name ### 4. Successful Container Addition (line 193-198) **Logs**: When container is successfully added to applicationContainerStatuses **Purpose**: Confirm containers ARE being processed **Data**: Application ID, container name, container status ### 5. Missing com.docker.compose.service Label (line 200-206) **Logs**: Containers skipped due to missing com.docker.compose.service **Purpose**: Identify the most likely root cause **Data**: Container name, application ID, all labels ## Why This Matters User reported applicationContainerStatuses is empty (`[]`) even though Sentinel is pushing updates. This logging will reveal: 1. Is Sentinel sending containers at all? 2. Are containers filtered by coolify.managed check? 3. Is com.docker.compose.service label missing? (most likely) 4. What labels IS Sentinel actually sending? ## Expected Findings Based on investigation, the issue is likely: - Sentinel is NOT sending com.docker.compose.service in labels - Or Sentinel uses a different label format/name - Containers pass all other checks but fail on line 190-206 ## Next Steps After logs appear, we'll see exactly which filter is blocking containers and can fix the root cause (likely need to extract com.docker.compose.service from container name or use a different label source). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andras Bacsai
parent
d2d9c1b2bc
commit
747a48b933
2 changed files with 64 additions and 14 deletions
|
|
@ -110,6 +110,13 @@ public function handle()
|
|||
|
||||
$this->containers = collect(data_get($data, 'containers'));
|
||||
|
||||
Log::debug('[STATUS-DEBUG] Raw Sentinel data received', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'container_count' => $this->containers->count(),
|
||||
'containers' => $this->containers->toArray(),
|
||||
]);
|
||||
ray('Raw Sentinel containers:', $this->containers->toArray());
|
||||
|
||||
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
|
||||
ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
|
||||
|
||||
|
|
@ -146,6 +153,24 @@ public function handle()
|
|||
$containerStatus = "$containerStatus ($containerHealth)";
|
||||
$labels = collect(data_get($container, 'labels'));
|
||||
$coolify_managed = $labels->has('coolify.managed');
|
||||
|
||||
Log::debug('[STATUS-DEBUG] Processing container from Sentinel', [
|
||||
'source' => 'PushServerUpdateJob (loop)',
|
||||
'container_name' => data_get($container, 'name'),
|
||||
'container_status' => $containerStatus,
|
||||
'labels' => $labels->toArray(),
|
||||
'has_coolify_managed' => $coolify_managed,
|
||||
]);
|
||||
|
||||
if (! $coolify_managed) {
|
||||
Log::debug('[STATUS-DEBUG] Container skipped - not coolify managed', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'container_name' => data_get($container, 'name'),
|
||||
]);
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if ($coolify_managed) {
|
||||
$name = data_get($container, 'name');
|
||||
if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) {
|
||||
|
|
@ -166,6 +191,19 @@ public function handle()
|
|||
$containerName = $labels->get('com.docker.compose.service');
|
||||
if ($containerName) {
|
||||
$this->applicationContainerStatuses->get($applicationId)->put($containerName, $containerStatus);
|
||||
Log::debug('[STATUS-DEBUG] Container added to applicationContainerStatuses', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'application_id' => $applicationId,
|
||||
'container_name' => $containerName,
|
||||
'container_status' => $containerStatus,
|
||||
]);
|
||||
} else {
|
||||
Log::debug('[STATUS-DEBUG] Container skipped - no com.docker.compose.service label', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'container_name' => data_get($container, 'name'),
|
||||
'application_id' => $applicationId,
|
||||
'labels' => $labels->toArray(),
|
||||
]);
|
||||
}
|
||||
} else {
|
||||
$previewKey = $applicationId.':'.$pullRequestId;
|
||||
|
|
@ -224,12 +262,22 @@ public function handle()
|
|||
|
||||
private function aggregateMultiContainerStatuses()
|
||||
{
|
||||
Log::debug('[STATUS-DEBUG] Starting aggregation of multi-container application statuses', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
]);
|
||||
ray('Starting aggregation of multi-container application statuses');
|
||||
ray($this->applicationContainerStatuses->toArray());
|
||||
if ($this->applicationContainerStatuses->isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
foreach ($this->applicationContainerStatuses as $applicationId => $containerStatuses) {
|
||||
$application = $this->applications->where('id', $applicationId)->first();
|
||||
Log::debug('[STATUS-DEBUG] Processing application for aggregation', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'app_id' => $applicationId,
|
||||
'container_statuses' => $containerStatuses->toArray(),
|
||||
]);
|
||||
if (! $application) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -297,22 +345,22 @@ private function aggregateMultiContainerStatuses()
|
|||
// All containers are exited
|
||||
$aggregatedStatus = 'exited (unhealthy)';
|
||||
}
|
||||
|
||||
Log::debug('[STATUS-DEBUG] Sentinel status change', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'app_id' => $application->id,
|
||||
'app_name' => $application->name,
|
||||
'old_status' => $application->status,
|
||||
'new_status' => $aggregatedStatus,
|
||||
'container_statuses' => $relevantStatuses->toArray(),
|
||||
'flags' => [
|
||||
'hasRunning' => $hasRunning,
|
||||
'hasUnhealthy' => $hasUnhealthy,
|
||||
'hasUnknown' => $hasUnknown,
|
||||
],
|
||||
]);
|
||||
// Update application status with aggregated result
|
||||
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
|
||||
Log::debug('[STATUS-DEBUG] Sentinel status change', [
|
||||
'source' => 'PushServerUpdateJob',
|
||||
'app_id' => $application->id,
|
||||
'app_name' => $application->name,
|
||||
'old_status' => $application->status,
|
||||
'new_status' => $aggregatedStatus,
|
||||
'container_statuses' => $relevantStatuses->toArray(),
|
||||
'flags' => [
|
||||
'hasRunning' => $hasRunning,
|
||||
'hasUnhealthy' => $hasUnhealthy,
|
||||
'hasUnknown' => $hasUnknown,
|
||||
],
|
||||
]);
|
||||
|
||||
$application->status = $aggregatedStatus;
|
||||
$application->save();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
use App\Http\Middleware\ApiAllowed;
|
||||
use App\Jobs\PushServerUpdateJob;
|
||||
use App\Models\Server;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
use Illuminate\Support\Facades\Route;
|
||||
|
||||
Route::get('/health', [OtherController::class, 'healthcheck']);
|
||||
|
|
@ -158,6 +159,7 @@
|
|||
'prefix' => 'v1',
|
||||
], function () {
|
||||
Route::post('/sentinel/push', function () {
|
||||
Log::info('Received Sentinel push request', ['ip' => request()->ip(), 'user_agent' => request()->header('User-Agent')]);
|
||||
$token = request()->header('Authorization');
|
||||
if (! $token) {
|
||||
return response()->json(['message' => 'Unauthorized'], 401);
|
||||
|
|
|
|||
Loading…
Reference in a new issue