From d2d9c1b2bcaccaaba826bb06816ea5dd7679088e Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 19 Nov 2025 13:52:08 +0100 Subject: [PATCH] debug: add comprehensive status change logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added detailed debug logging to all status update paths to help diagnose why "unhealthy" status appears in the UI. ## Logging Added ### 1. PushServerUpdateJob (Sentinel updates) **Location**: Lines 303-315 **Logs**: Status changes from Sentinel push updates **Data tracked**: - Old vs new status - Container statuses that led to aggregation - Status flags (hasRunning, hasUnhealthy, hasUnknown) ### 2. GetContainersStatus (SSH updates) **Location**: Lines 441-449, 346-354, 358-365 **Logs**: Status changes from SSH-based checks **Scenarios**: - Normal status aggregation - Recently restarted containers (kept as degraded) - Applications not running (set to exited) **Data tracked**: - Old vs new status - Container statuses - Restart count and timing - Whether containers exist ### 3. Application Model Status Accessor **Location**: Lines 706-712, 726-732 **Logs**: When status is set without explicit health information **Issue**: Highlights cases where health defaults to "unhealthy" **Data tracked**: - Raw value passed to setter - Final result after default applied ## How to Use ### Enable Debug Logging Edit `.env` or `config/logging.php` to set log level to debug: ``` LOG_LEVEL=debug ``` ### Monitor Logs ```bash tail -f storage/logs/laravel.log | grep STATUS-DEBUG ``` ### Log Format All logs use `[STATUS-DEBUG]` prefix for easy filtering: ``` [2025-11-19 13:00:00] local.DEBUG: [STATUS-DEBUG] Sentinel status change { "source": "PushServerUpdateJob", "app_id": 123, "app_name": "my-app", "old_status": "running:unknown", "new_status": "running:healthy", "container_statuses": [...], "flags": {...} } ``` ## What to Look For 1. **Default to unhealthy**: Check Application model accessor logs 2. **Status flipping**: Compare timestamps between Sentinel and SSH updates 3. **Incorrect aggregation**: Check flags and container_statuses 4. **Stale database values**: Check if old_status persists across multiple logs ## Next Steps After gathering logs, we can: 1. Identify the exact source of "unhealthy" status 2. Determine if it's a default issue, aggregation bug, or timing problem 3. Apply targeted fix based on evidence 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- app/Actions/Docker/GetContainersStatus.php | 27 ++++++++++++++++++++++ app/Jobs/PushServerUpdateJob.php | 14 +++++++++++ app/Models/Application.php | 15 ++++++++++++ 3 files changed, 56 insertions(+) diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index 72ece0562..5be73f278 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -11,6 +11,7 @@ use Illuminate\Support\Arr; use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Log; use Lorisleiva\Actions\Concerns\AsAction; class GetContainersStatus @@ -342,9 +343,26 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti if ($recentlyRestarted) { // Keep it as degraded if it was recently in a crash loop + Log::debug('[STATUS-DEBUG] Recently restarted - keeping degraded', [ + 'source' => 'GetContainersStatus (not running)', + 'app_id' => $application->id, + 'app_name' => $application->name, + 'old_status' => $application->status, + 'new_status' => 'degraded (unhealthy)', + 'restart_count' => $application->restart_count, + 'last_restart_at' => $application->last_restart_at, + ]); $application->update(['status' => 'degraded (unhealthy)']); } else { // Reset restart count when application exits completely + Log::debug('[STATUS-DEBUG] Application not running', [ + 'source' => 'GetContainersStatus (not running)', + 'app_id' => $application->id, + 'app_name' => $application->name, + 'old_status' => $application->status, + 'new_status' => 'exited', + 'containers_exist' => ! $this->containers->isEmpty(), + ]); $application->update([ 'status' => 'exited', 'restart_count' => 0, @@ -437,6 +455,15 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti if ($aggregatedStatus) { $statusFromDb = $application->status; if ($statusFromDb !== $aggregatedStatus) { + Log::debug('[STATUS-DEBUG] SSH status change', [ + 'source' => 'GetContainersStatus', + 'app_id' => $application->id, + 'app_name' => $application->name, + 'old_status' => $statusFromDb, + 'new_status' => $aggregatedStatus, + 'container_statuses' => $containerStatuses->toArray(), + 'max_restart_count' => $maxRestartCount, + ]); $application->update(['status' => $aggregatedStatus]); } else { $application->update(['last_online_at' => now()]); diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 32baf3f07..8b3f56e14 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -21,6 +21,7 @@ use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Queue\SerializesModels; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Log; use Laravel\Horizon\Contracts\Silenced; class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced @@ -299,6 +300,19 @@ private function aggregateMultiContainerStatuses() // Update application status with aggregated result if ($aggregatedStatus && $application->status !== $aggregatedStatus) { + Log::debug('[STATUS-DEBUG] Sentinel status change', [ + 'source' => 'PushServerUpdateJob', + 'app_id' => $application->id, + 'app_name' => $application->name, + 'old_status' => $application->status, + 'new_status' => $aggregatedStatus, + 'container_statuses' => $relevantStatuses->toArray(), + 'flags' => [ + 'hasRunning' => $hasRunning, + 'hasUnhealthy' => $hasUnhealthy, + 'hasUnknown' => $hasUnknown, + ], + ]); $application->status = $aggregatedStatus; $application->save(); } diff --git a/app/Models/Application.php b/app/Models/Application.php index c2ba6e773..4669c5008 100644 --- a/app/Models/Application.php +++ b/app/Models/Application.php @@ -12,6 +12,7 @@ use Illuminate\Database\Eloquent\Relations\HasMany; use Illuminate\Database\Eloquent\SoftDeletes; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Validator; use Illuminate\Support\Str; use OpenApi\Attributes as OA; @@ -702,6 +703,13 @@ public function status(): Attribute } else { $status = $value; $health = 'unhealthy'; + Log::debug('[STATUS-DEBUG] Status set without health - defaulting to unhealthy', [ + 'source' => 'Application model accessor', + 'app_id' => $this->id, + 'app_name' => $this->name, + 'raw_value' => $value, + 'result' => "$status:$health", + ]); } return "$status:$health"; @@ -715,6 +723,13 @@ public function status(): Attribute } else { $status = $value; $health = 'unhealthy'; + Log::debug('[STATUS-DEBUG] Status set without health (multi-server) - defaulting to unhealthy', [ + 'source' => 'Application model accessor', + 'app_id' => $this->id, + 'app_name' => $this->name, + 'raw_value' => $value, + 'result' => "$status:$health", + ]); } return "$status:$health";