debug: add comprehensive status change logging
Added detailed debug logging to all status update paths to help
diagnose why "unhealthy" status appears in the UI.
## Logging Added
### 1. PushServerUpdateJob (Sentinel updates)
**Location**: Lines 303-315
**Logs**: Status changes from Sentinel push updates
**Data tracked**:
- Old vs new status
- Container statuses that led to aggregation
- Status flags (hasRunning, hasUnhealthy, hasUnknown)
### 2. GetContainersStatus (SSH updates)
**Location**: Lines 441-449, 346-354, 358-365
**Logs**: Status changes from SSH-based checks
**Scenarios**:
- Normal status aggregation
- Recently restarted containers (kept as degraded)
- Applications not running (set to exited)
**Data tracked**:
- Old vs new status
- Container statuses
- Restart count and timing
- Whether containers exist
### 3. Application Model Status Accessor
**Location**: Lines 706-712, 726-732
**Logs**: When status is set without explicit health information
**Issue**: Highlights cases where health defaults to "unhealthy"
**Data tracked**:
- Raw value passed to setter
- Final result after default applied
## How to Use
### Enable Debug Logging
Edit `.env` or `config/logging.php` to set log level to debug:
```
LOG_LEVEL=debug
```
### Monitor Logs
```bash
tail -f storage/logs/laravel.log | grep STATUS-DEBUG
```
### Log Format
All logs use `[STATUS-DEBUG]` prefix for easy filtering:
```
[2025-11-19 13:00:00] local.DEBUG: [STATUS-DEBUG] Sentinel status change
{
"source": "PushServerUpdateJob",
"app_id": 123,
"app_name": "my-app",
"old_status": "running:unknown",
"new_status": "running:healthy",
"container_statuses": [...],
"flags": {...}
}
```
## What to Look For
1. **Default to unhealthy**: Check Application model accessor logs
2. **Status flipping**: Compare timestamps between Sentinel and SSH updates
3. **Incorrect aggregation**: Check flags and container_statuses
4. **Stale database values**: Check if old_status persists across multiple logs
## Next Steps
After gathering logs, we can:
1. Identify the exact source of "unhealthy" status
2. Determine if it's a default issue, aggregation bug, or timing problem
3. Apply targeted fix based on evidence
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
128c0b00ec
commit
d2d9c1b2bc
3 changed files with 56 additions and 0 deletions
|
|
@ -11,6 +11,7 @@
|
||||||
use Illuminate\Support\Arr;
|
use Illuminate\Support\Arr;
|
||||||
use Illuminate\Support\Collection;
|
use Illuminate\Support\Collection;
|
||||||
use Illuminate\Support\Facades\DB;
|
use Illuminate\Support\Facades\DB;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
use Lorisleiva\Actions\Concerns\AsAction;
|
use Lorisleiva\Actions\Concerns\AsAction;
|
||||||
|
|
||||||
class GetContainersStatus
|
class GetContainersStatus
|
||||||
|
|
@ -342,9 +343,26 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
|
||||||
|
|
||||||
if ($recentlyRestarted) {
|
if ($recentlyRestarted) {
|
||||||
// Keep it as degraded if it was recently in a crash loop
|
// Keep it as degraded if it was recently in a crash loop
|
||||||
|
Log::debug('[STATUS-DEBUG] Recently restarted - keeping degraded', [
|
||||||
|
'source' => 'GetContainersStatus (not running)',
|
||||||
|
'app_id' => $application->id,
|
||||||
|
'app_name' => $application->name,
|
||||||
|
'old_status' => $application->status,
|
||||||
|
'new_status' => 'degraded (unhealthy)',
|
||||||
|
'restart_count' => $application->restart_count,
|
||||||
|
'last_restart_at' => $application->last_restart_at,
|
||||||
|
]);
|
||||||
$application->update(['status' => 'degraded (unhealthy)']);
|
$application->update(['status' => 'degraded (unhealthy)']);
|
||||||
} else {
|
} else {
|
||||||
// Reset restart count when application exits completely
|
// Reset restart count when application exits completely
|
||||||
|
Log::debug('[STATUS-DEBUG] Application not running', [
|
||||||
|
'source' => 'GetContainersStatus (not running)',
|
||||||
|
'app_id' => $application->id,
|
||||||
|
'app_name' => $application->name,
|
||||||
|
'old_status' => $application->status,
|
||||||
|
'new_status' => 'exited',
|
||||||
|
'containers_exist' => ! $this->containers->isEmpty(),
|
||||||
|
]);
|
||||||
$application->update([
|
$application->update([
|
||||||
'status' => 'exited',
|
'status' => 'exited',
|
||||||
'restart_count' => 0,
|
'restart_count' => 0,
|
||||||
|
|
@ -437,6 +455,15 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti
|
||||||
if ($aggregatedStatus) {
|
if ($aggregatedStatus) {
|
||||||
$statusFromDb = $application->status;
|
$statusFromDb = $application->status;
|
||||||
if ($statusFromDb !== $aggregatedStatus) {
|
if ($statusFromDb !== $aggregatedStatus) {
|
||||||
|
Log::debug('[STATUS-DEBUG] SSH status change', [
|
||||||
|
'source' => 'GetContainersStatus',
|
||||||
|
'app_id' => $application->id,
|
||||||
|
'app_name' => $application->name,
|
||||||
|
'old_status' => $statusFromDb,
|
||||||
|
'new_status' => $aggregatedStatus,
|
||||||
|
'container_statuses' => $containerStatuses->toArray(),
|
||||||
|
'max_restart_count' => $maxRestartCount,
|
||||||
|
]);
|
||||||
$application->update(['status' => $aggregatedStatus]);
|
$application->update(['status' => $aggregatedStatus]);
|
||||||
} else {
|
} else {
|
||||||
$application->update(['last_online_at' => now()]);
|
$application->update(['last_online_at' => now()]);
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@
|
||||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||||
use Illuminate\Queue\SerializesModels;
|
use Illuminate\Queue\SerializesModels;
|
||||||
use Illuminate\Support\Collection;
|
use Illuminate\Support\Collection;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
use Laravel\Horizon\Contracts\Silenced;
|
use Laravel\Horizon\Contracts\Silenced;
|
||||||
|
|
||||||
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
|
||||||
|
|
@ -299,6 +300,19 @@ private function aggregateMultiContainerStatuses()
|
||||||
|
|
||||||
// Update application status with aggregated result
|
// Update application status with aggregated result
|
||||||
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
|
if ($aggregatedStatus && $application->status !== $aggregatedStatus) {
|
||||||
|
Log::debug('[STATUS-DEBUG] Sentinel status change', [
|
||||||
|
'source' => 'PushServerUpdateJob',
|
||||||
|
'app_id' => $application->id,
|
||||||
|
'app_name' => $application->name,
|
||||||
|
'old_status' => $application->status,
|
||||||
|
'new_status' => $aggregatedStatus,
|
||||||
|
'container_statuses' => $relevantStatuses->toArray(),
|
||||||
|
'flags' => [
|
||||||
|
'hasRunning' => $hasRunning,
|
||||||
|
'hasUnhealthy' => $hasUnhealthy,
|
||||||
|
'hasUnknown' => $hasUnknown,
|
||||||
|
],
|
||||||
|
]);
|
||||||
$application->status = $aggregatedStatus;
|
$application->status = $aggregatedStatus;
|
||||||
$application->save();
|
$application->save();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@
|
||||||
use Illuminate\Database\Eloquent\Relations\HasMany;
|
use Illuminate\Database\Eloquent\Relations\HasMany;
|
||||||
use Illuminate\Database\Eloquent\SoftDeletes;
|
use Illuminate\Database\Eloquent\SoftDeletes;
|
||||||
use Illuminate\Support\Collection;
|
use Illuminate\Support\Collection;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
use Illuminate\Support\Facades\Validator;
|
use Illuminate\Support\Facades\Validator;
|
||||||
use Illuminate\Support\Str;
|
use Illuminate\Support\Str;
|
||||||
use OpenApi\Attributes as OA;
|
use OpenApi\Attributes as OA;
|
||||||
|
|
@ -702,6 +703,13 @@ public function status(): Attribute
|
||||||
} else {
|
} else {
|
||||||
$status = $value;
|
$status = $value;
|
||||||
$health = 'unhealthy';
|
$health = 'unhealthy';
|
||||||
|
Log::debug('[STATUS-DEBUG] Status set without health - defaulting to unhealthy', [
|
||||||
|
'source' => 'Application model accessor',
|
||||||
|
'app_id' => $this->id,
|
||||||
|
'app_name' => $this->name,
|
||||||
|
'raw_value' => $value,
|
||||||
|
'result' => "$status:$health",
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
return "$status:$health";
|
return "$status:$health";
|
||||||
|
|
@ -715,6 +723,13 @@ public function status(): Attribute
|
||||||
} else {
|
} else {
|
||||||
$status = $value;
|
$status = $value;
|
||||||
$health = 'unhealthy';
|
$health = 'unhealthy';
|
||||||
|
Log::debug('[STATUS-DEBUG] Status set without health (multi-server) - defaulting to unhealthy', [
|
||||||
|
'source' => 'Application model accessor',
|
||||||
|
'app_id' => $this->id,
|
||||||
|
'app_name' => $this->name,
|
||||||
|
'raw_value' => $value,
|
||||||
|
'result' => "$status:$health",
|
||||||
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
return "$status:$health";
|
return "$status:$health";
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue