diff --git a/app/Actions/Docker/GetContainersStatus.php b/app/Actions/Docker/GetContainersStatus.php index a1476e120..3631cca24 100644 --- a/app/Actions/Docker/GetContainersStatus.php +++ b/app/Actions/Docker/GetContainersStatus.php @@ -199,12 +199,26 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti $isPublic = data_get($database, 'is_public'); $foundDatabases[] = $database->id; $statusFromDb = $database->status; + + // Track restart count for databases (single-container) + $restartCount = data_get($container, 'RestartCount', 0); + $previousRestartCount = $database->restart_count ?? 0; + if ($statusFromDb !== $containerStatus) { - $database->update(['status' => $containerStatus]); + $updateData = ['status' => $containerStatus]; } else { - $database->update(['last_online_at' => now()]); + $updateData = ['last_online_at' => now()]; } + // Update restart tracking if restart count increased + if ($restartCount > $previousRestartCount) { + $updateData['restart_count'] = $restartCount; + $updateData['last_restart_at'] = now(); + $updateData['last_restart_type'] = 'crash'; + } + + $database->update($updateData); + if ($isPublic) { $foundTcpProxy = $this->containers->filter(function ($value, $key) use ($uuid) { if ($this->server->isSwarm()) { @@ -365,7 +379,13 @@ public function handle(Server $server, ?Collection $containers = null, ?Collecti if (str($database->status)->startsWith('exited')) { continue; } - $database->update(['status' => 'exited']); + // Reset restart tracking when database exits completely + $database->update([ + 'status' => 'exited', + 'restart_count' => 0, + 'last_restart_at' => null, + 'last_restart_type' => null, + ]); $name = data_get($database, 'name'); $fqdn = data_get($database, 'fqdn'); diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 611e06b0b..e5a6e0c99 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -237,8 +237,9 @@ public function handle() $this->foundProxy = true; } elseif ($type === 'service' && $this->isRunning($containerStatus)) { } else { - if ($this->allDatabaseUuids->contains($uuid) && $this->isRunning($containerStatus)) { + if ($this->allDatabaseUuids->contains($uuid) && $this->isActiveOrTransient($containerStatus)) { $this->foundDatabaseUuids->push($uuid); + // TCP proxy should only be started/managed when database is actually running if ($this->allTcpProxyUuids->contains($uuid) && $this->isRunning($containerStatus)) { $this->updateDatabaseStatus($uuid, $containerStatus, tcpProxy: true); } else { @@ -503,20 +504,28 @@ private function updateDatabaseStatus(string $databaseUuid, string $containerSta private function updateNotFoundDatabaseStatus() { $notFoundDatabaseUuids = $this->allDatabaseUuids->diff($this->foundDatabaseUuids); - if ($notFoundDatabaseUuids->isNotEmpty()) { - $notFoundDatabaseUuids->each(function ($databaseUuid) { - $database = $this->databases->where('uuid', $databaseUuid)->first(); - if ($database) { - if ($database->status !== 'exited') { - $database->status = 'exited'; - $database->save(); - } - if ($database->is_public) { - StopDatabaseProxy::dispatch($database); - } - } - }); + if ($notFoundDatabaseUuids->isEmpty()) { + return; } + + // Only protection: Verify we received any container data at all + // If containers collection is completely empty, Sentinel might have failed + if ($this->containers->isEmpty()) { + return; + } + + $notFoundDatabaseUuids->each(function ($databaseUuid) { + $database = $this->databases->where('uuid', $databaseUuid)->first(); + if ($database) { + if (! str($database->status)->startsWith('exited')) { + $database->status = 'exited'; + $database->save(); + } + if ($database->is_public) { + StopDatabaseProxy::dispatch($database); + } + } + }); } private function updateServiceSubStatus(string $serviceId, string $subType, string $subId, string $containerStatus) @@ -576,6 +585,23 @@ private function isRunning(string $containerStatus) return str($containerStatus)->contains('running'); } + /** + * Check if container is in an active or transient state. + * Active states: running + * Transient states: restarting, starting, created, paused + * + * These states indicate the container exists and should be tracked. + * Terminal states (exited, dead, removing) should NOT be tracked. + */ + private function isActiveOrTransient(string $containerStatus): bool + { + return str($containerStatus)->contains('running') || + str($containerStatus)->contains('restarting') || + str($containerStatus)->contains('starting') || + str($containerStatus)->contains('created') || + str($containerStatus)->contains('paused'); + } + private function checkLogDrainContainer() { if ($this->server->isLogDrainEnabled() && $this->foundLogDrainContainer === false) { diff --git a/app/Models/StandaloneClickhouse.php b/app/Models/StandaloneClickhouse.php index 7acef5aae..a76d55abb 100644 --- a/app/Models/StandaloneClickhouse.php +++ b/app/Models/StandaloneClickhouse.php @@ -18,6 +18,9 @@ class StandaloneClickhouse extends BaseModel protected $casts = [ 'clickhouse_password' => 'encrypted', + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', ]; protected static function booted() @@ -247,6 +250,7 @@ protected function internalDbUrl(): Attribute $encodedUser = rawurlencode($this->clickhouse_admin_user); $encodedPass = rawurlencode($this->clickhouse_admin_password); $database = $this->clickhouse_db ?? 'default'; + return "clickhouse://{$encodedUser}:{$encodedPass}@{$this->uuid}:9000/{$database}"; }, ); @@ -264,6 +268,7 @@ protected function externalDbUrl(): Attribute $encodedUser = rawurlencode($this->clickhouse_admin_user); $encodedPass = rawurlencode($this->clickhouse_admin_password); $database = $this->clickhouse_db ?? 'default'; + return "clickhouse://{$encodedUser}:{$encodedPass}@{$serverIp}:{$this->public_port}/{$database}"; } diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.php index 47170056f..f5337b1d5 100644 --- a/app/Models/StandaloneDragonfly.php +++ b/app/Models/StandaloneDragonfly.php @@ -18,6 +18,9 @@ class StandaloneDragonfly extends BaseModel protected $casts = [ 'dragonfly_password' => 'encrypted', + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', ]; protected static function booted() diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.php index 266110d0a..ab24cae2c 100644 --- a/app/Models/StandaloneKeydb.php +++ b/app/Models/StandaloneKeydb.php @@ -18,6 +18,9 @@ class StandaloneKeydb extends BaseModel protected $casts = [ 'keydb_password' => 'encrypted', + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', ]; protected static function booted() diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index aa7f2d31a..e48cfc1e6 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -19,6 +19,9 @@ class StandaloneMariadb extends BaseModel protected $casts = [ 'mariadb_password' => 'encrypted', + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', ]; protected static function booted() diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 9046ab013..9e271b19a 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -16,6 +16,12 @@ class StandaloneMongodb extends BaseModel protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; + protected $casts = [ + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', + ]; + protected static function booted() { static::created(function ($database) { diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index 719387b36..377765697 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -19,6 +19,9 @@ class StandaloneMysql extends BaseModel protected $casts = [ 'mysql_password' => 'encrypted', 'mysql_root_password' => 'encrypted', + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', ]; protected static function booted() diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index 03080fd3d..d9993426a 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -19,6 +19,9 @@ class StandalonePostgresql extends BaseModel protected $casts = [ 'init_scripts' => 'array', 'postgres_password' => 'encrypted', + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', ]; protected static function booted() diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index 6aca8af9a..684bcaeb7 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -16,6 +16,12 @@ class StandaloneRedis extends BaseModel protected $appends = ['internal_db_url', 'external_db_url', 'database_type', 'server_status']; + protected $casts = [ + 'restart_count' => 'integer', + 'last_restart_at' => 'datetime', + 'last_restart_type' => 'string', + ]; + protected static function booted() { static::created(function ($database) { diff --git a/database/migrations/2025_12_17_000002_add_restart_tracking_to_standalone_databases.php b/database/migrations/2025_12_17_000002_add_restart_tracking_to_standalone_databases.php new file mode 100644 index 000000000..2798affd4 --- /dev/null +++ b/database/migrations/2025_12_17_000002_add_restart_tracking_to_standalone_databases.php @@ -0,0 +1,66 @@ +tables as $table) { + if (! Schema::hasColumn($table, 'restart_count')) { + Schema::table($table, function (Blueprint $blueprint) { + $blueprint->integer('restart_count')->default(0)->after('status'); + }); + } + + if (! Schema::hasColumn($table, 'last_restart_at')) { + Schema::table($table, function (Blueprint $blueprint) { + $blueprint->timestamp('last_restart_at')->nullable()->after('restart_count'); + }); + } + + if (! Schema::hasColumn($table, 'last_restart_type')) { + Schema::table($table, function (Blueprint $blueprint) { + $blueprint->string('last_restart_type', 10)->nullable()->after('last_restart_at'); + }); + } + } + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + $columns = ['restart_count', 'last_restart_at', 'last_restart_type']; + + foreach ($this->tables as $table) { + foreach ($columns as $column) { + if (Schema::hasColumn($table, $column)) { + Schema::table($table, function (Blueprint $blueprint) use ($column) { + $blueprint->dropColumn($column); + }); + } + } + } + } +};