From d446cd4f31ce8f808b1f9ba6814c73b02c7b966d Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Tue, 15 Oct 2024 13:39:19 +0200 Subject: [PATCH] sentinel updates --- app/Actions/Server/StartSentinel.php | 39 +++--- app/Actions/Server/StopSentinel.php | 2 + app/Console/Kernel.php | 9 +- app/Jobs/PushServerUpdateJob.php | 48 +++++++- app/Livewire/Server/Charts.php | 12 +- app/Livewire/Server/Form.php | 93 ++++++-------- app/Models/Server.php | 113 +++++++++++++----- app/Models/ServerSetting.php | 3 +- bootstrap/helpers/shared.php | 2 +- ..._07_18_123458_add_force_cleanup_server.php | 2 +- ...pdate_metrics_token_in_server_settings.php | 16 ++- database/seeders/DatabaseSeeder.php | 1 + .../seeders/GenerateSentinelTokenSeeder.php | 25 ++++ database/seeders/ProductionSeeder.php | 1 + openapi.yaml | 2 +- .../views/components/forms/checkbox.blade.php | 5 +- .../views/livewire/server/form.blade.php | 52 +++++--- routes/api.php | 8 +- versions.json | 5 +- 19 files changed, 293 insertions(+), 145 deletions(-) create mode 100644 database/seeders/GenerateSentinelTokenSeeder.php diff --git a/app/Actions/Server/StartSentinel.php b/app/Actions/Server/StartSentinel.php index e7c613eb0..cfea6afd8 100644 --- a/app/Actions/Server/StartSentinel.php +++ b/app/Actions/Server/StartSentinel.php @@ -17,39 +17,46 @@ public function handle(Server $server, $version = 'next', bool $restart = false) } $metrics_history = $server->settings->sentinel_metrics_history_days; $refresh_rate = $server->settings->sentinel_metrics_refresh_rate_seconds; + $push_interval = $server->settings->sentinel_push_interval_seconds; $token = $server->settings->sentinel_token; $endpoint = InstanceSettings::get()->fqdn; - if (isDev()) { + $mount_dir = '/data/coolify/sentinel'; + $image = "ghcr.io/coollabsio/sentinel:$version"; + + if ($server->isLocalhost()) { $endpoint = 'http://host.docker.internal:8000'; + } else { + if (! $endpoint) { + throw new \Exception('You should set FQDN in Instance Settings.'); + } } - if (! $endpoint) { - throw new \Exception('You should set FQDN in Instance Settings.'); - } - // Ensure the endpoint is using HTTPS - $endpoint = str($endpoint)->replace('http://', 'https://')->value(); $environments = [ 'TOKEN' => $token, - 'ENDPOINT' => $endpoint, - 'COLLECTOR_ENABLED' => 'true', + 'PUSH_ENDPOINT' => $endpoint, + 'PUSH_INTERVAL_SECONDS' => $push_interval, + 'COLLECTOR_ENABLED' => $server->isMetricsEnabled() ? 'true' : 'false', 'COLLECTOR_REFRESH_RATE_SECONDS' => $refresh_rate, 'COLLECTOR_RETENTION_PERIOD_DAYS' => $metrics_history, ]; if (isDev()) { - data_set($environments, 'GIN_MODE', 'debug'); - } - $mount_dir = '/data/coolify/sentinel'; - if (isDev()) { + data_set($environments, 'DEBUG', 'true'); $mount_dir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/sentinel'; + $image = 'sentinel'; } - $docker_environments = '-e "'.implode('" -e "', array_map(fn ($key, $value) => "$key=$value", array_keys($environments), $environments)).'"'; - $docker_command = "docker run --pull always --rm -d $docker_environments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mount_dir:/app/db --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway ghcr.io/coollabsio/sentinel:$version"; + $docker_environments = '-e "' . implode('" -e "', array_map(fn($key, $value) => "$key=$value", array_keys($environments), $environments)) . '"'; - return instant_remote_process([ + $docker_command = "docker run -d $docker_environments --name coolify-sentinel -v /var/run/docker.sock:/var/run/docker.sock -v $mount_dir:/app/db --pid host --health-cmd \"curl --fail http://127.0.0.1:8888/api/health || exit 1\" --health-interval 10s --health-retries 3 --add-host=host.docker.internal:host-gateway $image"; + + instant_remote_process([ 'docker rm -f coolify-sentinel || true', "mkdir -p $mount_dir", $docker_command, "chown -R 9999:root $mount_dir", "chmod -R 700 $mount_dir", - ], $server, true); + ], $server); + + $server->settings->is_sentinel_enabled = true; + $server->settings->save(); + $server->sentinelUpdateAt(); } } diff --git a/app/Actions/Server/StopSentinel.php b/app/Actions/Server/StopSentinel.php index 21ffca3bd..68972f0f2 100644 --- a/app/Actions/Server/StopSentinel.php +++ b/app/Actions/Server/StopSentinel.php @@ -3,6 +3,7 @@ namespace App\Actions\Server; use App\Models\Server; +use Carbon\Carbon; use Lorisleiva\Actions\Concerns\AsAction; class StopSentinel @@ -12,5 +13,6 @@ class StopSentinel public function handle(Server $server) { instant_remote_process(['docker rm -f coolify-sentinel'], $server, false); + $server->sentinelUpdateAt(isReset: true); } } diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 6da32b461..a689b35b8 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -12,7 +12,6 @@ use App\Jobs\PullTemplatesFromCDN; use App\Jobs\ScheduledTaskJob; use App\Jobs\ServerCheckJob; -use App\Jobs\ServerStorageCheckJob; use App\Jobs\UpdateCoolifyJob; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledTask; @@ -20,6 +19,7 @@ use App\Models\Team; use Illuminate\Console\Scheduling\Schedule; use Illuminate\Foundation\Console\Kernel as ConsoleKernel; +use Illuminate\Support\Carbon; class Kernel extends ConsoleKernel { @@ -38,7 +38,7 @@ protected function schedule(Schedule $schedule): void $schedule->job(new CleanupInstanceStuffsJob)->everyMinute()->onOneServer(); // Server Jobs $this->check_scheduled_backups($schedule); - // $this->check_resources($schedule); + $this->check_resources($schedule); $this->check_scheduled_tasks($schedule); $schedule->command('uploads:clear')->everyTwoMinutes(); @@ -115,7 +115,10 @@ private function check_resources($schedule) $servers = $this->all_servers->where('ip', '!=', '1.2.3.4'); } foreach ($servers as $server) { - $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); + $last_sentinel_update = $server->sentinel_updated_at; + if (Carbon::parse($last_sentinel_update)->isBefore(now()->subMinutes(4))) { + $schedule->job(new ServerCheckJob($server))->everyMinute()->onOneServer(); + } // $schedule->job(new ServerStorageCheckJob($server))->everyMinute()->onOneServer(); $serverTimezone = $server->settings->server_timezone; if ($server->settings->force_docker_cleanup) { diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index e029a3bf5..82e311d47 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -4,7 +4,9 @@ use App\Actions\Database\StartDatabaseProxy; use App\Actions\Database\StopDatabaseProxy; +use App\Actions\Proxy\CheckProxy; use App\Actions\Proxy\StartProxy; +use App\Actions\Server\InstallLogDrain; use App\Actions\Shared\ComplexStatusCheck; use App\Models\Application; use App\Models\ApplicationPreview; @@ -40,6 +42,8 @@ class PushServerUpdateJob implements ShouldQueue public Collection $allDatabaseUuids; + public Collection $allTcpProxyUuids; + public Collection $allServiceApplicationIds; public Collection $allApplicationPreviewsIds; @@ -59,6 +63,7 @@ class PushServerUpdateJob implements ShouldQueue public Collection $foundApplicationPreviewsIds; public bool $foundProxy = false; + public bool $foundLogDrainContainer = false; public function backoff(): int { @@ -87,6 +92,11 @@ public function handle() throw new \Exception('No data provided'); } $data = collect($this->data); + + $this->serverStatus(); + + $this->server->sentinelUpdateAt(); + $this->containers = collect(data_get($data, 'containers')); if ($this->containers->isEmpty()) { return; @@ -122,6 +132,10 @@ public function handle() $labels = collect(data_get($container, 'labels')); $coolify_managed = $labels->has('coolify.managed'); if ($coolify_managed) { + $name = data_get($container, 'name'); + if ($name === 'coolify-log-drain' && $this->isRunning($containerStatus)) { + $this->foundLogDrainContainer = true; + } if ($labels->has('coolify.applicationId')) { $applicationId = $labels->get('coolify.applicationId'); $pullRequestId = data_get($labels, 'coolify.pullRequestId', '0'); @@ -153,7 +167,6 @@ public function handle() } } else { - $name = data_get($container, 'name'); $uuid = $labels->get('com.docker.compose.service'); $type = $labels->get('coolify.type'); if ($name === 'coolify-proxy' && $this->isRunning($containerStatus)) { @@ -182,12 +195,23 @@ public function handle() $this->updateNotFoundServiceStatus(); $this->updateAdditionalServersStatus(); + + $this->checkLogDrainContainer(); + } catch (\Exception $e) { throw $e; } } + private function serverStatus(){ + if ($this->server->isFunctional() === false) { + throw new \Exception('Server is not ready.'); + } + if ($this->server->status() === false) { + throw new \Exception('Server is not reachable.'); + } + } private function updateApplicationStatus(string $applicationId, string $containerStatus) { $application = $this->applications->where('id', $applicationId)->first(); @@ -247,9 +271,19 @@ private function updateNotFoundApplicationPreviewStatus() private function updateProxyStatus() { // If proxy is not found, start it - if (! $this->foundProxy && $this->server->isProxyShouldRun()) { - ray('Proxy not found, starting it.'); - StartProxy::dispatch($this->server); + if ($this->server->isProxyShouldRun()) { + if ($this->foundProxy === false) { + try { + if (CheckProxy::run($this->server)) { + StartProxy::run($this->server, false); + } + } catch (\Throwable $e) { + logger()->error($e); + } + } else { + $connectProxyToDockerNetworks = connectProxyToNetworks($this->server); + instant_remote_process($connectProxyToDockerNetworks, $this->server, false); + } } } @@ -361,4 +395,10 @@ private function isRunning(string $containerStatus) { return str($containerStatus)->contains('running'); } + + private function checkLogDrainContainer(){ + if ($this->server->isLogDrainEnabled() && $this->foundLogDrainContainer === false) { + InstallLogDrain::dispatch($this->server); + } + } } diff --git a/app/Livewire/Server/Charts.php b/app/Livewire/Server/Charts.php index 0921c7fa4..09b31c0b0 100644 --- a/app/Livewire/Server/Charts.php +++ b/app/Livewire/Server/Charts.php @@ -34,12 +34,12 @@ public function loadData() try { $cpuMetrics = $this->server->getCpuMetrics($this->interval); $memoryMetrics = $this->server->getMemoryMetrics($this->interval); - $cpuMetrics = collect($cpuMetrics)->map(function ($metric) { - return [$metric[0], $metric[1]]; - }); - $memoryMetrics = collect($memoryMetrics)->map(function ($metric) { - return [$metric[0], $metric[1]]; - }); + // $cpuMetrics = collect($cpuMetrics)->map(function ($metric) { + // return [$metric[0], $metric[1]]; + // }); + // $memoryMetrics = collect($memoryMetrics)->map(function ($metric) { + // return [$metric[0], $metric[1]]; + // }); $this->dispatch("refreshChartData-{$this->chartId}-cpu", [ 'seriesData' => $cpuMetrics, ]); diff --git a/app/Livewire/Server/Form.php b/app/Livewire/Server/Form.php index 0bb7e4742..48c7c0ae7 100644 --- a/app/Livewire/Server/Form.php +++ b/app/Livewire/Server/Form.php @@ -58,8 +58,9 @@ public function getListeners() 'server.settings.sentinel_token' => 'required', 'server.settings.sentinel_metrics_refresh_rate_seconds' => 'required|integer|min:1', 'server.settings.sentinel_metrics_history_days' => 'required|integer|min:1', + 'server.settings.sentinel_push_interval_seconds' => 'required|integer|min:10', 'wildcard_domain' => 'nullable|url', - 'server.settings.is_server_api_enabled' => 'required|boolean', + 'server.settings.is_sentinel_enabled' => 'required|boolean', 'server.settings.server_timezone' => 'required|string|timezone', 'server.settings.force_docker_cleanup' => 'required|boolean', 'server.settings.docker_cleanup_frequency' => 'required_if:server.settings.force_docker_cleanup,true|string', @@ -85,7 +86,8 @@ public function getListeners() 'server.settings.sentinel_token' => 'Metrics Token', 'server.settings.sentinel_metrics_refresh_rate_seconds' => 'Metrics Interval', 'server.settings.sentinel_metrics_history_days' => 'Metrics History', - 'server.settings.is_server_api_enabled' => 'Server API', + 'server.settings.sentinel_push_interval_seconds' => 'Push Interval', + 'server.settings.is_sentinel_enabled' => 'Server API', 'server.settings.server_timezone' => 'Server Timezone', 'server.settings.delete_unused_volumes' => 'Delete Unused Volumes', 'server.settings.delete_unused_networks' => 'Delete Unused Networks', @@ -102,12 +104,17 @@ public function mount(Server $server) $this->server->settings->delete_unused_networks = $server->settings->delete_unused_networks; } + public function checkSyncStatus(){ + $this->server->refresh(); + $this->server->settings->refresh(); + } + public function regenerateSentinelToken() { try { $this->server->generateSentinelToken(); $this->server->settings->refresh(); - $this->dispatch('success', 'Metrics token regenerated.'); + $this->dispatch('success', 'Sentinel token regenerated. Please restart your Sentinel.'); } catch (\Throwable $e) { return handleError($e, $this); } @@ -143,18 +150,22 @@ public function updatedServerSettingsIsBuildServer() $this->dispatch('proxyStatusUpdated'); } - public function checkPortForServerApi() - { - try { - if ($this->server->settings->is_server_api_enabled === true) { - $this->server->checkServerApi(); - $this->dispatch('success', 'Server API is reachable.'); - } - } catch (\Throwable $e) { - return handleError($e, $this); + public function updatedServerSettingsIsSentinelEnabled($value){ + if($value === false){ + StopSentinel::dispatch($this->server); + $this->server->settings->is_metrics_enabled = false; + $this->server->settings->save(); + $this->server->sentinelUpdateAt(isReset: true); + } else { + StartSentinel::run($this->server); } } + public function updatedServerSettingsIsMetricsEnabled(){ + $this->restartSentinel(); + } + + public function instantSave() { try { @@ -165,19 +176,20 @@ public function instantSave() $this->server->save(); $this->dispatch('success', 'Server updated.'); $this->dispatch('refreshServerShow'); - if ($this->server->isSentinelEnabled()) { - PullSentinelImageJob::dispatchSync($this->server); - ray('Sentinel is enabled'); - if ($this->server->settings->isDirty('is_metrics_enabled')) { - $this->dispatch('reloadWindow'); - } - if ($this->server->settings->isDirty('is_server_api_enabled') && $this->server->settings->is_server_api_enabled === true) { - ray('Starting sentinel'); - } - } else { - ray('Sentinel is not enabled'); - StopSentinel::dispatch($this->server); - } + + // if ($this->server->isSentinelEnabled()) { + // PullSentinelImageJob::dispatchSync($this->server); + // ray('Sentinel is enabled'); + // if ($this->server->settings->isDirty('is_metrics_enabled')) { + // $this->dispatch('reloadWindow'); + // } + // if ($this->server->settings->isDirty('is_sentinel_enabled') && $this->server->settings->is_sentinel_enabled === true) { + // ray('Starting sentinel'); + // } + // } else { + // ray('Sentinel is not enabled'); + // StopSentinel::dispatch($this->server); + // } $this->server->settings->save(); // $this->checkPortForServerApi(); @@ -186,35 +198,12 @@ public function instantSave() } } - public function getPushData() - { - try { - if (! isDev()) { - throw new \Exception('This feature is only available in dev mode.'); - } - $response = Http::withHeaders([ - 'Authorization' => 'Bearer '.$this->server->settings->sentinel_token, - ])->post('http://host.docker.internal:8888/api/push', [ - 'data' => 'test', - ]); - if ($response->successful()) { - $this->dispatch('success', 'Push data sent.'); - - return; - } - $error = data_get($response->json(), 'error'); - throw new \Exception($error); - } catch (\Throwable $e) { - return handleError($e, $this); - } - } - public function restartSentinel() { try { $version = get_latest_sentinel_version(); StartSentinel::run($this->server, $version, true); - $this->dispatch('success', 'Sentinel restarted.'); + $this->dispatch('success', 'Sentinel started.'); } catch (\Throwable $e) { return handleError($e, $this); } @@ -307,10 +296,4 @@ public function manualCloudflareConfig() $this->server->refresh(); $this->dispatch('success', 'Cloudflare Tunnels enabled.'); } - - public function startSentinel() - { - StartSentinel::run($this->server); - $this->dispatch('success', 'Sentinel started.'); - } } diff --git a/app/Models/Server.php b/app/Models/Server.php index 9e947c20b..aac3ddddf 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -7,6 +7,7 @@ use App\Jobs\PullSentinelImageJob; use Illuminate\Database\Eloquent\Builder; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Support\Carbon; use Illuminate\Support\Collection; use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Process; @@ -16,6 +17,7 @@ use Spatie\SchemalessAttributes\Casts\SchemalessAttributes; use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Spatie\Url\Url; +use Illuminate\Support\Str; use Symfony\Component\Yaml\Yaml; #[OA\Schema( @@ -166,7 +168,7 @@ public function proxySet() public function setupDefault404Redirect() { - $dynamic_conf_path = $this->proxyPath().'/dynamic'; + $dynamic_conf_path = $this->proxyPath() . '/dynamic'; $proxy_type = $this->proxyType(); $redirect_url = $this->proxy->redirect_url; if ($proxy_type === ProxyTypes::TRAEFIK->value) { @@ -180,8 +182,8 @@ public function setupDefault404Redirect() respond 404 }'; $conf = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $conf; $base64 = base64_encode($conf); instant_remote_process([ @@ -243,8 +245,8 @@ public function setupDefault404Redirect() ]; $conf = Yaml::dump($dynamic_conf, 12, 2); $conf = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $conf; $base64 = base64_encode($conf); @@ -253,8 +255,8 @@ public function setupDefault404Redirect() redir $redirect_url }"; $conf = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $conf; $base64 = base64_encode($conf); } @@ -272,7 +274,7 @@ public function setupDefault404Redirect() public function setupDynamicProxyConfiguration() { $settings = instanceSettings(); - $dynamic_config_path = $this->proxyPath().'/dynamic'; + $dynamic_config_path = $this->proxyPath() . '/dynamic'; if ($this->proxyType() === ProxyTypes::TRAEFIK->value) { $file = "$dynamic_config_path/coolify.yaml"; if (empty($settings->fqdn) || (isCloud() && $this->id !== 0) || ! $this->isLocalhost()) { @@ -391,8 +393,8 @@ public function setupDynamicProxyConfiguration() } $yaml = Yaml::dump($traefik_dynamic_conf, 12, 2); $yaml = - "# This file is automatically generated by Coolify.\n". - "# Do not edit it manually (only if you know what are you doing).\n\n". + "# This file is automatically generated by Coolify.\n" . + "# Do not edit it manually (only if you know what are you doing).\n\n" . $yaml; $base64 = base64_encode($yaml); @@ -456,13 +458,13 @@ public function proxyPath() if (isDev()) { $proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/caddy'; } else { - $proxy_path = $proxy_path.'/caddy'; + $proxy_path = $proxy_path . '/caddy'; } } elseif ($proxyType === ProxyTypes::NGINX->value) { if (isDev()) { $proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/nginx'; } else { - $proxy_path = $proxy_path.'/nginx'; + $proxy_path = $proxy_path . '/nginx'; } } @@ -538,6 +540,16 @@ public function generateSentinelToken() return $encrypted; } + public function sentinelUpdateAt(bool $isReset = false) + { + $this->sentinel_updated_at = $isReset ? now()->subMinutes(6000) : now(); + $this->save(); + } + public function isSentinelLive() + { + return Carbon::parse($this->sentinel_updated_at)->isAfter(now()->subMinutes(4)); + } + public function isSentinelEnabled() { return $this->isMetricsEnabled() || $this->isServerApiEnabled(); @@ -550,7 +562,7 @@ public function isMetricsEnabled() public function isServerApiEnabled() { - return $this->settings->is_server_api_enabled; + return $this->settings->is_sentinel_enabled; } public function checkServerApi() @@ -591,7 +603,15 @@ public function getCpuMetrics(int $mins = 5) { if ($this->isMetricsEnabled()) { $from = now()->subMinutes($mins)->toIso8601ZuluString(); - $cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false); + if (isDev() && $this->id === 0) { + $process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/cpu/history?from=$from"); + if ($process->failed()) { + throw new \Exception($process->errorOutput()); + } + $cpu = $process->output(); + } else { + $cpu = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/cpu/history?from=$from'"], $this, false); + } if (str($cpu)->contains('error')) { $error = json_decode($cpu, true); $error = data_get($error, 'error', 'Something is not okay, are you okay?'); @@ -600,17 +620,12 @@ public function getCpuMetrics(int $mins = 5) } throw new \Exception($error); } - $cpu = str($cpu)->explode("\n")->skip(1)->all(); - $parsedCollection = collect($cpu)->flatMap(function ($item) { - return collect(explode("\n", trim($item)))->map(function ($line) { - [$time, $cpu_usage_percent] = explode(',', trim($line)); - $cpu_usage_percent = number_format($cpu_usage_percent, 0); - - return [(int) $time, (float) $cpu_usage_percent]; - }); + $cpu = json_decode($cpu, true); + $parsedCollection = collect($cpu)->map(function ($metric) { + return [(int)$metric['time'], (float)$metric['percent']]; }); + return $parsedCollection; - return $parsedCollection->toArray(); } } @@ -618,7 +633,15 @@ public function getMemoryMetrics(int $mins = 5) { if ($this->isMetricsEnabled()) { $from = now()->subMinutes($mins)->toIso8601ZuluString(); - $memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false); + if (isDev() && $this->id === 0) { + $process = Process::run("curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://host.docker.internal:8888/api/memory/history?from=$from"); + if ($process->failed()) { + throw new \Exception($process->errorOutput()); + } + $memory = $process->output(); + } else { + $memory = instant_remote_process(["docker exec coolify-sentinel sh -c 'curl -H \"Authorization: Bearer {$this->settings->sentinel_token}\" http://localhost:8888/api/memory/history?from=$from'"], $this, false); + } if (str($memory)->contains('error')) { $error = json_decode($memory, true); $error = data_get($error, 'error', 'Something is not okay, are you okay?'); @@ -627,14 +650,9 @@ public function getMemoryMetrics(int $mins = 5) } throw new \Exception($error); } - $memory = str($memory)->explode("\n")->skip(1)->all(); - $parsedCollection = collect($memory)->flatMap(function ($item) { - return collect(explode("\n", trim($item)))->map(function ($line) { - [$time, $used, $free, $usedPercent] = explode(',', trim($line)); - $usedPercent = number_format($usedPercent, 0); - - return [(int) $time, (float) $usedPercent]; - }); + $memory = json_decode($memory, true); + $parsedCollection = collect($memory)->map(function ($metric) { + return [(int)$metric['time'], (float)$metric['usedPercent']]; }); return $parsedCollection->toArray(); @@ -1054,6 +1072,37 @@ public function isSwarmWorker() return data_get($this, 'settings.is_swarm_worker'); } + public function status(): bool + { + ['uptime' => $uptime] = $this->validateConnection(false); + if ($uptime) { + if ($this->unreachable_notification_sent === true) { + $this->update(['unreachable_notification_sent' => false]); + } + } else { + // $this->server->team?->notify(new Unreachable($this->server)); + foreach ($this->applications as $application) { + $application->update(['status' => 'exited']); + } + foreach ($this->databases as $database) { + $database->update(['status' => 'exited']); + } + foreach ($this->services as $service) { + $apps = $service->applications()->get(); + $dbs = $service->databases()->get(); + foreach ($apps as $app) { + $app->update(['status' => 'exited']); + } + foreach ($dbs as $db) { + $db->update(['status' => 'exited']); + } + } + + return false; + } + + return true; + } public function validateConnection($isManualCheck = true) { config()->set('constants.ssh.mux_enabled', ! $isManualCheck); diff --git a/app/Models/ServerSetting.php b/app/Models/ServerSetting.php index f5e0f7b0b..f2eba4854 100644 --- a/app/Models/ServerSetting.php +++ b/app/Models/ServerSetting.php @@ -24,7 +24,7 @@ 'is_logdrain_newrelic_enabled' => ['type' => 'boolean'], 'is_metrics_enabled' => ['type' => 'boolean'], 'is_reachable' => ['type' => 'boolean'], - 'is_server_api_enabled' => ['type' => 'boolean'], + 'is_sentinel_enabled' => ['type' => 'boolean'], 'is_swarm_manager' => ['type' => 'boolean'], 'is_swarm_worker' => ['type' => 'boolean'], 'is_usable' => ['type' => 'boolean'], @@ -55,7 +55,6 @@ class ServerSetting extends Model 'docker_cleanup_threshold' => 'integer', 'sentinel_token' => 'encrypted', ]; - public function server() { return $this->belongsTo(Server::class); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 86c6def76..7ed806d43 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -164,7 +164,7 @@ function get_route_parameters(): array function get_latest_sentinel_version(): string { try { - $response = Http::get('https://cdn.coollabs.io/sentinel/versions.json'); + $response = Http::get('https://cdn.coollabs.io/coolify/versions.json'); $versions = $response->json(); return data_get($versions, 'sentinel.version'); diff --git a/database/migrations/2024_07_18_123458_add_force_cleanup_server.php b/database/migrations/2024_07_18_123458_add_force_cleanup_server.php index a33665bd0..ea3695b3f 100644 --- a/database/migrations/2024_07_18_123458_add_force_cleanup_server.php +++ b/database/migrations/2024_07_18_123458_add_force_cleanup_server.php @@ -12,7 +12,7 @@ public function up(): void { Schema::table('server_settings', function (Blueprint $table) { - $table->boolean('is_force_cleanup_enabled')->default(false)->after('is_sentinel_enabled'); + $table->boolean('is_force_cleanup_enabled')->default(false); }); } diff --git a/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php b/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php index 21c871cf4..051457600 100644 --- a/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php +++ b/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php @@ -15,12 +15,16 @@ public function up(): void $table->dropColumn('metrics_token'); $table->dropColumn('metrics_refresh_rate_seconds'); $table->dropColumn('metrics_history_days'); + $table->dropColumn('is_server_api_enabled'); + + $table->boolean('is_sentinel_enabled')->default(true); $table->text('sentinel_token')->nullable(); - $table->integer('sentinel_metrics_refresh_rate_seconds')->default(5); - $table->integer('sentinel_metrics_history_days')->default(30); + $table->integer('sentinel_metrics_refresh_rate_seconds')->default(10); + $table->integer('sentinel_metrics_history_days')->default(7); + $table->integer('sentinel_push_interval_seconds')->default(60); }); Schema::table('servers', function (Blueprint $table) { - $table->dateTime('sentinel_update_at')->default(now()); + $table->dateTime('sentinel_updated_at')->default(now()); }); } @@ -33,12 +37,16 @@ public function down(): void $table->string('metrics_token')->nullable(); $table->integer('metrics_refresh_rate_seconds')->default(5); $table->integer('metrics_history_days')->default(30); + $table->boolean('is_server_api_enabled')->default(false); + $table->dropColumn('sentinel_token'); $table->dropColumn('sentinel_metrics_refresh_rate_seconds'); $table->dropColumn('sentinel_metrics_history_days'); + $table->dropColumn('sentinel_push_interval_seconds'); + $table->dropColumn('is_sentinel_enabled'); }); Schema::table('servers', function (Blueprint $table) { - $table->dropColumn('sentinel_update_at'); + $table->dropColumn('sentinel_updated_at'); }); } }; diff --git a/database/seeders/DatabaseSeeder.php b/database/seeders/DatabaseSeeder.php index be5083108..1888f0440 100644 --- a/database/seeders/DatabaseSeeder.php +++ b/database/seeders/DatabaseSeeder.php @@ -26,6 +26,7 @@ public function run(): void S3StorageSeeder::class, StandalonePostgresqlSeeder::class, OauthSettingSeeder::class, + GenerateSentinelTokenSeeder::class, ]); } } diff --git a/database/seeders/GenerateSentinelTokenSeeder.php b/database/seeders/GenerateSentinelTokenSeeder.php new file mode 100644 index 000000000..d915f7259 --- /dev/null +++ b/database/seeders/GenerateSentinelTokenSeeder.php @@ -0,0 +1,25 @@ +settings->sentinel_token)->isEmpty()) { + $server->generateSentinelToken(); + } + } + }); + } catch (\Throwable $e) { + echo "Error: {$e->getMessage()}\n"; + ray($e->getMessage()); + } + } +} diff --git a/database/seeders/ProductionSeeder.php b/database/seeders/ProductionSeeder.php index 206f04d6b..a1a025e8d 100644 --- a/database/seeders/ProductionSeeder.php +++ b/database/seeders/ProductionSeeder.php @@ -186,6 +186,7 @@ public function run(): void $this->call(OauthSettingSeeder::class); $this->call(PopulateSshKeysDirectorySeeder::class); + $this->call(GenerateSentinelTokenSeeder::class); } } diff --git a/openapi.yaml b/openapi.yaml index 3521b7de4..0963857c9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4959,7 +4959,7 @@ components: type: boolean is_reachable: type: boolean - is_server_api_enabled: + is_sentinel_enabled: type: boolean is_swarm_manager: type: boolean diff --git a/resources/views/components/forms/checkbox.blade.php b/resources/views/components/forms/checkbox.blade.php index 439fc4ad2..fed6ad77f 100644 --- a/resources/views/components/forms/checkbox.blade.php +++ b/resources/views/components/forms/checkbox.blade.php @@ -14,7 +14,10 @@ 'w-full' => $fullWidth, ])> @if (!$hideLabel) -