From de75ae2eb4a1a610729be1864b1004653e20a315 Mon Sep 17 00:00:00 2001 From: Franck Kerbiriou Date: Thu, 29 Aug 2024 15:20:06 +0200 Subject: [PATCH 1/3] Add Calcom template --- public/svgs/calcom.svg | 9 ++++++ templates/compose/calcom.yaml | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 public/svgs/calcom.svg create mode 100644 templates/compose/calcom.yaml diff --git a/public/svgs/calcom.svg b/public/svgs/calcom.svg new file mode 100644 index 000000000..446b16655 --- /dev/null +++ b/public/svgs/calcom.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/templates/compose/calcom.yaml b/templates/compose/calcom.yaml new file mode 100644 index 000000000..ac737f8aa --- /dev/null +++ b/templates/compose/calcom.yaml @@ -0,0 +1,59 @@ +# documentation: https://cal.com/docs +# slogan: Scheduling infrastructure for everyone. +# tags: calcom,calendso,scheduling,open,source +# logo: svgs/calcom.svg +# port: 3000 + +services: + postgresql: + image: postgres:16-alpine + volumes: + - postgresql-data:/var/lib/postgresql/data + environment: + - POSTGRES_USER=$SERVICE_USER_POSTGRES + - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES + - POSTGRES_DB=${POSTGRES_DB:-calendso} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 20s + retries: 10 + calcom: + image: calcom.docker.scarf.sh/calcom/cal.com + environment: + # Some variables still uses Calcom previous name, Calendso + # Full list https://github.com/calcom/cal.com/blob/main/.env.example + - SERVICE_FQDN_CALCOM_3000 + - NEXT_PUBLIC_LICENSE_CONSENT=agree + - NODE_ENV=production + - NEXT_PUBLIC_WEBAPP_URL=$SERVICE_FQDN_CALCOM + - NEXT_PUBLIC_API_V2_URL=${SERVICE_FQDN_CALCOM}/api/v2 + # NEXTAUTH_URL=http://localhost:3000/api/auth + # It is highly recommended that the NEXTAUTH_SECRET must be overridden and very unique + # Use `openssl rand -base64 32` to generate a key + - NEXTAUTH_SECRET=$SERVICE_BASE64_CALCOM_SECRET + # Encryption key that will be used to encrypt CalDAV credentials, choose a random string, for example with `dd if=/dev/urandom bs=1K count=1 | md5sum` + - CALENDSO_ENCRYPTION_KEY=$SERVICE_BASE64_CALCOM_KEY + - POSTGRES_USER=$SERVICE_USER_POSTGRES + - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES + - POSTGRES_DB=${POSTGRES_DB:-calendso} + - DATABASE_HOST=postgresql + - DATABASE_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@${DATABASE_HOST}/${POSTGRES_DB:-calendso} + # Needed to run migrations while using a connection pooler like PgBouncer + # Use the same one as DATABASE_URL if you're not using a connection pooler + - DATABASE_DIRECT_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@${DATABASE_HOST}/${POSTGRES_DB:-calendso} + # GOOGLE_API_CREDENTIALS={} + # Set this to '1' if you don't want Cal to collect anonymous usage + - CALCOM_TELEMETRY_DISABLED=1 + # E-mail settings + # Configures the global From: header whilst sending emails. + - EMAIL_FROM=$EMAIL_FROM + - EMAIL_FROM_NAME=$EMAIL_FROM_NAME + # Configure SMTP settings (@see https://nodemailer.com/smtp/). + - EMAIL_SERVER_HOST=$EMAIL_SERVER_HOST + - EMAIL_SERVER_PORT=$EMAIL_SERVER_PORT + - EMAIL_SERVER_USER=$EMAIL_SERVER_USER + - EMAIL_SERVER_PASSWORD=$EMAIL_SERVER_PASSWORD + - NEXT_PUBLIC_APP_NAME="Cal.com" + depends_on: + - postgresql From 4f76dc835b6b577d03363a1961a6381750c78110 Mon Sep 17 00:00:00 2001 From: Franck Kerbiriou Date: Sat, 12 Oct 2024 14:04:57 -0500 Subject: [PATCH 2/3] Refactor service --- templates/compose/calcom.yaml | 66 ++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/templates/compose/calcom.yaml b/templates/compose/calcom.yaml index ac737f8aa..c7ea7744c 100644 --- a/templates/compose/calcom.yaml +++ b/templates/compose/calcom.yaml @@ -5,55 +5,63 @@ # port: 3000 services: - postgresql: - image: postgres:16-alpine - volumes: - - postgresql-data:/var/lib/postgresql/data - environment: - - POSTGRES_USER=$SERVICE_USER_POSTGRES - - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES - - POSTGRES_DB=${POSTGRES_DB:-calendso} - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] - interval: 5s - timeout: 20s - retries: 10 calcom: image: calcom.docker.scarf.sh/calcom/cal.com environment: # Some variables still uses Calcom previous name, Calendso + # # Full list https://github.com/calcom/cal.com/blob/main/.env.example - SERVICE_FQDN_CALCOM_3000 - NEXT_PUBLIC_LICENSE_CONSENT=agree - NODE_ENV=production - - NEXT_PUBLIC_WEBAPP_URL=$SERVICE_FQDN_CALCOM + - NEXT_PUBLIC_WEBAPP_URL=${SERVICE_FQDN_CALCOM} - NEXT_PUBLIC_API_V2_URL=${SERVICE_FQDN_CALCOM}/api/v2 - # NEXTAUTH_URL=http://localhost:3000/api/auth + # https://next-auth.js.org/configuration/options#nextauth_url + # From https://github.com/calcom/docker?tab=readme-ov-file#important-run-time-variables, it should be ${NEXT_PUBLIC_WEBAPP_URL}/api/auth + - NEXTAUTH_URL=${SERVICE_FQDN_CALCOM}/api/auth # It is highly recommended that the NEXTAUTH_SECRET must be overridden and very unique # Use `openssl rand -base64 32` to generate a key - - NEXTAUTH_SECRET=$SERVICE_BASE64_CALCOM_SECRET + - NEXTAUTH_SECRET=${NEXTAUTH_SECRET:-$SERVICE_BASE64_CALCOM_SECRET} # Encryption key that will be used to encrypt CalDAV credentials, choose a random string, for example with `dd if=/dev/urandom bs=1K count=1 | md5sum` - - CALENDSO_ENCRYPTION_KEY=$SERVICE_BASE64_CALCOM_KEY - - POSTGRES_USER=$SERVICE_USER_POSTGRES - - POSTGRES_PASSWORD=$SERVICE_PASSWORD_POSTGRES + - CALENDSO_ENCRYPTION_KEY=${CALENDSO_ENCRYPTION_KEY:-$SERVICE_BASE64_CALCOM_KEY} + - POSTGRES_USER=${SERVICE_USER_POSTGRES} + - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES} - POSTGRES_DB=${POSTGRES_DB:-calendso} - DATABASE_HOST=postgresql - - DATABASE_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@${DATABASE_HOST}/${POSTGRES_DB:-calendso} + - DATABASE_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@${DATABASE_HOST:-postgresql}/${POSTGRES_DB:-calendso} # Needed to run migrations while using a connection pooler like PgBouncer - # Use the same one as DATABASE_URL if you're not using a connection pooler - - DATABASE_DIRECT_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@${DATABASE_HOST}/${POSTGRES_DB:-calendso} + # Use the same one as DATABASE_URL if you are not using a connection pooler + - DATABASE_DIRECT_URL=postgresql://${SERVICE_USER_POSTGRES}:${SERVICE_PASSWORD_POSTGRES}@${DATABASE_HOST:-postgresql}/${POSTGRES_DB:-calendso} # GOOGLE_API_CREDENTIALS={} - # Set this to '1' if you don't want Cal to collect anonymous usage + # Set this to 1 if you don't want Cal to collect anonymous usage - CALCOM_TELEMETRY_DISABLED=1 # E-mail settings # Configures the global From: header whilst sending emails. - - EMAIL_FROM=$EMAIL_FROM - - EMAIL_FROM_NAME=$EMAIL_FROM_NAME + - EMAIL_FROM=${EMAIL_FROM} + - EMAIL_FROM_NAME=${EMAIL_FROM_NAME} # Configure SMTP settings (@see https://nodemailer.com/smtp/). - - EMAIL_SERVER_HOST=$EMAIL_SERVER_HOST - - EMAIL_SERVER_PORT=$EMAIL_SERVER_PORT - - EMAIL_SERVER_USER=$EMAIL_SERVER_USER - - EMAIL_SERVER_PASSWORD=$EMAIL_SERVER_PASSWORD + - EMAIL_SERVER_HOST=${EMAIL_SERVER_HOST} + - EMAIL_SERVER_PORT=${EMAIL_SERVER_PORT} + - EMAIL_SERVER_USER=${EMAIL_SERVER_USER} + - EMAIL_SERVER_PASSWORD=${EMAIL_SERVER_PASSWORD} - NEXT_PUBLIC_APP_NAME="Cal.com" + # More info on ALLOWED_HOSTNAMES https://github.com/calcom/cal.com/issues/12201 + - ALLOWED_HOSTNAMES=["${SERVICE_FQDN_CALCOM}"] depends_on: - postgresql + postgresql: + image: postgres:16-alpine + environment: + - POSTGRES_USER=${SERVICE_USER_POSTGRES} + - POSTGRES_PASSWORD=${SERVICE_PASSWORD_POSTGRES} + - POSTGRES_DB=${POSTGRES_DATABASE:-calcom} + volumes: + - calcom-postgresql-data:/var/lib/postgresql/data + healthcheck: + test: + - CMD-SHELL + - pg_isready -U ${SERVICE_USER_POSTGRES} -d ${POSTGRES_DATABASE:-calcom} + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped From b2e515f770553bd18e1972c5906073331766a14b Mon Sep 17 00:00:00 2001 From: Andras Bacsai Date: Mon, 14 Oct 2024 13:32:36 +0200 Subject: [PATCH 3/3] sentinel --- app/Console/Kernel.php | 2 +- app/Jobs/PushServerUpdateJob.php | 101 ++++++++++++++++-- app/Models/Server.php | 31 +++--- ...pdate_metrics_token_in_server_settings.php | 6 ++ .../views/livewire/server/form.blade.php | 2 +- routes/api.php | 7 +- 6 files changed, 120 insertions(+), 29 deletions(-) diff --git a/app/Console/Kernel.php b/app/Console/Kernel.php index 1430fcdd1..6da32b461 100644 --- a/app/Console/Kernel.php +++ b/app/Console/Kernel.php @@ -38,7 +38,7 @@ protected function schedule(Schedule $schedule): void $schedule->job(new CleanupInstanceStuffsJob)->everyMinute()->onOneServer(); // Server Jobs $this->check_scheduled_backups($schedule); - $this->check_resources($schedule); + // $this->check_resources($schedule); $this->check_scheduled_tasks($schedule); $schedule->command('uploads:clear')->everyTwoMinutes(); diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php index 27aa58201..226cf9392 100644 --- a/app/Jobs/PushServerUpdateJob.php +++ b/app/Jobs/PushServerUpdateJob.php @@ -2,14 +2,16 @@ namespace App\Jobs; +use App\Actions\Proxy\StartProxy; +use App\Models\Application; +use App\Models\ApplicationPreview; use App\Models\Server; use Illuminate\Bus\Queueable; -use Illuminate\Contracts\Queue\ShouldBeEncrypted; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; use Illuminate\Queue\SerializesModels; -use Illuminate\Support\Arr; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Log; class PushServerUpdateJob implements ShouldQueue @@ -25,11 +27,19 @@ public function backoff(): int return isDev() ? 1 : 3; } - public function __construct(public Server $server, public $data) {} + public function __construct(public Server $server, public $data) + { + // TODO: Handle multiple servers + // TODO: Handle Preview deployments + // TODO: Handle DB TCP proxies + // TODO: Handle DBs + // TODO: Handle services + // TODO: Handle proxies + } public function handle() { - if (!$this->data) { + if (! $this->data) { throw new \Exception('No data provided'); } $data = collect($this->data); @@ -37,17 +47,90 @@ public function handle() if ($containers->isEmpty()) { return; } + $foundApplicationIds = collect(); + $foundServiceIds = collect(); + $foundProxy = false; foreach ($containers as $container) { - $containerStatus = data_get($container, 'status', 'exited'); - $containerHealth = data_get($container, 'health', 'unhealthy'); + $containerStatus = data_get($container, 'state', 'exited'); + $containerHealth = data_get($container, 'health_status', 'unhealthy'); $containerStatus = "$containerStatus ($containerHealth)"; $labels = collect(data_get($container, 'labels')); - if ($labels->has('coolify.applicationId')) { - $applicationId = $labels->get('coolify.applicationId'); + $coolify_managed = $labels->has('coolify.managed'); + if ($coolify_managed) { + if ($labels->has('coolify.applicationId')) { + $applicationId = $labels->get('coolify.applicationId'); + $pullRequestId = data_get($labels, 'coolify.pullRequestId', '0'); + $foundApplicationIds->push($applicationId); + try { + $this->updateApplicationStatus($applicationId, $pullRequestId, $containerStatus); + } catch (\Exception $e) { + Log::error($e); + } + } elseif ($labels->has('coolify.serviceId')) { + $serviceId = $labels->get('coolify.serviceId'); + $foundServiceIds->push($serviceId); + Log::info("Service: $serviceId, $containerStatus"); + } else { + $name = data_get($container, 'name'); + $uuid = $labels->get('com.docker.compose.service'); + $type = $labels->get('coolify.type'); + if ($name === 'coolify-proxy') { + $foundProxy = true; + Log::info("Proxy: $uuid, $containerStatus"); + } elseif ($type === 'service') { + Log::info("Service: $uuid, $containerStatus"); + } else { + Log::info("Database: $uuid, $containerStatus"); + } + } } - Log::info("$applicationId, $containerStatus"); + } + + // If proxy is not found, start it + if (! $foundProxy && $this->server->isProxyShouldRun()) { + Log::info('Proxy not found, starting it'); + StartProxy::dispatch($this->server); + } + + // Update not found applications + $allApplicationIds = $this->server->applications()->pluck('id'); + $notFoundApplicationIds = $allApplicationIds->diff($foundApplicationIds); + if ($notFoundApplicationIds->isNotEmpty()) { + Log::info('Not found application ids', ['application_ids' => $notFoundApplicationIds]); + $this->updateNotFoundApplications($notFoundApplicationIds); } } + private function updateApplicationStatus(string $applicationId, string $pullRequestId, string $containerStatus) + { + if ($pullRequestId === '0') { + $application = Application::find($applicationId); + if (! $application) { + return; + } + $application->status = $containerStatus; + $application->save(); + Log::info('Application updated', ['application_id' => $applicationId, 'status' => $containerStatus]); + } else { + $application = ApplicationPreview::where('application_id', $applicationId)->where('pull_request_id', $pullRequestId)->first(); + if (! $application) { + return; + } + $application->status = $containerStatus; + $application->save(); + } + } + private function updateNotFoundApplications(Collection $applicationIds) + { + $applicationIds->each(function ($applicationId) { + Log::info('Updating application status', ['application_id' => $applicationId, 'status' => 'exited']); + $application = Application::find($applicationId); + if ($application) { + $application->status = 'exited'; + $application->save(); + Log::info('Application status updated', ['application_id' => $applicationId, 'status' => 'exited']); + } + }); + } } diff --git a/app/Models/Server.php b/app/Models/Server.php index cd7667c70..9e947c20b 100644 --- a/app/Models/Server.php +++ b/app/Models/Server.php @@ -17,8 +17,6 @@ use Spatie\SchemalessAttributes\SchemalessAttributesTrait; use Spatie\Url\Url; use Symfony\Component\Yaml\Yaml; -use Illuminate\Support\Str; - #[OA\Schema( description: 'Server model', @@ -168,7 +166,7 @@ public function proxySet() public function setupDefault404Redirect() { - $dynamic_conf_path = $this->proxyPath() . '/dynamic'; + $dynamic_conf_path = $this->proxyPath().'/dynamic'; $proxy_type = $this->proxyType(); $redirect_url = $this->proxy->redirect_url; if ($proxy_type === ProxyTypes::TRAEFIK->value) { @@ -182,8 +180,8 @@ public function setupDefault404Redirect() respond 404 }'; $conf = - "# This file is automatically generated by Coolify.\n" . - "# Do not edit it manually (only if you know what are you doing).\n\n" . + "# This file is automatically generated by Coolify.\n". + "# Do not edit it manually (only if you know what are you doing).\n\n". $conf; $base64 = base64_encode($conf); instant_remote_process([ @@ -245,8 +243,8 @@ public function setupDefault404Redirect() ]; $conf = Yaml::dump($dynamic_conf, 12, 2); $conf = - "# This file is automatically generated by Coolify.\n" . - "# Do not edit it manually (only if you know what are you doing).\n\n" . + "# This file is automatically generated by Coolify.\n". + "# Do not edit it manually (only if you know what are you doing).\n\n". $conf; $base64 = base64_encode($conf); @@ -255,8 +253,8 @@ public function setupDefault404Redirect() redir $redirect_url }"; $conf = - "# This file is automatically generated by Coolify.\n" . - "# Do not edit it manually (only if you know what are you doing).\n\n" . + "# This file is automatically generated by Coolify.\n". + "# Do not edit it manually (only if you know what are you doing).\n\n". $conf; $base64 = base64_encode($conf); } @@ -274,7 +272,7 @@ public function setupDefault404Redirect() public function setupDynamicProxyConfiguration() { $settings = instanceSettings(); - $dynamic_config_path = $this->proxyPath() . '/dynamic'; + $dynamic_config_path = $this->proxyPath().'/dynamic'; if ($this->proxyType() === ProxyTypes::TRAEFIK->value) { $file = "$dynamic_config_path/coolify.yaml"; if (empty($settings->fqdn) || (isCloud() && $this->id !== 0) || ! $this->isLocalhost()) { @@ -393,8 +391,8 @@ public function setupDynamicProxyConfiguration() } $yaml = Yaml::dump($traefik_dynamic_conf, 12, 2); $yaml = - "# This file is automatically generated by Coolify.\n" . - "# Do not edit it manually (only if you know what are you doing).\n\n" . + "# This file is automatically generated by Coolify.\n". + "# Do not edit it manually (only if you know what are you doing).\n\n". $yaml; $base64 = base64_encode($yaml); @@ -458,13 +456,13 @@ public function proxyPath() if (isDev()) { $proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/caddy'; } else { - $proxy_path = $proxy_path . '/caddy'; + $proxy_path = $proxy_path.'/caddy'; } } elseif ($proxyType === ProxyTypes::NGINX->value) { if (isDev()) { $proxy_path = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/proxy/nginx'; } else { - $proxy_path = $proxy_path . '/nginx'; + $proxy_path = $proxy_path.'/nginx'; } } @@ -536,8 +534,10 @@ public function generateSentinelToken() $encrypted = encrypt($token); $this->settings->sentinel_token = $encrypted; $this->settings->save(); + return $encrypted; } + public function isSentinelEnabled() { return $this->isMetricsEnabled() || $this->isServerApiEnabled(); @@ -989,7 +989,8 @@ public function team() public function isProxyShouldRun() { - if ($this->proxyType() === ProxyTypes::NONE->value || $this->settings->is_build_server) { + // TODO: Do we need "|| $this->proxy->force_stop" here? + if ($this->proxyType() === ProxyTypes::NONE->value || $this->isBuildServer()) { return false; } diff --git a/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php b/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php index 32b1e5349..21c871cf4 100644 --- a/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php +++ b/database/migrations/2024_10_14_090416_update_metrics_token_in_server_settings.php @@ -19,6 +19,9 @@ public function up(): void $table->integer('sentinel_metrics_refresh_rate_seconds')->default(5); $table->integer('sentinel_metrics_history_days')->default(30); }); + Schema::table('servers', function (Blueprint $table) { + $table->dateTime('sentinel_update_at')->default(now()); + }); } /** @@ -34,5 +37,8 @@ public function down(): void $table->dropColumn('sentinel_metrics_refresh_rate_seconds'); $table->dropColumn('sentinel_metrics_history_days'); }); + Schema::table('servers', function (Blueprint $table) { + $table->dropColumn('sentinel_update_at'); + }); } }; diff --git a/resources/views/livewire/server/form.blade.php b/resources/views/livewire/server/form.blade.php index d3f51625a..ace297712 100644 --- a/resources/views/livewire/server/form.blade.php +++ b/resources/views/livewire/server/form.blade.php @@ -282,7 +282,7 @@ functional issues.

{{-- @endif --}} @if (isDev()) - Get Push Data + Push Test {{--
Start Sentinel diff --git a/routes/api.php b/routes/api.php index 76fd93141..db07921a4 100644 --- a/routes/api.php +++ b/routes/api.php @@ -15,7 +15,6 @@ use App\Http\Middleware\OnlyRootApiToken; use App\Jobs\PushServerUpdateJob; use App\Models\Server; -use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Route; Route::get('/health', [OtherController::class, 'healthcheck']); @@ -137,7 +136,7 @@ ], function () { Route::post('/sentinel/push', function () { $token = request()->header('Authorization'); - if (!$token) { + if (! $token) { return response()->json(['message' => 'Unauthorized'], 401); } $naked_token = str_replace('Bearer ', '', $token); @@ -145,11 +144,13 @@ $decrypted_token = json_decode($decrypted, true); $server_uuid = data_get($decrypted_token, 'server_uuid'); $server = Server::where('uuid', $server_uuid)->first(); - if (!$server) { + if (! $server) { return response()->json(['message' => 'Server not found'], 404); } $data = request()->all(); + $server->update(['sentinel_update_at' => now()]); PushServerUpdateJob::dispatch($server, $data); + return response()->json(['message' => 'ok'], 200); }); });