From 06dfcff55951a80c799e52fa261c4ad85f1e30e7 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Sat, 4 Oct 2025 19:19:15 +0200 Subject: [PATCH] refactor(deployment): standardize environment variable handling in ApplicationDeploymentJob - Replaced the use of a dynamic env_filename with a consistent .env file reference across deployment methods. - Simplified the generation and saving of build-time and runtime environment variables, ensuring they are always written to the .env file. - Enhanced clarity in the deployment process by removing redundant logic and ensuring environment variables are handled uniformly. --- app/Jobs/ApplicationDeploymentJob.php | 434 ++++++++++++++++++-------- 1 file changed, 303 insertions(+), 131 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 93f730335..7bbcd67c9 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -116,16 +116,12 @@ class ApplicationDeploymentJob implements ShouldBeEncrypted, ShouldQueue private $env_args; - private $environment_variables; - private $env_nixpacks_args; private $docker_compose; private $docker_compose_base64; - private ?string $env_filename = null; - private ?string $nixpacks_plan = null; private Collection $nixpacks_plan_json; @@ -356,7 +352,7 @@ public function handle(): void } $this->application_deployment_queue->addLogEntry("Gracefully shutting down build container: {$this->deployment_uuid}"); - $this->graceful_shutdown_container($this->deployment_uuid); + // $this->graceful_shutdown_container($this->deployment_uuid); ServiceStatusChanged::dispatch(data_get($this->application, 'environment.project.team.id')); } @@ -576,7 +572,6 @@ private function deploy_docker_compose_buildpack() if ($this->application->settings->is_raw_compose_deployment_enabled) { $this->application->oldRawParser(); $yaml = $composeFile = $this->application->docker_compose_raw; - $this->generate_runtime_environment_variables(); // For raw compose, we cannot automatically add secrets configuration // User must define it manually in their docker-compose file @@ -585,16 +580,14 @@ private function deploy_docker_compose_buildpack() } } else { $composeFile = $this->application->parse(pull_request_id: $this->pull_request_id, preview_id: data_get($this->preview, 'id')); - $this->generate_runtime_environment_variables(); - if (filled($this->env_filename)) { - $services = collect(data_get($composeFile, 'services', [])); - $services = $services->map(function ($service, $name) { - $service['env_file'] = [$this->env_filename]; + // Always add .env file to services + $services = collect(data_get($composeFile, 'services', [])); + $services = $services->map(function ($service, $name) { + $service['env_file'] = ['.env']; - return $service; - }); - $composeFile['services'] = $services->toArray(); - } + return $service; + }); + $composeFile['services'] = $services->toArray(); if (empty($composeFile)) { $this->application_deployment_queue->addLogEntry('Failed to parse docker-compose file.'); $this->fail('Failed to parse docker-compose file.'); @@ -620,6 +613,9 @@ private function deploy_docker_compose_buildpack() // Build new container to limit downtime. $this->application_deployment_queue->addLogEntry('Pulling & building required images.'); + // Save build-time .env file BEFORE the build + $this->save_buildtime_environment_variables(); + if ($this->docker_compose_custom_build_command) { // Prepend DOCKER_BUILDKIT=1 if BuildKit is supported $build_command = $this->docker_compose_custom_build_command; @@ -635,9 +631,8 @@ private function deploy_docker_compose_buildpack() if ($this->dockerBuildkitSupported) { $command = "DOCKER_BUILDKIT=1 {$command}"; } - if (filled($this->env_filename)) { - $command .= " --env-file {$this->workdir}/{$this->env_filename}"; - } + // Always use .env file + $command .= " --env-file {$this->workdir}/.env"; if ($this->force_rebuild) { $command .= " --project-name {$this->application->uuid} --project-directory {$this->workdir} -f {$this->workdir}{$this->docker_compose_location} build --pull --no-cache"; } else { @@ -657,6 +652,10 @@ private function deploy_docker_compose_buildpack() ); } + // Save runtime environment variables AFTER the build + // This overwrites the build-time .env with ALL variables (build-time + runtime) + $this->save_runtime_environment_variables(); + $this->stop_running_container(force: true); $this->application_deployment_queue->addLogEntry('Starting new application.'); $networkId = $this->application->uuid; @@ -690,9 +689,8 @@ private function deploy_docker_compose_buildpack() $this->docker_compose_location = '/docker-compose.yaml'; $command = "{$this->coolify_variables} docker compose"; - if (filled($this->env_filename)) { - $command .= " --env-file {$server_workdir}/{$this->env_filename}"; - } + // Always use .env file + $command .= " --env-file {$server_workdir}/.env"; $command .= " --project-directory {$server_workdir} -f {$server_workdir}{$this->docker_compose_location} up -d"; $this->execute_remote_command( ['command' => $command, 'hidden' => true], @@ -707,9 +705,8 @@ private function deploy_docker_compose_buildpack() } else { $command = "{$this->coolify_variables} docker compose"; if ($this->preserveRepository) { - if (filled($this->env_filename)) { - $command .= " --env-file {$server_workdir}/{$this->env_filename}"; - } + // Always use .env file + $command .= " --env-file {$server_workdir}/.env"; $command .= " --project-name {$this->application->uuid} --project-directory {$server_workdir} -f {$server_workdir}{$this->docker_compose_location} up -d"; $this->write_deployment_configurations(); @@ -717,9 +714,8 @@ private function deploy_docker_compose_buildpack() ['command' => $command, 'hidden' => true], ); } else { - if (filled($this->env_filename)) { - $command .= " --env-file {$this->workdir}/{$this->env_filename}"; - } + // Always use .env file + $command .= " --env-file {$this->workdir}/.env"; $command .= " --project-name {$this->application->uuid} --project-directory {$this->workdir} -f {$this->workdir}{$this->docker_compose_location} up -d"; $this->execute_remote_command( [executeInDocker($this->deployment_uuid, $command), 'hidden' => true], @@ -753,9 +749,18 @@ private function deploy_dockerfile_buildpack() } $this->cleanup_git(); $this->generate_compose_file(); + + // Save build-time .env file BEFORE the build + $this->save_buildtime_environment_variables(); + $this->generate_build_env_variables(); $this->add_build_env_variables_to_dockerfile(); $this->build_image(); + + // Save runtime environment variables AFTER the build + // This overwrites the build-time .env with ALL variables (build-time + runtime) + $this->save_runtime_environment_variables(); + $this->push_to_docker_registry(); $this->rolling_update(); } @@ -779,11 +784,15 @@ private function deploy_nixpacks_buildpack() $this->cleanup_git(); $this->generate_nixpacks_confs(); $this->generate_compose_file(); + + // Save build-time .env file BEFORE the build for Nixpacks + $this->save_buildtime_environment_variables(); + $this->generate_build_env_variables(); $this->build_image(); // For Nixpacks, save runtime environment variables AFTER the build - // to prevent them from being accessible during the build process + // This overwrites the build-time .env with ALL variables (build-time + runtime) $this->save_runtime_environment_variables(); $this->push_to_docker_registry(); $this->rolling_update(); @@ -807,7 +816,16 @@ private function deploy_static_buildpack() $this->clone_repository(); $this->cleanup_git(); $this->generate_compose_file(); + + // Save build-time .env file BEFORE the build + $this->save_buildtime_environment_variables(); + $this->build_static_image(); + + // Save runtime environment variables AFTER the build + // This overwrites the build-time .env with ALL variables (build-time + runtime) + $this->save_runtime_environment_variables(); + $this->push_to_docker_registry(); $this->rolling_update(); } @@ -1060,8 +1078,6 @@ private function generate_runtime_environment_variables() $envs->push($key.'='.$item); }); if ($this->pull_request_id === 0) { - $this->env_filename = '.env'; - // Generate SERVICE_ variables first for dockercompose if ($this->build_pack === 'dockercompose') { $domains = collect(json_decode($this->application->docker_compose_domains)) ?? collect([]); @@ -1120,8 +1136,6 @@ private function generate_runtime_environment_variables() $envs->push('HOST=0.0.0.0'); } } else { - $this->env_filename = '.env'; - // Generate SERVICE_ variables first for dockercompose preview if ($this->build_pack === 'dockercompose') { $domains = collect(json_decode(data_get($this->preview, 'docker_compose_domains'))) ?? collect([]); @@ -1176,99 +1190,250 @@ private function generate_runtime_environment_variables() $envs->push('HOST=0.0.0.0'); } } - if ($envs->isEmpty()) { - if ($this->env_filename) { - if ($this->use_build_server) { - $this->server = $this->original_server; - $this->execute_remote_command( - [ - 'command' => "rm -f $this->configuration_dir/{$this->env_filename}", - 'hidden' => true, - 'ignore_errors' => true, - ] - ); - $this->server = $this->build_server; - $this->execute_remote_command( - [ - 'command' => "rm -f $this->configuration_dir/{$this->env_filename}", - 'hidden' => true, - 'ignore_errors' => true, - ] - ); - } else { - $this->execute_remote_command( - [ - 'command' => "rm -f $this->configuration_dir/{$this->env_filename}", - 'hidden' => true, - 'ignore_errors' => true, - ] - ); - } - } - $this->env_filename = null; - } else { - // For Nixpacks builds, we save the .env file AFTER the build to prevent - // runtime-only variables from being accessible during the build process - if ($this->application->build_pack !== 'nixpacks' || $this->skip_build) { - $envs_base64 = base64_encode($envs->implode("\n")); - $this->execute_remote_command( - [ - executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d | tee $this->workdir/{$this->env_filename} > /dev/null"), - ], - ); - if ($this->use_build_server) { - $this->server = $this->original_server; - $this->execute_remote_command( - [ - "echo '$envs_base64' | base64 -d | tee $this->configuration_dir/{$this->env_filename} > /dev/null", - ] - ); - $this->server = $this->build_server; - } else { - $this->execute_remote_command( - [ - "echo '$envs_base64' | base64 -d | tee $this->configuration_dir/{$this->env_filename} > /dev/null", - ] - ); - } - } - } - $this->environment_variables = $envs; + // Return the generated environment variables instead of storing them globally + return $envs; } private function save_runtime_environment_variables() { - // This method saves the .env file with runtime variables - // It should be called AFTER the build for Nixpacks to prevent runtime-only variables - // from being accessible during the build process + // This method saves the .env file with ALL runtime variables + // For builds, it should be called AFTER the build to include runtime-only variables - if ($this->environment_variables && $this->environment_variables->isNotEmpty() && $this->env_filename) { - $envs_base64 = base64_encode($this->environment_variables->implode("\n")); + // Generate runtime environment variables locally + $environment_variables = $this->generate_runtime_environment_variables(); - // Write .env file to workdir (for container runtime) + // Handle empty environment variables + if ($environment_variables->isEmpty()) { + // For Docker Compose, we need to create an empty .env file + // because we always reference it in the compose file + if ($this->build_pack === 'dockercompose') { + $this->application_deployment_queue->addLogEntry('Creating empty .env file (no environment variables defined).'); + + // Create empty .env file + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "touch $this->workdir/.env"), + ] + ); + + // Also create in configuration directory + if ($this->use_build_server) { + $this->server = $this->original_server; + $this->execute_remote_command( + [ + "touch $this->configuration_dir/.env", + ] + ); + $this->server = $this->build_server; + } else { + $this->execute_remote_command( + [ + "touch $this->configuration_dir/.env", + ] + ); + } + } else { + // For non-Docker Compose deployments, clean up any existing .env files + if ($this->use_build_server) { + $this->server = $this->original_server; + $this->execute_remote_command( + [ + 'command' => "rm -f $this->configuration_dir/.env", + 'hidden' => true, + 'ignore_errors' => true, + ] + ); + $this->server = $this->build_server; + $this->execute_remote_command( + [ + 'command' => "rm -f $this->configuration_dir/.env", + 'hidden' => true, + 'ignore_errors' => true, + ] + ); + } else { + $this->execute_remote_command( + [ + 'command' => "rm -f $this->configuration_dir/.env", + 'hidden' => true, + 'ignore_errors' => true, + ] + ); + } + } + + return; + } + + // Write the environment variables to file + $envs_base64 = base64_encode($environment_variables->implode("\n")); + + // Write .env file to workdir (for container runtime) + $this->application_deployment_queue->addLogEntry('Creating .env file with runtime variables for build phase.', hidden: true); + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d | tee $this->workdir/.env > /dev/null"), + ], + [ + executeInDocker($this->deployment_uuid, "cat $this->workdir/.env"), + 'hidden' => true, + + ] + ); + + // Write .env file to configuration directory + if ($this->use_build_server) { + $this->server = $this->original_server; $this->execute_remote_command( [ - executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d | tee $this->workdir/{$this->env_filename} > /dev/null"), + "echo '$envs_base64' | base64 -d | tee $this->configuration_dir/.env > /dev/null", + ] + ); + $this->server = $this->build_server; + } else { + $this->execute_remote_command( + [ + "echo '$envs_base64' | base64 -d | tee $this->configuration_dir/.env > /dev/null", + ] + ); + } + } + + private function generate_buildtime_environment_variables() + { + $envs = collect([]); + $coolify_envs = $this->generate_coolify_env_variables(); + + // Add COOLIFY variables + $coolify_envs->each(function ($item, $key) use ($envs) { + $envs->push($key.'='.$item); + }); + + // Add SERVICE_NAME variables for Docker Compose builds + if ($this->build_pack === 'dockercompose') { + if ($this->pull_request_id === 0) { + // Generate SERVICE_NAME for dockercompose services from processed compose + if ($this->application->settings->is_raw_compose_deployment_enabled) { + $dockerCompose = Yaml::parse($this->application->docker_compose_raw); + } else { + $dockerCompose = Yaml::parse($this->application->docker_compose); + } + $services = data_get($dockerCompose, 'services', []); + foreach ($services as $serviceName => $_) { + $envs->push('SERVICE_NAME_'.str($serviceName)->upper().'='.$serviceName); + } + + // Generate SERVICE_FQDN & SERVICE_URL for non-PR deployments + $domains = collect(json_decode($this->application->docker_compose_domains)) ?? collect([]); + foreach ($domains as $forServiceName => $domain) { + $parsedDomain = data_get($domain, 'domain'); + if (filled($parsedDomain)) { + $parsedDomain = str($parsedDomain)->explode(',')->first(); + $coolifyUrl = Url::fromString($parsedDomain); + $coolifyScheme = $coolifyUrl->getScheme(); + $coolifyFqdn = $coolifyUrl->getHost(); + $coolifyUrl = $coolifyUrl->withScheme($coolifyScheme)->withHost($coolifyFqdn)->withPort(null); + $envs->push('SERVICE_URL_'.str($forServiceName)->upper().'='.$coolifyUrl->__toString()); + $envs->push('SERVICE_FQDN_'.str($forServiceName)->upper().'='.$coolifyFqdn); + } + } + } else { + // Generate SERVICE_NAME for preview deployments + $rawDockerCompose = Yaml::parse($this->application->docker_compose_raw); + $rawServices = data_get($rawDockerCompose, 'services', []); + foreach ($rawServices as $rawServiceName => $_) { + $envs->push('SERVICE_NAME_'.str($rawServiceName)->upper().'='.addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id)); + } + + // Generate SERVICE_FQDN & SERVICE_URL for preview deployments with PR-specific domains + $domains = collect(json_decode(data_get($this->preview, 'docker_compose_domains'))) ?? collect([]); + foreach ($domains as $forServiceName => $domain) { + $parsedDomain = data_get($domain, 'domain'); + if (filled($parsedDomain)) { + $parsedDomain = str($parsedDomain)->explode(',')->first(); + $coolifyUrl = Url::fromString($parsedDomain); + $coolifyScheme = $coolifyUrl->getScheme(); + $coolifyFqdn = $coolifyUrl->getHost(); + $coolifyUrl = $coolifyUrl->withScheme($coolifyScheme)->withHost($coolifyFqdn)->withPort(null); + $envs->push('SERVICE_URL_'.str($forServiceName)->upper().'='.$coolifyUrl->__toString()); + $envs->push('SERVICE_FQDN_'.str($forServiceName)->upper().'='.$coolifyFqdn); + } + } + } + } + + // Add build-time user variables only + if ($this->pull_request_id === 0) { + $sorted_environment_variables = $this->application->environment_variables() + ->where('key', 'not like', 'NIXPACKS_%') + ->where('is_buildtime', true) // ONLY build-time variables + ->orderBy($this->application->settings->is_env_sorting_enabled ? 'key' : 'id') + ->get(); + + // For Docker Compose, filter out SERVICE_FQDN and SERVICE_URL as we generate these + if ($this->build_pack === 'dockercompose') { + $sorted_environment_variables = $sorted_environment_variables->filter(function ($env) { + return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_'); + }); + } + + foreach ($sorted_environment_variables as $env) { + $envs->push($env->key.'='.$env->real_value); + } + } else { + $sorted_environment_variables = $this->application->environment_variables_preview() + ->where('key', 'not like', 'NIXPACKS_%') + ->where('is_buildtime', true) // ONLY build-time variables + ->orderBy($this->application->settings->is_env_sorting_enabled ? 'key' : 'id') + ->get(); + + // For Docker Compose, filter out SERVICE_FQDN and SERVICE_URL as we generate these with PR-specific values + if ($this->build_pack === 'dockercompose') { + $sorted_environment_variables = $sorted_environment_variables->filter(function ($env) { + return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_'); + }); + } + + foreach ($sorted_environment_variables as $env) { + $envs->push($env->key.'='.$env->real_value); + } + } + + // Return the generated environment variables + return $envs; + } + + private function save_buildtime_environment_variables() + { + // Generate build-time environment variables locally + $environment_variables = $this->generate_buildtime_environment_variables(); + + // Save .env file for build phase + if ($environment_variables->isNotEmpty()) { + $envs_base64 = base64_encode($environment_variables->implode("\n")); + + $this->application_deployment_queue->addLogEntry('Creating .env file with build-time variables for build phase.', hidden: true); + + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "echo '$envs_base64' | base64 -d | tee $this->workdir/.env > /dev/null"), + ], + [ + executeInDocker($this->deployment_uuid, "cat $this->workdir/.env"), + 'hidden' => true, ], ); + } elseif ($this->build_pack === 'dockercompose') { + // For Docker Compose, create an empty .env file even if there are no build-time variables + // This ensures the file exists when referenced in docker-compose commands + $this->application_deployment_queue->addLogEntry('Creating empty .env file for build phase (no build-time variables defined).', hidden: true); - // Write .env file to configuration directory - if ($this->use_build_server) { - $this->server = $this->original_server; - $this->execute_remote_command( - [ - "echo '$envs_base64' | base64 -d | tee $this->configuration_dir/{$this->env_filename} > /dev/null", - ] - ); - $this->server = $this->build_server; - } else { - $this->execute_remote_command( - [ - "echo '$envs_base64' | base64 -d | tee $this->configuration_dir/{$this->env_filename} > /dev/null", - ] - ); - } + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "touch $this->workdir/.env"), + ] + ); } } @@ -1483,15 +1648,18 @@ private function deploy_pull_request() $this->generate_nixpacks_confs(); } $this->generate_compose_file(); + + // Save build-time .env file BEFORE the build + $this->save_buildtime_environment_variables(); + $this->generate_build_env_variables(); if ($this->application->build_pack === 'dockerfile') { $this->add_build_env_variables_to_dockerfile(); } $this->build_image(); - // For Nixpacks, save runtime environment variables AFTER the build - if ($this->application->build_pack === 'nixpacks') { - $this->save_runtime_environment_variables(); - } + + // This overwrites the build-time .env with ALL variables (build-time + runtime) + $this->save_runtime_environment_variables(); $this->push_to_docker_registry(); $this->rolling_update(); } @@ -2005,7 +2173,6 @@ private function generate_compose_file() $persistent_storages = $this->generate_local_persistent_volumes(); $persistent_file_volumes = $this->application->fileStorages()->get(); $volume_names = $this->generate_local_persistent_volumes_only_volume_names(); - $this->generate_runtime_environment_variables(); if (data_get($this->application, 'custom_labels')) { $this->application->parseContainerLabels(); $labels = collect(preg_split("/\r\n|\n|\r/", base64_decode($this->application->custom_labels))); @@ -2074,9 +2241,8 @@ private function generate_compose_file() ], ], ]; - if (filled($this->env_filename)) { - $docker_compose['services'][$this->container_name]['env_file'] = [$this->env_filename]; - } + // Always use .env file + $docker_compose['services'][$this->container_name]['env_file'] = ['.env']; $docker_compose['services'][$this->container_name]['healthcheck'] = [ 'test' => [ 'CMD-SHELL', @@ -2368,13 +2534,12 @@ private function build_image() // Coolify variables are already included in the secrets from generate_build_env_variables // build_secrets is already a string at this point } else { - // Traditional build args approach - $this->environment_variables->filter(function ($key, $value) { - return str($key)->startsWith('COOLIFY_'); - })->each(function ($key, $value) { + // Traditional build args approach - generate COOLIFY_ variables locally + // Generate COOLIFY_ variables locally for build args + $coolify_envs = $this->generate_coolify_env_variables(); + $coolify_envs->each(function ($value, $key) { $this->build_args->push("--build-arg '{$key}'"); }); - $this->build_args = $this->build_args instanceof \Illuminate\Support\Collection ? $this->build_args->implode(' ') : (string) $this->build_args; @@ -2904,6 +3069,7 @@ private function add_build_env_variables_to_dockerfile() executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->dockerfile_location}"), 'hidden' => true, 'save' => 'dockerfile', + 'ignore_errors' => true, ]); $dockerfile = collect(str($this->saved_outputs->get('dockerfile'))->trim()->explode("\n")); if ($this->pull_request_id === 0) { @@ -2962,10 +3128,17 @@ private function add_build_env_variables_to_dockerfile() } $dockerfile_base64 = base64_encode($dockerfile->implode("\n")); - $this->execute_remote_command([ - executeInDocker($this->deployment_uuid, "echo '{$dockerfile_base64}' | base64 -d | tee {$this->workdir}{$this->dockerfile_location} > /dev/null"), - 'hidden' => true, - ]); + $this->application_deployment_queue->addLogEntry('Final Dockerfile:', type: 'info'); + $this->execute_remote_command( + [ + executeInDocker($this->deployment_uuid, "echo '{$dockerfile_base64}' | base64 -d | tee {$this->workdir}{$this->dockerfile_location} > /dev/null"), + 'hidden' => true, + ], + [ + executeInDocker($this->deployment_uuid, "cat {$this->workdir}{$this->dockerfile_location}"), + 'hidden' => true, + 'ignore_errors' => true, + ]); } } @@ -3123,7 +3296,6 @@ private function modify_dockerfiles_for_compose($composeFile) $argsToAdd->push("ARG {$key}"); } - ray($argsToAdd); if ($argsToAdd->isEmpty()) { $this->application_deployment_queue->addLogEntry("Service {$serviceName}: No build-time variables to add.");