From 2216832f67efef68b2af1649bf03101dc8c2ff01 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Fri, 3 Oct 2025 09:20:05 +0200 Subject: [PATCH] fix(deployments): enhance builder container management and environment variable handling - Added a new method to restart the builder container with the actual commit value, ensuring accurate deployment. - Improved the generation of environment variables by consolidating user-defined and Coolify-specific variables. - Updated Dockerfile modification logic to handle environment variables more effectively, including support for multiline variables. - Enhanced logging for better visibility during deployment processes. --- app/Jobs/ApplicationDeploymentJob.php | 116 +++++++++++++++++--------- 1 file changed, 76 insertions(+), 40 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 04b11b9b4..965eab68e 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -1526,7 +1526,6 @@ private function prepare_builder_image() $this->dockerConfigFileExists = instant_remote_process(["test -f {$this->serverUserHomeDir}/.docker/config.json && echo 'OK' || echo 'NOK'"], $this->server); $env_flags = $this->generate_docker_env_flags_for_secrets(); - if ($this->use_build_server) { if ($this->dockerConfigFileExists === 'NOK') { throw new RuntimeException('Docker config file (~/.docker/config.json) not found on the build server. Please run "docker login" to login to the docker registry on the server.'); @@ -1553,6 +1552,18 @@ private function prepare_builder_image() $this->run_pre_deployment_command(); } + private function restart_builder_container_with_actual_commit() + { + // Stop and remove the current helper container + $this->graceful_shutdown_container($this->deployment_uuid); + + // Clear cached env_args to force regeneration with actual SOURCE_COMMIT value + $this->env_args = null; + + // Restart the helper container with updated environment variables (including actual SOURCE_COMMIT) + $this->prepare_builder_image(); + } + private function deploy_to_additional_destinations() { if ($this->application->additional_networks->count() === 0) { @@ -1621,6 +1632,8 @@ private function set_coolify_variables() if (isset($this->application->git_branch)) { $this->coolify_variables .= "COOLIFY_BRANCH={$this->application->git_branch} "; } + $this->coolify_variables .= "COOLIFY_RESOURCE_UUID={$this->application->uuid} "; + $this->coolify_variables .= "COOLIFY_CONTAINER_NAME={$this->container_name} "; } private function check_git_if_build_needed() @@ -1688,6 +1701,12 @@ private function check_git_if_build_needed() $this->application_deployment_queue->save(); } $this->set_coolify_variables(); + + // Restart helper container with actual SOURCE_COMMIT value + if ($this->application->settings->use_build_secrets && $this->commit !== 'HEAD') { + $this->application_deployment_queue->addLogEntry('Restarting helper container with actual SOURCE_COMMIT value.'); + $this->restart_builder_container_with_actual_commit(); + } } private function clone_repository() @@ -1936,7 +1955,6 @@ private function generate_env_variables() $this->env_args = collect([]); $this->env_args->put('SOURCE_COMMIT', $this->commit); $coolify_envs = $this->generate_coolify_env_variables(); - // For build process, include only environment variables where is_buildtime = true if ($this->pull_request_id === 0) { // Get environment variables that are marked as available during build @@ -1991,6 +2009,12 @@ private function generate_env_variables() } } } + + // Merge COOLIFY_* variables into env_args for build process + // This ensures they're available for both build args and build secrets + $coolify_envs->each(function ($value, $key) { + $this->env_args->put($key, $value); + }); } private function generate_compose_file() @@ -2610,6 +2634,8 @@ private function build_image() } else { // Dockerfile buildpack if ($this->dockerBuildkitSupported) { + // Modify the Dockerfile to use build secrets + $this->modify_dockerfile_for_secrets("{$this->workdir}{$this->dockerfile_location}"); // Use BuildKit with secrets $secrets_flags = $this->build_secrets ? " {$this->build_secrets}" : ''; if ($this->force_rebuild) { @@ -2764,7 +2790,6 @@ private function generate_build_env_variables() $this->generate_env_variables(); $variables = collect([])->merge($this->env_args); } - // Analyze build variables for potential issues if ($variables->isNotEmpty()) { $this->analyzeBuildTimeVariables($variables); @@ -2809,9 +2834,13 @@ private function generate_docker_env_flags_for_secrets() return ''; } - $variables = $this->pull_request_id === 0 - ? $this->application->environment_variables()->where('key', 'not like', 'NIXPACKS_%')->where('is_buildtime', true)->get() - : $this->application->environment_variables_preview()->where('key', 'not like', 'NIXPACKS_%')->where('is_buildtime', true)->get(); + // Generate env variables if not already done + // This populates $this->env_args with both user-defined and COOLIFY_* variables + if (! $this->env_args || $this->env_args->isEmpty()) { + $this->generate_env_variables(); + } + + $variables = $this->env_args; if ($variables->isEmpty()) { return ''; @@ -2819,12 +2848,19 @@ private function generate_docker_env_flags_for_secrets() $secrets_hash = $this->generate_secrets_hash($variables); + // Get database env vars to check for multiline flag + $env_vars = $this->pull_request_id === 0 + ? $this->application->environment_variables()->where('is_buildtime', true)->get() + : $this->application->environment_variables_preview()->where('is_buildtime', true)->get(); + // Map to simple array format for the helper function - $vars_array = $variables->map(function ($env) { + $vars_array = $variables->map(function ($value, $key) use ($env_vars) { + $env = $env_vars->firstWhere('key', $key); + return [ - 'key' => $env->key, - 'value' => $env->real_value, - 'is_multiline' => $env->is_multiline, + 'key' => $key, + 'value' => $value, + 'is_multiline' => $env ? $env->is_multiline : false, ]; }); @@ -2890,7 +2926,6 @@ private function add_build_env_variables_to_dockerfile() 'save' => 'dockerfile', ]); $dockerfile = collect(str($this->saved_outputs->get('dockerfile'))->trim()->explode("\n")); - if ($this->pull_request_id === 0) { // Only add environment variables that are available during build $envs = $this->application->environment_variables() @@ -2975,16 +3010,19 @@ private function modify_dockerfile_for_secrets($dockerfile_path) $dockerfile->prepend('# syntax=docker/dockerfile:1'); } - // Get environment variables for secrets - $variables = $this->pull_request_id === 0 - ? $this->application->environment_variables()->where('key', 'not like', 'NIXPACKS_%')->where('is_buildtime', true)->get() - : $this->application->environment_variables_preview()->where('key', 'not like', 'NIXPACKS_%')->where('is_buildtime', true)->get(); + // Generate env variables if not already done + // This populates $this->env_args with both user-defined and COOLIFY_* variables + if (! $this->env_args || $this->env_args->isEmpty()) { + $this->generate_env_variables(); + } + + $variables = $this->env_args; if ($variables->isEmpty()) { return; } // Generate mount strings for all secrets - $mountStrings = $variables->map(fn ($env) => "--mount=type=secret,id={$env->key},env={$env->key}")->implode(' '); + $mountStrings = $variables->map(fn ($value, $key) => "--mount=type=secret,id={$key},env={$key}")->implode(' '); // Add mount for the secrets hash to ensure cache invalidation $mountStrings .= ' --mount=type=secret,id=COOLIFY_BUILD_SECRETS_HASH,env=COOLIFY_BUILD_SECRETS_HASH'; @@ -3012,8 +3050,6 @@ private function modify_dockerfile_for_secrets($dockerfile_path) executeInDocker($this->deployment_uuid, "echo '{$dockerfile_base64}' | base64 -d | tee {$dockerfile_path} > /dev/null"), 'hidden' => true, ]); - - $this->application_deployment_queue->addLogEntry('Modified Dockerfile to use build secrets.'); } } @@ -3023,15 +3059,13 @@ private function modify_dockerfiles_for_compose($composeFile) return; } - $variables = $this->pull_request_id === 0 - ? $this->application->environment_variables() - ->where('key', 'not like', 'NIXPACKS_%') - ->where('is_buildtime', true) - ->get() - : $this->application->environment_variables_preview() - ->where('key', 'not like', 'NIXPACKS_%') - ->where('is_buildtime', true) - ->get(); + // Generate env variables if not already done + // This populates $this->env_args with both user-defined and COOLIFY_* variables + if (! $this->env_args || $this->env_args->isEmpty()) { + $this->generate_env_variables(); + } + + $variables = $this->env_args; if ($variables->isEmpty()) { $this->application_deployment_queue->addLogEntry('No build-time variables to add to Dockerfiles.'); @@ -3105,8 +3139,8 @@ private function modify_dockerfiles_for_compose($composeFile) $isMultiStage = count($fromIndices) > 1; $argsToAdd = collect([]); - foreach ($variables as $env) { - $argsToAdd->push("ARG {$env->key}"); + foreach ($variables as $key => $value) { + $argsToAdd->push("ARG {$key}"); } ray($argsToAdd); @@ -3177,19 +3211,22 @@ private function modify_dockerfiles_for_compose($composeFile) private function add_build_secrets_to_compose($composeFile) { - // Get environment variables for secrets - $variables = $this->pull_request_id === 0 - ? $this->application->environment_variables()->where('key', 'not like', 'NIXPACKS_%')->get() - : $this->application->environment_variables_preview()->where('key', 'not like', 'NIXPACKS_%')->get(); + // Generate env variables if not already done + // This populates $this->env_args with both user-defined and COOLIFY_* variables + if (! $this->env_args || $this->env_args->isEmpty()) { + $this->generate_env_variables(); + } + + $variables = $this->env_args; if ($variables->isEmpty()) { return $composeFile; } $secrets = []; - foreach ($variables as $env) { - $secrets[$env->key] = [ - 'environment' => $env->key, + foreach ($variables as $key => $value) { + $secrets[$key] = [ + 'environment' => $key, ]; } @@ -3204,9 +3241,9 @@ private function add_build_secrets_to_compose($composeFile) if (! isset($service['build']['secrets'])) { $service['build']['secrets'] = []; } - foreach ($variables as $env) { - if (! in_array($env->key, $service['build']['secrets'])) { - $service['build']['secrets'][] = $env->key; + foreach ($variables as $key => $value) { + if (! in_array($key, $service['build']['secrets'])) { + $service['build']['secrets'][] = $key; } } } @@ -3325,7 +3362,6 @@ private function next(string $status) queue_next_deployment($this->application); if ($status === ApplicationDeploymentStatus::FINISHED->value) { - ray($this->application->team()->id); event(new ApplicationConfigurationChanged($this->application->team()->id)); if (! $this->only_this_server) {