From e23ab1e621d57d37c53c3176d6af3fb0b819dea4 Mon Sep 17 00:00:00 2001 From: Arnaud B Date: Mon, 8 Sep 2025 15:15:57 +0200 Subject: [PATCH 01/50] feat(deployment): add SERVICE_NAME variables for service discovery This change introduces automatically generated `SERVICE_NAME_` environment variables for each service within a Docker Compose deployment. This allows services to reliably reference each other by name, which is particularly useful in pull request environments where container names are dynamically suffixed. - The application parser now generates and injects these `SERVICE_NAME` variables into the environment of all services in the compose file. - `ApplicationDeploymentJob` is updated to correctly handle and filter these new variables during deployment. - UI components and the `EnvironmentVariableProtection` trait have been updated to make these generated variables read-only, preventing accidental modification. This commit introduces two new helper functions to standardize resource naming for pull request deployments: - `addPreviewDeploymentSuffix()`: Generates a consistent suffix format (-pr-{id}) for resource names in preview deployments - `generateDockerComposeServiceName()`: Creates SERVICE_NAME environment variables for Docker Compose services --- app/Jobs/ApplicationDeploymentJob.php | 36 ++++++++++++++----- .../Shared/EnvironmentVariable/All.php | 2 +- .../Shared/EnvironmentVariable/Show.php | 2 +- app/Traits/EnvironmentVariableProtection.php | 2 +- bootstrap/helpers/parsers.php | 18 ++++++---- bootstrap/helpers/shared.php | 36 +++++++++++++------ 6 files changed, 67 insertions(+), 29 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index d77adebb9..c4a9bb2cd 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -221,7 +221,7 @@ public function __construct(public int $application_deployment_queue_id) if ($this->pull_request_id === 0) { $this->container_name = $this->application->settings->custom_internal_name; } else { - $this->container_name = "{$this->application->settings->custom_internal_name}-pr-{$this->pull_request_id}"; + $this->container_name = addPreviewDeploymentSuffix($this->application->settings->custom_internal_name, $this->pull_request_id); } } @@ -706,8 +706,8 @@ private function write_deployment_configurations() if ($this->pull_request_id === 0) { $composeFileName = "$mainDir/docker-compose.yaml"; } else { - $composeFileName = "$mainDir/docker-compose-pr-{$this->pull_request_id}.yaml"; - $this->docker_compose_location = "/docker-compose-pr-{$this->pull_request_id}.yaml"; + $composeFileName = "$mainDir/".addPreviewDeploymentSuffix('docker-compose', $this->pull_request_id).'.yaml'; + $this->docker_compose_location = '/'.addPreviewDeploymentSuffix('docker-compose', $this->pull_request_id).'.yaml'; } $this->execute_remote_command([ "mkdir -p $mainDir", @@ -898,10 +898,10 @@ private function save_environment_variables() } if ($this->build_pack === 'dockercompose') { $sorted_environment_variables = $sorted_environment_variables->filter(function ($env) { - return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_'); + return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_') && ! str($env->key)->startsWith('SERVICE_NAME_'); }); $sorted_environment_variables_preview = $sorted_environment_variables_preview->filter(function ($env) { - return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_'); + return ! str($env->key)->startsWith('SERVICE_FQDN_') && ! str($env->key)->startsWith('SERVICE_URL_') && ! str($env->key)->startsWith('SERVICE_NAME_'); }); } $ports = $this->application->main_port(); @@ -942,9 +942,20 @@ private function save_environment_variables() $envs->push('SERVICE_FQDN_'.str($forServiceName)->upper().'='.$coolifyFqdn); } } + + // Generate SERVICE_NAME for dockercompose services from processed compose + if ($this->application->settings->is_raw_compose_deployment_enabled) { + $dockerCompose = Yaml::parse($this->application->docker_compose_raw); + } else { + $dockerCompose = Yaml::parse($this->application->docker_compose); + } + $services = data_get($dockerCompose, 'services', []); + foreach ($services as $serviceName => $_) { + $envs->push('SERVICE_NAME_'.str($serviceName)->upper().'='.$serviceName); + } } } else { - $this->env_filename = ".env-pr-$this->pull_request_id"; + $this->env_filename = addPreviewDeploymentSuffix(".env", $this->pull_request_id); foreach ($sorted_environment_variables_preview as $env) { $envs->push($env->key.'='.$env->real_value); } @@ -975,6 +986,13 @@ private function save_environment_variables() $envs->push('SERVICE_FQDN_'.str($forServiceName)->upper().'='.$coolifyFqdn); } } + + // Generate SERVICE_NAME for dockercompose services + $rawDockerCompose = Yaml::parse($this->application->docker_compose_raw); + $rawServices = data_get($rawDockerCompose, 'services', []); + foreach ($rawServices as $rawServiceName => $_) { + $envs->push('SERVICE_NAME_'.str($rawServiceName)->upper().'='.addPreviewDeploymentSuffix($rawServiceName, $this->pull_request_id)); + } } } if ($envs->isEmpty()) { @@ -1986,7 +2004,7 @@ private function generate_local_persistent_volumes() $volume_name = $persistentStorage->name; } if ($this->pull_request_id !== 0) { - $volume_name = $volume_name.'-pr-'.$this->pull_request_id; + $volume_name = addPreviewDeploymentSuffix($volume_name, $this->pull_request_id); } $local_persistent_volumes[] = $volume_name.':'.$persistentStorage->mount_path; } @@ -2004,7 +2022,7 @@ private function generate_local_persistent_volumes_only_volume_names() $name = $persistentStorage->name; if ($this->pull_request_id !== 0) { - $name = $name.'-pr-'.$this->pull_request_id; + $name = addPreviewDeploymentSuffix($name, $this->pull_request_id); } $local_persistent_volumes_names[$name] = [ @@ -2301,7 +2319,7 @@ private function stop_running_container(bool $force = false) $containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id); if ($this->pull_request_id === 0) { $containers = $containers->filter(function ($container) { - return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== $this->container_name.'-pr-'.$this->pull_request_id; + return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== addPreviewDeploymentSuffix($this->container_name, $this->pull_request_id); }); } $containers->each(function ($container) { diff --git a/app/Livewire/Project/Shared/EnvironmentVariable/All.php b/app/Livewire/Project/Shared/EnvironmentVariable/All.php index 3631a43c8..141263ba2 100644 --- a/app/Livewire/Project/Shared/EnvironmentVariable/All.php +++ b/app/Livewire/Project/Shared/EnvironmentVariable/All.php @@ -257,7 +257,7 @@ private function updateOrCreateVariables($isPreview, $variables) { $count = 0; foreach ($variables as $key => $value) { - if (str($key)->startsWith('SERVICE_FQDN') || str($key)->startsWith('SERVICE_URL')) { + if (str($key)->startsWith('SERVICE_FQDN') || str($key)->startsWith('SERVICE_URL') || str($key)->startsWith('SERVICE_NAME')) { continue; } $method = $isPreview ? 'environment_variables_preview' : 'environment_variables'; diff --git a/app/Livewire/Project/Shared/EnvironmentVariable/Show.php b/app/Livewire/Project/Shared/EnvironmentVariable/Show.php index 1a9daf77b..f8b06bff8 100644 --- a/app/Livewire/Project/Shared/EnvironmentVariable/Show.php +++ b/app/Livewire/Project/Shared/EnvironmentVariable/Show.php @@ -128,7 +128,7 @@ public function syncData(bool $toModel = false) public function checkEnvs() { $this->isDisabled = false; - if (str($this->env->key)->startsWith('SERVICE_FQDN') || str($this->env->key)->startsWith('SERVICE_URL')) { + if (str($this->env->key)->startsWith('SERVICE_FQDN') || str($this->env->key)->startsWith('SERVICE_URL') || str($this->env->key)->startsWith('SERVICE_NAME')) { $this->isDisabled = true; } if ($this->env->is_shown_once) { diff --git a/app/Traits/EnvironmentVariableProtection.php b/app/Traits/EnvironmentVariableProtection.php index b6b8d2687..ecc484966 100644 --- a/app/Traits/EnvironmentVariableProtection.php +++ b/app/Traits/EnvironmentVariableProtection.php @@ -14,7 +14,7 @@ trait EnvironmentVariableProtection */ protected function isProtectedEnvironmentVariable(string $key): bool { - return str($key)->startsWith('SERVICE_FQDN') || str($key)->startsWith('SERVICE_URL'); + return str($key)->startsWith('SERVICE_FQDN_') || str($key)->startsWith('SERVICE_URL_') || str($key)->startsWith('SERVICE_NAME_'); } /** diff --git a/bootstrap/helpers/parsers.php b/bootstrap/helpers/parsers.php index f7041c3da..f162039a2 100644 --- a/bootstrap/helpers/parsers.php +++ b/bootstrap/helpers/parsers.php @@ -454,6 +454,12 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int } } + // generate SERVICE_NAME variables for docker compose services + $serviceNameEnvironments = collect([]); + if ($resource->build_pack === 'dockercompose') { + $serviceNameEnvironments = generateDockerComposeServiceName($services, $pullRequestId); + } + // Parse the rest of the services foreach ($services as $serviceName => $service) { $image = data_get_str($service, 'image'); @@ -567,7 +573,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int } $source = replaceLocalSource($source, $mainDirectory); if ($isPullRequest) { - $source = $source."-pr-$pullRequestId"; + $source = addPreviewDeploymentSuffix($source, $pull_request_id); } LocalFileVolume::updateOrCreate( [ @@ -610,7 +616,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int $name = "{$uuid}_{$slugWithoutUuid}"; if ($isPullRequest) { - $name = "{$name}-pr-$pullRequestId"; + $name = addPreviewDeploymentSuffix($name, $pull_request_id); } if (is_string($volume)) { $parsed = parseDockerVolumeString($volume); @@ -651,11 +657,11 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int $newDependsOn = collect([]); $depends_on->each(function ($dependency, $condition) use ($pullRequestId, $newDependsOn) { if (is_numeric($condition)) { - $dependency = "$dependency-pr-$pullRequestId"; + $dependency = addPreviewDeploymentSuffix($dependency, $pullRequestId); $newDependsOn->put($condition, $dependency); } else { - $condition = "$condition-pr-$pullRequestId"; + $condition = addPreviewDeploymentSuffix($condition, $pullRequestId); $newDependsOn->put($condition, $dependency); } }); @@ -1082,7 +1088,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int $payload['volumes'] = $volumesParsed; } if ($environment->count() > 0 || $coolifyEnvironments->count() > 0) { - $payload['environment'] = $environment->merge($coolifyEnvironments); + $payload['environment'] = $environment->merge($coolifyEnvironments)->merge($serviceNameEnvironments); } if ($logging) { $payload['logging'] = $logging; @@ -1091,7 +1097,7 @@ function applicationParser(Application $resource, int $pull_request_id = 0, ?int $payload['depends_on'] = $depends_on; } if ($isPullRequest) { - $serviceName = "{$serviceName}-pr-{$pullRequestId}"; + $serviceName = addPreviewDeploymentSuffix($serviceName, $pullRequestId); } $parsedServices->put($serviceName, $payload); diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 9c30282b4..a3127e880 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -2058,12 +2058,12 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $name = $name->replaceFirst('~', $dir); } if ($pull_request_id !== 0) { - $name = $name."-pr-$pull_request_id"; + $name = addPreviewDeploymentSuffix($name, $pull_request_id); } $volume = str("$name:$mount"); } else { if ($pull_request_id !== 0) { - $name = $name."-pr-$pull_request_id"; + $name = addPreviewDeploymentSuffix($name, $pull_request_id); $volume = str("$name:$mount"); if ($topLevelVolumes->has($name)) { $v = $topLevelVolumes->get($name); @@ -2102,7 +2102,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $name = $volume->before(':'); $mount = $volume->after(':'); if ($pull_request_id !== 0) { - $name = $name."-pr-$pull_request_id"; + $name = addPreviewDeploymentSuffix($name, $pull_request_id); } $volume = str("$name:$mount"); } @@ -2121,7 +2121,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $source = str($source)->replaceFirst('~', $dir); } if ($pull_request_id !== 0) { - $source = $source."-pr-$pull_request_id"; + $source = addPreviewDeploymentSuffix($source, $pull_request_id); } if ($read_only) { data_set($volume, 'source', $source.':'.$target.':ro'); @@ -2130,7 +2130,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal } } else { if ($pull_request_id !== 0) { - $source = $source."-pr-$pull_request_id"; + $source = addPreviewDeploymentSuffix($source, $pull_request_id); } if ($read_only) { data_set($volume, 'source', $source.':'.$target.':ro'); @@ -2182,13 +2182,13 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $name = $name->replaceFirst('~', $dir); } if ($pull_request_id !== 0) { - $name = $name."-pr-$pull_request_id"; + $name = addPreviewDeploymentSuffix($name, $pull_request_id); } $volume = str("$name:$mount"); } else { if ($pull_request_id !== 0) { $uuid = $resource->uuid; - $name = $uuid."-$name-pr-$pull_request_id"; + $name = $uuid.'-'.addPreviewDeploymentSuffix($name, $pull_request_id); $volume = str("$name:$mount"); if ($topLevelVolumes->has($name)) { $v = $topLevelVolumes->get($name); @@ -2230,7 +2230,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal $name = $volume->before(':'); $mount = $volume->after(':'); if ($pull_request_id !== 0) { - $name = $name."-pr-$pull_request_id"; + $name = addPreviewDeploymentSuffix($name, $pull_request_id); } $volume = str("$name:$mount"); } @@ -2258,7 +2258,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal if ($pull_request_id === 0) { $source = $uuid."-$source"; } else { - $source = $uuid."-$source-pr-$pull_request_id"; + $source = $uuid.'-'.addPreviewDeploymentSuffix($source, $pull_request_id); } if ($read_only) { data_set($volume, 'source', $source.':'.$target.':ro'); @@ -2298,7 +2298,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal if ($pull_request_id !== 0 && count($serviceDependencies) > 0) { $serviceDependencies = $serviceDependencies->map(function ($dependency) use ($pull_request_id) { - return $dependency."-pr-$pull_request_id"; + return addPreviewDeploymentSuffix($dependency, $pull_request_id); }); data_set($service, 'depends_on', $serviceDependencies->toArray()); } @@ -2692,7 +2692,7 @@ function parseDockerComposeFile(Service|Application $resource, bool $isNew = fal }); if ($pull_request_id !== 0) { $services->each(function ($service, $serviceName) use ($pull_request_id, $services) { - $services[$serviceName."-pr-$pull_request_id"] = $service; + $services[addPreviewDeploymentSuffix($serviceName, $pull_request_id)] = $service; data_forget($services, $serviceName); }); } @@ -3072,3 +3072,17 @@ function parseDockerfileInterval(string $something) return $seconds; } + +function addPreviewDeploymentSuffix(string $name, int $pull_request_id = 0): string +{ + return ($pull_request_id === 0)? $name : $name.'-pr-'.$pull_request_id; +} + +function generateDockerComposeServiceName(mixed $services, int $pullRequestId = 0) : Collection +{ + $collection = collect([]); + foreach ($services as $serviceName => $_) { + $collection->put('SERVICE_NAME_'.str($serviceName)->upper(), addPreviewDeploymentSuffix($serviceName,$pullRequestId)); + } + return $collection; +} From 8f2a45b8dcd620e9e40a1ca0d216b9c935431433 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 9 Sep 2025 16:29:04 +0200 Subject: [PATCH 02/50] docs(testing-patterns): add important note to always run tests inside the `coolify` container for clarity --- .cursor/rules/testing-patterns.mdc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cursor/rules/testing-patterns.mdc b/.cursor/rules/testing-patterns.mdc index 010b76544..a0e64dbae 100644 --- a/.cursor/rules/testing-patterns.mdc +++ b/.cursor/rules/testing-patterns.mdc @@ -9,6 +9,8 @@ alwaysApply: false Coolify employs **comprehensive testing strategies** using modern PHP testing frameworks to ensure reliability of deployment operations, infrastructure management, and user interactions. +!Important: Always run tests inside `coolify` container. + ## Testing Framework Stack ### Core Testing Tools From a06c79776eb829f73639f06c46f91a8191225e2c Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 9 Sep 2025 16:46:24 +0200 Subject: [PATCH 03/50] feat(dev-command): dispatch CheckHelperImageJob during instance initialization to enhance setup process --- app/Console/Commands/Dev.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/Console/Commands/Dev.php b/app/Console/Commands/Dev.php index a4cfde6f8..8f26d78ff 100644 --- a/app/Console/Commands/Dev.php +++ b/app/Console/Commands/Dev.php @@ -2,6 +2,7 @@ namespace App\Console\Commands; +use App\Jobs\CheckHelperImageJob; use App\Models\InstanceSettings; use Illuminate\Console\Command; use Illuminate\Support\Facades\Artisan; @@ -44,5 +45,6 @@ public function init() } else { echo "Instance already initialized.\n"; } + CheckHelperImageJob::dispatch(); } } From a60d6dadc7c81d1beb01b1cf83e98778e72b9c34 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Tue, 9 Sep 2025 16:46:38 +0200 Subject: [PATCH 04/50] fix(private-key): implement transaction handling and error verification for private key storage operations --- app/Models/PrivateKey.php | 85 ++++++- tests/Unit/PrivateKeyStorageTest.php | 316 +++++++++++++++++++++++++++ 2 files changed, 391 insertions(+), 10 deletions(-) create mode 100644 tests/Unit/PrivateKeyStorageTest.php diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index f70f32bc4..851be6947 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -4,6 +4,7 @@ use App\Traits\HasSafeStringAttribute; use DanHarrin\LivewireRateLimiting\WithRateLimiting; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Storage; use Illuminate\Validation\ValidationException; use OpenApi\Attributes as OA; @@ -99,11 +100,18 @@ public static function validatePrivateKey($privateKey) public static function createAndStore(array $data) { - $privateKey = new self($data); - $privateKey->save(); - $privateKey->storeInFileSystem(); + return DB::transaction(function () use ($data) { + $privateKey = new self($data); + $privateKey->save(); - return $privateKey; + try { + $privateKey->storeInFileSystem(); + } catch (\Exception $e) { + throw new \Exception('Failed to store SSH key: '.$e->getMessage()); + } + + return $privateKey; + }); } public static function generateNewKeyPair($type = 'rsa') @@ -150,16 +158,66 @@ public static function validateAndExtractPublicKey($privateKey) public function storeInFileSystem() { + ray('storing private key in filesystem', $this->uuid); $filename = "ssh_key@{$this->uuid}"; - Storage::disk('ssh-keys')->put($filename, $this->private_key); + $disk = Storage::disk('ssh-keys'); - return "/var/www/html/storage/app/ssh/keys/{$filename}"; + // Ensure the storage directory exists and is writable + $this->ensureStorageDirectoryExists(); + + // Attempt to store the private key + $success = $disk->put($filename, $this->private_key); + + if (! $success) { + throw new \Exception("Failed to write SSH key to filesystem. Check disk space and permissions for: {$this->getKeyLocation()}"); + } + + // Verify the file was actually created and has content + if (! $disk->exists($filename)) { + throw new \Exception("SSH key file was not created: {$this->getKeyLocation()}"); + } + + $storedContent = $disk->get($filename); + if (empty($storedContent) || $storedContent !== $this->private_key) { + $disk->delete($filename); // Clean up the bad file + throw new \Exception("SSH key file content verification failed: {$this->getKeyLocation()}"); + } + + return $this->getKeyLocation(); } public static function deleteFromStorage(self $privateKey) { $filename = "ssh_key@{$privateKey->uuid}"; - Storage::disk('ssh-keys')->delete($filename); + $disk = Storage::disk('ssh-keys'); + + if ($disk->exists($filename)) { + $disk->delete($filename); + } + } + + protected function ensureStorageDirectoryExists() + { + $disk = Storage::disk('ssh-keys'); + $directoryPath = ''; + + if (! $disk->exists($directoryPath)) { + $success = $disk->makeDirectory($directoryPath); + if (! $success) { + throw new \Exception('Failed to create SSH keys storage directory'); + } + } + + // Check if directory is writable by attempting a test file + $testFilename = '.test_write_'.uniqid(); + $testSuccess = $disk->put($testFilename, 'test'); + + if (! $testSuccess) { + throw new \Exception('SSH keys storage directory is not writable'); + } + + // Clean up test file + $disk->delete($testFilename); } public function getKeyLocation() @@ -169,10 +227,17 @@ public function getKeyLocation() public function updatePrivateKey(array $data) { - $this->update($data); - $this->storeInFileSystem(); + return DB::transaction(function () use ($data) { + $this->update($data); - return $this; + try { + $this->storeInFileSystem(); + } catch (\Exception $e) { + throw new \Exception('Failed to update SSH key: '.$e->getMessage()); + } + + return $this; + }); } public function servers() diff --git a/tests/Unit/PrivateKeyStorageTest.php b/tests/Unit/PrivateKeyStorageTest.php new file mode 100644 index 000000000..00f39e3df --- /dev/null +++ b/tests/Unit/PrivateKeyStorageTest.php @@ -0,0 +1,316 @@ +actingAs(\App\Models\User::factory()->create()); + } + + protected function getValidPrivateKey(): string + { + return '-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevAAAAJi/QySHv0Mk +hwAAAAtzc2gtZWQyNTUxOQAAACBbhpqHhqv6aI67Mj9abM3DVbmcfYhZAhC7ca4d9UCevA +AAAECBQw4jg1WRT2IGHMncCiZhURCts2s24HoDS0thHnnRKVuGmoeGq/pojrsyP1pszcNV +uZx9iFkCELtxrh31QJ68AAAAEXNhaWxANzZmZjY2ZDJlMmRkAQIDBA== +-----END OPENSSH PRIVATE KEY-----'; + } + + /** @test */ + public function it_successfully_stores_private_key_in_filesystem() + { + Storage::fake('ssh-keys'); + + $privateKey = PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + + $this->assertDatabaseHas('private_keys', [ + 'id' => $privateKey->id, + 'name' => 'Test Key', + ]); + + $filename = "ssh_key@{$privateKey->uuid}"; + Storage::disk('ssh-keys')->assertExists($filename); + + $storedContent = Storage::disk('ssh-keys')->get($filename); + $this->assertEquals($privateKey->private_key, $storedContent); + } + + /** @test */ + public function it_throws_exception_when_storage_fails() + { + Storage::fake('ssh-keys'); + + // Mock Storage::put to return false (simulating storage failure) + Storage::shouldReceive('disk') + ->with('ssh-keys') + ->andReturn( + \Mockery::mock() + ->shouldReceive('exists') + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::any(), 'test') + ->andReturn(true) + ->shouldReceive('delete') + ->with(\Mockery::any()) + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::pattern('/ssh_key@/'), \Mockery::any()) + ->andReturn(false) // Simulate storage failure + ->getMock() + ); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('Failed to write SSH key to filesystem'); + + PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + + // Assert that no database record was created due to transaction rollback + $this->assertDatabaseMissing('private_keys', [ + 'name' => 'Test Key', + ]); + } + + /** @test */ + public function it_throws_exception_when_storage_directory_is_not_writable() + { + Storage::fake('ssh-keys'); + + // Mock Storage disk to simulate directory not writable + Storage::shouldReceive('disk') + ->with('ssh-keys') + ->andReturn( + \Mockery::mock() + ->shouldReceive('exists') + ->with('') + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::pattern('/\.test_write_/'), 'test') + ->andReturn(false) // Simulate directory not writable + ->getMock() + ); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('SSH keys storage directory is not writable'); + + PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + } + + /** @test */ + public function it_creates_storage_directory_if_not_exists() + { + Storage::fake('ssh-keys'); + + // Mock Storage disk to simulate directory not existing, then being created + Storage::shouldReceive('disk') + ->with('ssh-keys') + ->andReturn( + \Mockery::mock() + ->shouldReceive('exists') + ->with('') + ->andReturn(false) // Directory doesn't exist + ->shouldReceive('makeDirectory') + ->with('') + ->andReturn(true) // Successfully create directory + ->shouldReceive('put') + ->with(\Mockery::pattern('/\.test_write_/'), 'test') + ->andReturn(true) // Directory is writable after creation + ->shouldReceive('delete') + ->with(\Mockery::pattern('/\.test_write_/')) + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::pattern('/ssh_key@/'), \Mockery::any()) + ->andReturn(true) + ->shouldReceive('exists') + ->with(\Mockery::pattern('/ssh_key@/')) + ->andReturn(true) + ->shouldReceive('get') + ->with(\Mockery::pattern('/ssh_key@/')) + ->andReturn($this->getValidPrivateKey()) + ->getMock() + ); + + $privateKey = PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + + $this->assertDatabaseHas('private_keys', [ + 'id' => $privateKey->id, + 'name' => 'Test Key', + ]); + } + + /** @test */ + public function it_throws_exception_when_file_content_verification_fails() + { + Storage::fake('ssh-keys'); + + // Mock Storage disk to simulate file being created but with wrong content + Storage::shouldReceive('disk') + ->with('ssh-keys') + ->andReturn( + \Mockery::mock() + ->shouldReceive('exists') + ->with('') + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::pattern('/\.test_write_/'), 'test') + ->andReturn(true) + ->shouldReceive('delete') + ->with(\Mockery::pattern('/\.test_write_/')) + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::pattern('/ssh_key@/'), \Mockery::any()) + ->andReturn(true) // File created successfully + ->shouldReceive('exists') + ->with(\Mockery::pattern('/ssh_key@/')) + ->andReturn(true) // File exists + ->shouldReceive('get') + ->with(\Mockery::pattern('/ssh_key@/')) + ->andReturn('corrupted content') // But content is wrong + ->shouldReceive('delete') + ->with(\Mockery::pattern('/ssh_key@/')) + ->andReturn(true) // Clean up bad file + ->getMock() + ); + + $this->expectException(\Exception::class); + $this->expectExceptionMessage('SSH key file content verification failed'); + + PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + + // Assert that no database record was created due to transaction rollback + $this->assertDatabaseMissing('private_keys', [ + 'name' => 'Test Key', + ]); + } + + /** @test */ + public function it_successfully_deletes_private_key_from_filesystem() + { + Storage::fake('ssh-keys'); + + $privateKey = PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + + $filename = "ssh_key@{$privateKey->uuid}"; + Storage::disk('ssh-keys')->assertExists($filename); + + $privateKey->delete(); + + Storage::disk('ssh-keys')->assertMissing($filename); + } + + /** @test */ + public function it_handles_database_transaction_rollback_on_storage_failure() + { + Storage::fake('ssh-keys'); + + // Count initial private keys + $initialCount = PrivateKey::count(); + + // Mock storage failure after database save + Storage::shouldReceive('disk') + ->with('ssh-keys') + ->andReturn( + \Mockery::mock() + ->shouldReceive('exists') + ->with('') + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::pattern('/\.test_write_/'), 'test') + ->andReturn(true) + ->shouldReceive('delete') + ->with(\Mockery::pattern('/\.test_write_/')) + ->andReturn(true) + ->shouldReceive('put') + ->with(\Mockery::pattern('/ssh_key@/'), \Mockery::any()) + ->andReturn(false) // Storage fails + ->getMock() + ); + + try { + PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + } catch (\Exception $e) { + // Expected exception + } + + // Assert that database was rolled back + $this->assertEquals($initialCount, PrivateKey::count()); + $this->assertDatabaseMissing('private_keys', [ + 'name' => 'Test Key', + ]); + } + + /** @test */ + public function it_successfully_updates_private_key_with_transaction() + { + Storage::fake('ssh-keys'); + + $privateKey = PrivateKey::createAndStore([ + 'name' => 'Test Key', + 'description' => 'Test Description', + 'private_key' => $this->getValidPrivateKey(), + 'team_id' => currentTeam()->id, + ]); + + $newPrivateKey = str_replace('Test', 'Updated', $this->getValidPrivateKey()); + + $privateKey->updatePrivateKey([ + 'name' => 'Updated Key', + 'private_key' => $newPrivateKey, + ]); + + $this->assertDatabaseHas('private_keys', [ + 'id' => $privateKey->id, + 'name' => 'Updated Key', + ]); + + $filename = "ssh_key@{$privateKey->uuid}"; + $storedContent = Storage::disk('ssh-keys')->get($filename); + $this->assertEquals($newPrivateKey, $storedContent); + } +} From 13d52e0e9afb5e9ad9a4768e51fc804108c45bac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Sep 2025 02:01:32 +0000 Subject: [PATCH 05/50] chore(deps-dev): bump vite from 6.3.5 to 6.3.6 Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 6.3.5 to 6.3.6. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v6.3.6/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v6.3.6/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-version: 6.3.6 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 68 ++++++++++++++++++++++++++++++++++++++++++++--- package.json | 2 +- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 34b2c1dd5..56e48288c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "pusher-js": "8.4.0", "tailwind-scrollbar": "4.0.2", "tailwindcss": "4.1.10", - "vite": "6.3.5", + "vite": "6.3.6", "vue": "3.5.16" } }, @@ -1131,6 +1131,66 @@ "node": ">=14.0.0" } }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.4.3", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.0.2", + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.4.3", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.0.2", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.10", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.4.3", + "@emnapi/runtime": "^1.4.3", + "@tybys/wasm-util": "^0.9.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/@tybys/wasm-util": { + "version": "0.9.0", + "dev": true, + "inBundle": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi/node_modules/tslib": { + "version": "2.8.0", + "dev": true, + "inBundle": true, + "license": "0BSD", + "optional": true + }, "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { "version": "4.1.10", "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.1.10.tgz", @@ -2635,9 +2695,9 @@ "license": "MIT" }, "node_modules/vite": { - "version": "6.3.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.5.tgz", - "integrity": "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==", + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.6.tgz", + "integrity": "sha512-0msEVHJEScQbhkbVTb/4iHZdJ6SXp/AvxL2sjwYQFfBqleHtnCqv1J3sa9zbWz/6kW1m9Tfzn92vW+kZ1WV6QA==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index 10ec71415..e29c5e8e6 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "pusher-js": "8.4.0", "tailwind-scrollbar": "4.0.2", "tailwindcss": "4.1.10", - "vite": "6.3.5", + "vite": "6.3.6", "vue": "3.5.16" }, "dependencies": { From cead87d650cd44b8ab389a4aca72b0656c1be4a3 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 08:19:25 +0200 Subject: [PATCH 06/50] refactor(private-key): remove debugging statement from storeInFileSystem method for cleaner code --- app/Models/PrivateKey.php | 1 - 1 file changed, 1 deletion(-) diff --git a/app/Models/PrivateKey.php b/app/Models/PrivateKey.php index 851be6947..c210f3c5b 100644 --- a/app/Models/PrivateKey.php +++ b/app/Models/PrivateKey.php @@ -158,7 +158,6 @@ public static function validateAndExtractPublicKey($privateKey) public function storeInFileSystem() { - ray('storing private key in filesystem', $this->uuid); $filename = "ssh_key@{$this->uuid}"; $disk = Storage::disk('ssh-keys'); From b433f17dac4f651391ca65ef8a30a5ca49af766b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 08:19:38 +0200 Subject: [PATCH 07/50] feat(ssh-multiplexing): enhance multiplexed connection management with health checks and metadata caching --- app/Helpers/SshMultiplexingHelper.php | 129 +++++++++++++++++++++++++- config/constants.php | 3 + 2 files changed, 128 insertions(+), 4 deletions(-) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index 8caa2880a..bf9561f5a 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -4,7 +4,9 @@ use App\Models\PrivateKey; use App\Models\Server; +use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\Hash; +use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Process; class SshMultiplexingHelper @@ -30,6 +32,7 @@ public static function ensureMultiplexedConnection(Server $server): bool $sshConfig = self::serverSshConfiguration($server); $muxSocket = $sshConfig['muxFilename']; + // Check if connection exists $checkCommand = "ssh -O check -o ControlPath=$muxSocket "; if (data_get($server, 'settings.is_cloudflare_tunnel')) { $checkCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" '; @@ -41,6 +44,18 @@ public static function ensureMultiplexedConnection(Server $server): bool return self::establishNewMultiplexedConnection($server); } + // Connection exists, check if it needs refresh due to age + if (self::isConnectionExpired($server)) { + return self::refreshMultiplexedConnection($server); + } + + // Perform health check if enabled + if (config('constants.ssh.mux_health_check_enabled')) { + if (! self::isConnectionHealthy($server)) { + return self::refreshMultiplexedConnection($server); + } + } + return true; } @@ -65,6 +80,9 @@ public static function establishNewMultiplexedConnection(Server $server): bool return false; } + // Store connection metadata for tracking + self::storeConnectionMetadata($server); + return true; } @@ -79,6 +97,9 @@ public static function removeMuxFile(Server $server) } $closeCommand .= "{$server->user}@{$server->ip}"; Process::run($closeCommand); + + // Clear connection metadata from cache + self::clearConnectionMetadata($server); } public static function generateScpCommand(Server $server, string $source, string $dest) @@ -94,8 +115,18 @@ public static function generateScpCommand(Server $server, string $source, string if ($server->isIpv6()) { $scp_command .= '-6 '; } - if (self::isMultiplexingEnabled() && self::ensureMultiplexedConnection($server)) { - $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + if (self::isMultiplexingEnabled()) { + try { + if (self::ensureMultiplexedConnection($server)) { + $scp_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + } + } catch (\Exception $e) { + Log::warning('SSH multiplexing failed for SCP, falling back to non-multiplexed connection', [ + 'server' => $server->name ?? $server->ip, + 'error' => $e->getMessage(), + ]); + // Continue without multiplexing + } } if (data_get($server, 'settings.is_cloudflare_tunnel')) { @@ -130,8 +161,16 @@ public static function generateSshCommand(Server $server, string $command) $ssh_command = "timeout $timeout ssh "; - if (self::isMultiplexingEnabled() && self::ensureMultiplexedConnection($server)) { - $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + $multiplexingSuccessful = false; + if (self::isMultiplexingEnabled()) { + try { + $multiplexingSuccessful = self::ensureMultiplexedConnection($server); + if ($multiplexingSuccessful) { + $ssh_command .= "-o ControlMaster=auto -o ControlPath=$muxSocket -o ControlPersist={$muxPersistTime} "; + } + } catch (\Exception $e) { + // Continue without multiplexing + } } if (data_get($server, 'settings.is_cloudflare_tunnel')) { @@ -186,4 +225,86 @@ private static function getCommonSshOptions(Server $server, string $sshKeyLocati return $options; } + + /** + * Check if the multiplexed connection is healthy by running a test command + */ + public static function isConnectionHealthy(Server $server): bool + { + $sshConfig = self::serverSshConfiguration($server); + $muxSocket = $sshConfig['muxFilename']; + $healthCheckTimeout = config('constants.ssh.mux_health_check_timeout'); + + $healthCommand = "timeout $healthCheckTimeout ssh -o ControlMaster=auto -o ControlPath=$muxSocket "; + if (data_get($server, 'settings.is_cloudflare_tunnel')) { + $healthCommand .= '-o ProxyCommand="cloudflared access ssh --hostname %h" '; + } + $healthCommand .= "{$server->user}@{$server->ip} 'echo \"health_check_ok\"'"; + + $process = Process::run($healthCommand); + $isHealthy = $process->exitCode() === 0 && str_contains($process->output(), 'health_check_ok'); + + return $isHealthy; + } + + /** + * Check if the connection has exceeded its maximum age + */ + public static function isConnectionExpired(Server $server): bool + { + $connectionAge = self::getConnectionAge($server); + $maxAge = config('constants.ssh.mux_max_age'); + + return $connectionAge !== null && $connectionAge > $maxAge; + } + + /** + * Get the age of the current connection in seconds + */ + public static function getConnectionAge(Server $server): ?int + { + $cacheKey = "ssh_mux_connection_time_{$server->uuid}"; + $connectionTime = Cache::get($cacheKey); + + if ($connectionTime === null) { + return null; + } + + return time() - $connectionTime; + } + + /** + * Refresh a multiplexed connection by closing and re-establishing it + */ + public static function refreshMultiplexedConnection(Server $server): bool + { + Log::debug('Refreshing SSH multiplexed connection', [ + 'server' => $server->name ?? $server->ip, + 'age' => self::getConnectionAge($server), + ]); + + // Close existing connection + self::removeMuxFile($server); + + // Establish new connection + return self::establishNewMultiplexedConnection($server); + } + + /** + * Store connection metadata when a new connection is established + */ + private static function storeConnectionMetadata(Server $server): void + { + $cacheKey = "ssh_mux_connection_time_{$server->uuid}"; + Cache::put($cacheKey, time(), config('constants.ssh.mux_persist_time') + 300); // Cache slightly longer than persist time + } + + /** + * Clear connection metadata from cache + */ + private static function clearConnectionMetadata(Server $server): void + { + $cacheKey = "ssh_mux_connection_time_{$server->uuid}"; + Cache::forget($cacheKey); + } } diff --git a/config/constants.php b/config/constants.php index 652af5ff4..0d29c997e 100644 --- a/config/constants.php +++ b/config/constants.php @@ -59,6 +59,9 @@ 'ssh' => [ 'mux_enabled' => env('MUX_ENABLED', env('SSH_MUX_ENABLED', true)), 'mux_persist_time' => env('SSH_MUX_PERSIST_TIME', 3600), + 'mux_health_check_enabled' => env('SSH_MUX_HEALTH_CHECK_ENABLED', true), + 'mux_health_check_timeout' => env('SSH_MUX_HEALTH_CHECK_TIMEOUT', 5), + 'mux_max_age' => env('SSH_MUX_MAX_AGE', 1800), // 30 minutes 'connection_timeout' => 10, 'server_interval' => 20, 'command_timeout' => 7200, From 40f2471c5ab6f98afb836045c2e7f27a3c535b9b Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 08:38:36 +0200 Subject: [PATCH 08/50] feat(ssh-multiplexing): add connection age metadata handling to improve multiplexed connection management --- app/Helpers/SshMultiplexingHelper.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/Helpers/SshMultiplexingHelper.php b/app/Helpers/SshMultiplexingHelper.php index bf9561f5a..f847f33cc 100644 --- a/app/Helpers/SshMultiplexingHelper.php +++ b/app/Helpers/SshMultiplexingHelper.php @@ -44,6 +44,12 @@ public static function ensureMultiplexedConnection(Server $server): bool return self::establishNewMultiplexedConnection($server); } + // Connection exists, ensure we have metadata for age tracking + if (self::getConnectionAge($server) === null) { + // Existing connection but no metadata, store current time as fallback + self::storeConnectionMetadata($server); + } + // Connection exists, check if it needs refresh due to age if (self::isConnectionExpired($server)) { return self::refreshMultiplexedConnection($server); @@ -278,11 +284,6 @@ public static function getConnectionAge(Server $server): ?int */ public static function refreshMultiplexedConnection(Server $server): bool { - Log::debug('Refreshing SSH multiplexed connection', [ - 'server' => $server->name ?? $server->ip, - 'age' => self::getConnectionAge($server), - ]); - // Close existing connection self::removeMuxFile($server); From 52312e9de6d4a9011dee065e3492a6f637230add Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 09:30:43 +0200 Subject: [PATCH 09/50] refactor(github-webhook): restructure application processing by grouping applications by server for improved deployment handling --- app/Http/Controllers/Webhook/Github.php | 518 ++++++++++++------------ bootstrap/helpers/applications.php | 2 +- 2 files changed, 266 insertions(+), 254 deletions(-) diff --git a/app/Http/Controllers/Webhook/Github.php b/app/Http/Controllers/Webhook/Github.php index b940bf394..5ba9c08e7 100644 --- a/app/Http/Controllers/Webhook/Github.php +++ b/app/Http/Controllers/Webhook/Github.php @@ -97,162 +97,168 @@ public function manual(Request $request) return response("Nothing to do. No applications found with branch '$base_branch'."); } } - foreach ($applications as $application) { - $webhook_secret = data_get($application, 'manual_webhook_secret_github'); - $hmac = hash_hmac('sha256', $request->getContent(), $webhook_secret); - if (! hash_equals($x_hub_signature_256, $hmac) && ! isDev()) { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'failed', - 'message' => 'Invalid signature.', - ]); + $applicationsByServer = $applications->groupBy(function ($app) { + return $app->destination->server_id; + }); - continue; - } - $isFunctional = $application->destination->server->isFunctional(); - if (! $isFunctional) { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'failed', - 'message' => 'Server is not functional.', - ]); - - continue; - } - if ($x_github_event === 'push') { - if ($application->isDeployable()) { - $is_watch_path_triggered = $application->isWatchPathsTriggered($changed_files); - if ($is_watch_path_triggered || is_null($application->watch_paths)) { - $deployment_uuid = new Cuid2; - $result = queue_application_deployment( - application: $application, - deployment_uuid: $deployment_uuid, - force_rebuild: false, - commit: data_get($payload, 'after', 'HEAD'), - is_webhook: true, - ); - if ($result['status'] === 'skipped') { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'skipped', - 'message' => $result['message'], - ]); - } else { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'success', - 'message' => 'Deployment queued.', - 'application_uuid' => $application->uuid, - 'application_name' => $application->name, - 'deployment_uuid' => $result['deployment_uuid'], - ]); - } - } else { - $paths = str($application->watch_paths)->explode("\n"); - $return_payloads->push([ - 'status' => 'failed', - 'message' => 'Changed files do not match watch paths. Ignoring deployment.', - 'application_uuid' => $application->uuid, - 'application_name' => $application->name, - 'details' => [ - 'changed_files' => $changed_files, - 'watch_paths' => $paths, - ], - ]); - } - } else { + foreach ($applicationsByServer as $serverId => $serverApplications) { + foreach ($serverApplications as $application) { + $webhook_secret = data_get($application, 'manual_webhook_secret_github'); + $hmac = hash_hmac('sha256', $request->getContent(), $webhook_secret); + if (! hash_equals($x_hub_signature_256, $hmac) && ! isDev()) { $return_payloads->push([ + 'application' => $application->name, 'status' => 'failed', - 'message' => 'Deployments disabled.', - 'application_uuid' => $application->uuid, - 'application_name' => $application->name, + 'message' => 'Invalid signature.', ]); + + continue; } - } - if ($x_github_event === 'pull_request') { - if ($action === 'opened' || $action === 'synchronize' || $action === 'reopened') { - if ($application->isPRDeployable()) { - // Check if PR deployments from public contributors are restricted - if (! $application->settings->is_pr_deployments_public_enabled) { - $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR']; - if (! in_array($author_association, $trustedAssociations)) { + $isFunctional = $application->destination->server->isFunctional(); + if (! $isFunctional) { + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'failed', + 'message' => 'Server is not functional.', + ]); + + continue; + } + if ($x_github_event === 'push') { + if ($application->isDeployable()) { + $is_watch_path_triggered = $application->isWatchPathsTriggered($changed_files); + if ($is_watch_path_triggered || is_null($application->watch_paths)) { + $deployment_uuid = new Cuid2; + $result = queue_application_deployment( + application: $application, + deployment_uuid: $deployment_uuid, + force_rebuild: false, + commit: data_get($payload, 'after', 'HEAD'), + is_webhook: true, + ); + if ($result['status'] === 'skipped') { $return_payloads->push([ 'application' => $application->name, - 'status' => 'failed', - 'message' => 'PR deployments are restricted to repository members and contributors. Author association: '.$author_association, + 'status' => 'skipped', + 'message' => $result['message'], ]); - - continue; - } - } - $deployment_uuid = new Cuid2; - $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); - if (! $found) { - if ($application->build_pack === 'dockercompose') { - $pr_app = ApplicationPreview::create([ - 'git_type' => 'github', - 'application_id' => $application->id, - 'pull_request_id' => $pull_request_id, - 'pull_request_html_url' => $pull_request_html_url, - 'docker_compose_domains' => $application->docker_compose_domains, - ]); - $pr_app->generate_preview_fqdn_compose(); } else { - $pr_app = ApplicationPreview::create([ - 'git_type' => 'github', - 'application_id' => $application->id, - 'pull_request_id' => $pull_request_id, - 'pull_request_html_url' => $pull_request_html_url, + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'success', + 'message' => 'Deployment queued.', + 'application_uuid' => $application->uuid, + 'application_name' => $application->name, + 'deployment_uuid' => $result['deployment_uuid'], ]); - $pr_app->generate_preview_fqdn(); } + } else { + $paths = str($application->watch_paths)->explode("\n"); + $return_payloads->push([ + 'status' => 'failed', + 'message' => 'Changed files do not match watch paths. Ignoring deployment.', + 'application_uuid' => $application->uuid, + 'application_name' => $application->name, + 'details' => [ + 'changed_files' => $changed_files, + 'watch_paths' => $paths, + ], + ]); } + } else { + $return_payloads->push([ + 'status' => 'failed', + 'message' => 'Deployments disabled.', + 'application_uuid' => $application->uuid, + 'application_name' => $application->name, + ]); + } + } + if ($x_github_event === 'pull_request') { + if ($action === 'opened' || $action === 'synchronize' || $action === 'reopened') { + if ($application->isPRDeployable()) { + // Check if PR deployments from public contributors are restricted + if (! $application->settings->is_pr_deployments_public_enabled) { + $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR']; + if (! in_array($author_association, $trustedAssociations)) { + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'failed', + 'message' => 'PR deployments are restricted to repository members and contributors. Author association: '.$author_association, + ]); - $result = queue_application_deployment( - application: $application, - pull_request_id: $pull_request_id, - deployment_uuid: $deployment_uuid, - force_rebuild: false, - commit: data_get($payload, 'head.sha', 'HEAD'), - is_webhook: true, - git_type: 'github' - ); - if ($result['status'] === 'skipped') { + continue; + } + } + $deployment_uuid = new Cuid2; + $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); + if (! $found) { + if ($application->build_pack === 'dockercompose') { + $pr_app = ApplicationPreview::create([ + 'git_type' => 'github', + 'application_id' => $application->id, + 'pull_request_id' => $pull_request_id, + 'pull_request_html_url' => $pull_request_html_url, + 'docker_compose_domains' => $application->docker_compose_domains, + ]); + $pr_app->generate_preview_fqdn_compose(); + } else { + $pr_app = ApplicationPreview::create([ + 'git_type' => 'github', + 'application_id' => $application->id, + 'pull_request_id' => $pull_request_id, + 'pull_request_html_url' => $pull_request_html_url, + ]); + $pr_app->generate_preview_fqdn(); + } + } + + $result = queue_application_deployment( + application: $application, + pull_request_id: $pull_request_id, + deployment_uuid: $deployment_uuid, + force_rebuild: false, + commit: data_get($payload, 'head.sha', 'HEAD'), + is_webhook: true, + git_type: 'github' + ); + if ($result['status'] === 'skipped') { + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'skipped', + 'message' => $result['message'], + ]); + } else { + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'success', + 'message' => 'Preview deployment queued.', + ]); + } + } else { $return_payloads->push([ 'application' => $application->name, - 'status' => 'skipped', - 'message' => $result['message'], + 'status' => 'failed', + 'message' => 'Preview deployments disabled.', + ]); + } + } + if ($action === 'closed') { + $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); + if ($found) { + DeleteResourceJob::dispatch($found); + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'success', + 'message' => 'Preview deployment closed.', ]); } else { $return_payloads->push([ 'application' => $application->name, - 'status' => 'success', - 'message' => 'Preview deployment queued.', + 'status' => 'failed', + 'message' => 'No preview deployment found.', ]); } - } else { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'failed', - 'message' => 'Preview deployments disabled.', - ]); - } - } - if ($action === 'closed') { - $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); - if ($found) { - DeleteResourceJob::dispatch($found); - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'success', - 'message' => 'Preview deployment closed.', - ]); - } else { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'failed', - 'message' => 'No preview deployment found.', - ]); } } } @@ -358,141 +364,147 @@ public function normal(Request $request) return response("Nothing to do. No applications found with branch '$base_branch'."); } } - foreach ($applications as $application) { - $isFunctional = $application->destination->server->isFunctional(); - if (! $isFunctional) { - $return_payloads->push([ - 'status' => 'failed', - 'message' => 'Server is not functional.', - 'application_uuid' => $application->uuid, - 'application_name' => $application->name, - ]); + $applicationsByServer = $applications->groupBy(function ($app) { + return $app->destination->server_id; + }); - continue; - } - if ($x_github_event === 'push') { - if ($application->isDeployable()) { - $is_watch_path_triggered = $application->isWatchPathsTriggered($changed_files); - if ($is_watch_path_triggered || is_null($application->watch_paths)) { - $deployment_uuid = new Cuid2; - $result = queue_application_deployment( - application: $application, - deployment_uuid: $deployment_uuid, - commit: data_get($payload, 'after', 'HEAD'), - force_rebuild: false, - is_webhook: true, - ); - $return_payloads->push([ - 'status' => $result['status'], - 'message' => $result['message'], - 'application_uuid' => $application->uuid, - 'application_name' => $application->name, - 'deployment_uuid' => $result['deployment_uuid'], - ]); - } else { - $paths = str($application->watch_paths)->explode("\n"); - $return_payloads->push([ - 'status' => 'failed', - 'message' => 'Changed files do not match watch paths. Ignoring deployment.', - 'application_uuid' => $application->uuid, - 'application_name' => $application->name, - 'details' => [ - 'changed_files' => $changed_files, - 'watch_paths' => $paths, - ], - ]); - } - } else { + foreach ($applicationsByServer as $serverId => $serverApplications) { + foreach ($serverApplications as $application) { + $isFunctional = $application->destination->server->isFunctional(); + if (! $isFunctional) { $return_payloads->push([ 'status' => 'failed', - 'message' => 'Deployments disabled.', + 'message' => 'Server is not functional.', 'application_uuid' => $application->uuid, 'application_name' => $application->name, ]); - } - } - if ($x_github_event === 'pull_request') { - if ($action === 'opened' || $action === 'synchronize' || $action === 'reopened') { - if ($application->isPRDeployable()) { - // Check if PR deployments from public contributors are restricted - if (! $application->settings->is_pr_deployments_public_enabled) { - $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR']; - if (! in_array($author_association, $trustedAssociations)) { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'failed', - 'message' => 'PR deployments are restricted to repository members and contributors. Author association: '.$author_association, - ]); - continue; - } - } - $deployment_uuid = new Cuid2; - $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); - if (! $found) { - ApplicationPreview::create([ - 'git_type' => 'github', - 'application_id' => $application->id, - 'pull_request_id' => $pull_request_id, - 'pull_request_html_url' => $pull_request_html_url, + continue; + } + if ($x_github_event === 'push') { + if ($application->isDeployable()) { + $is_watch_path_triggered = $application->isWatchPathsTriggered($changed_files); + if ($is_watch_path_triggered || is_null($application->watch_paths)) { + $deployment_uuid = new Cuid2; + $result = queue_application_deployment( + application: $application, + deployment_uuid: $deployment_uuid, + commit: data_get($payload, 'after', 'HEAD'), + force_rebuild: false, + is_webhook: true, + ); + $return_payloads->push([ + 'status' => $result['status'], + 'message' => $result['message'], + 'application_uuid' => $application->uuid, + 'application_name' => $application->name, + 'deployment_uuid' => $result['deployment_uuid'], + ]); + } else { + $paths = str($application->watch_paths)->explode("\n"); + $return_payloads->push([ + 'status' => 'failed', + 'message' => 'Changed files do not match watch paths. Ignoring deployment.', + 'application_uuid' => $application->uuid, + 'application_name' => $application->name, + 'details' => [ + 'changed_files' => $changed_files, + 'watch_paths' => $paths, + ], ]); } - $result = queue_application_deployment( - application: $application, - pull_request_id: $pull_request_id, - deployment_uuid: $deployment_uuid, - force_rebuild: false, - commit: data_get($payload, 'head.sha', 'HEAD'), - is_webhook: true, - git_type: 'github' - ); - if ($result['status'] === 'skipped') { + } else { + $return_payloads->push([ + 'status' => 'failed', + 'message' => 'Deployments disabled.', + 'application_uuid' => $application->uuid, + 'application_name' => $application->name, + ]); + } + } + if ($x_github_event === 'pull_request') { + if ($action === 'opened' || $action === 'synchronize' || $action === 'reopened') { + if ($application->isPRDeployable()) { + // Check if PR deployments from public contributors are restricted + if (! $application->settings->is_pr_deployments_public_enabled) { + $trustedAssociations = ['OWNER', 'MEMBER', 'COLLABORATOR', 'CONTRIBUTOR']; + if (! in_array($author_association, $trustedAssociations)) { + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'failed', + 'message' => 'PR deployments are restricted to repository members and contributors. Author association: '.$author_association, + ]); + + continue; + } + } + $deployment_uuid = new Cuid2; + $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); + if (! $found) { + ApplicationPreview::create([ + 'git_type' => 'github', + 'application_id' => $application->id, + 'pull_request_id' => $pull_request_id, + 'pull_request_html_url' => $pull_request_html_url, + ]); + } + $result = queue_application_deployment( + application: $application, + pull_request_id: $pull_request_id, + deployment_uuid: $deployment_uuid, + force_rebuild: false, + commit: data_get($payload, 'head.sha', 'HEAD'), + is_webhook: true, + git_type: 'github' + ); + if ($result['status'] === 'skipped') { + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'skipped', + 'message' => $result['message'], + ]); + } else { + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'success', + 'message' => 'Preview deployment queued.', + ]); + } + } else { $return_payloads->push([ 'application' => $application->name, - 'status' => 'skipped', - 'message' => $result['message'], + 'status' => 'failed', + 'message' => 'Preview deployments disabled.', + ]); + } + } + if ($action === 'closed' || $action === 'close') { + $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); + if ($found) { + $containers = getCurrentApplicationContainerStatus($application->destination->server, $application->id, $pull_request_id); + if ($containers->isNotEmpty()) { + $containers->each(function ($container) use ($application) { + $container_name = data_get($container, 'Names'); + instant_remote_process(["docker rm -f $container_name"], $application->destination->server); + }); + } + + ApplicationPullRequestUpdateJob::dispatchSync(application: $application, preview: $found, status: ProcessStatus::CLOSED); + + DeleteResourceJob::dispatch($found); + + $return_payloads->push([ + 'application' => $application->name, + 'status' => 'success', + 'message' => 'Preview deployment closed.', ]); } else { $return_payloads->push([ 'application' => $application->name, - 'status' => 'success', - 'message' => 'Preview deployment queued.', + 'status' => 'failed', + 'message' => 'No preview deployment found.', ]); } - } else { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'failed', - 'message' => 'Preview deployments disabled.', - ]); - } - } - if ($action === 'closed' || $action === 'close') { - $found = ApplicationPreview::where('application_id', $application->id)->where('pull_request_id', $pull_request_id)->first(); - if ($found) { - $containers = getCurrentApplicationContainerStatus($application->destination->server, $application->id, $pull_request_id); - if ($containers->isNotEmpty()) { - $containers->each(function ($container) use ($application) { - $container_name = data_get($container, 'Names'); - instant_remote_process(["docker rm -f $container_name"], $application->destination->server); - }); - } - - ApplicationPullRequestUpdateJob::dispatchSync(application: $application, preview: $found, status: ProcessStatus::CLOSED); - - DeleteResourceJob::dispatch($found); - - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'success', - 'message' => 'Preview deployment closed.', - ]); - } else { - $return_payloads->push([ - 'application' => $application->name, - 'status' => 'failed', - 'message' => 'No preview deployment found.', - ]); } } } diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 919b2bde5..975c6fcf1 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -147,7 +147,7 @@ function next_after_cancel(?Server $server = null) foreach ($next_found as $next) { $server = Server::find($next->server_id); $concurrent_builds = $server->settings->concurrent_builds; - $inprogress_deployments = ApplicationDeploymentQueue::where('server_id', $next->server_id)->whereIn('status', [ApplicationDeploymentStatus::QUEUED])->get()->sortByDesc('created_at'); + $inprogress_deployments = ApplicationDeploymentQueue::where('server_id', $next->server_id)->whereIn('status', [ApplicationDeploymentStatus::IN_PROGRESS])->get()->sortByDesc('created_at'); if ($inprogress_deployments->count() < $concurrent_builds) { $next->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, From b6113839ec0bd49cb4c57853674f3bdf7c007e82 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 12:49:42 +0200 Subject: [PATCH 10/50] refactor(deployment): enhance queuing logic to support concurrent deployments by including pull request ID in checks --- bootstrap/helpers/applications.php | 56 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 975c6fcf1..6c4f8fd22 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -68,7 +68,7 @@ function queue_application_deployment(Application $application, string $deployme ApplicationDeploymentJob::dispatch( application_deployment_queue_id: $deployment->id, ); - } elseif (next_queuable($server_id, $application_id, $commit)) { + } elseif (next_queuable($server_id, $application_id, $commit, $pull_request_id)) { ApplicationDeploymentJob::dispatch( application_deployment_queue_id: $deployment->id, ); @@ -93,32 +93,32 @@ function force_start_deployment(ApplicationDeploymentQueue $deployment) function queue_next_deployment(Application $application) { $server_id = $application->destination->server_id; - $next_found = ApplicationDeploymentQueue::where('server_id', $server_id)->where('status', ApplicationDeploymentStatus::QUEUED)->get()->sortBy('created_at')->first(); - if ($next_found) { - $next_found->update([ - 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, - ]); + $queued_deployments = ApplicationDeploymentQueue::where('server_id', $server_id) + ->where('status', ApplicationDeploymentStatus::QUEUED) + ->get() + ->sortBy('created_at'); - ApplicationDeploymentJob::dispatch( - application_deployment_queue_id: $next_found->id, - ); + foreach ($queued_deployments as $next_deployment) { + // Check if this queued deployment can actually run + if (next_queuable($next_deployment->server_id, $next_deployment->application_id, $next_deployment->commit, $next_deployment->pull_request_id)) { + $next_deployment->update([ + 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, + ]); + + ApplicationDeploymentJob::dispatch( + application_deployment_queue_id: $next_deployment->id, + ); + break; + } } } -function next_queuable(string $server_id, string $application_id, string $commit = 'HEAD'): bool +function next_queuable(string $server_id, string $application_id, string $commit = 'HEAD', int $pull_request_id = 0): bool { - // Check if there's already a deployment in progress for this application and commit - $existing_deployment = ApplicationDeploymentQueue::where('application_id', $application_id) - ->where('commit', $commit) - ->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value) - ->first(); - - if ($existing_deployment) { - return false; - } - - // Check if there's any deployment in progress for this application + // Check if there's already a deployment in progress for this application with the same pull_request_id + // This allows normal deployments and PR deployments to run concurrently $in_progress = ApplicationDeploymentQueue::where('application_id', $application_id) + ->where('pull_request_id', $pull_request_id) ->where('status', ApplicationDeploymentStatus::IN_PROGRESS->value) ->exists(); @@ -142,13 +142,15 @@ function next_queuable(string $server_id, string $application_id, string $commit function next_after_cancel(?Server $server = null) { if ($server) { - $next_found = ApplicationDeploymentQueue::where('server_id', data_get($server, 'id'))->where('status', ApplicationDeploymentStatus::QUEUED)->get()->sortBy('created_at'); + $next_found = ApplicationDeploymentQueue::where('server_id', data_get($server, 'id')) + ->where('status', ApplicationDeploymentStatus::QUEUED) + ->get() + ->sortBy('created_at'); + if ($next_found->count() > 0) { foreach ($next_found as $next) { - $server = Server::find($next->server_id); - $concurrent_builds = $server->settings->concurrent_builds; - $inprogress_deployments = ApplicationDeploymentQueue::where('server_id', $next->server_id)->whereIn('status', [ApplicationDeploymentStatus::IN_PROGRESS])->get()->sortByDesc('created_at'); - if ($inprogress_deployments->count() < $concurrent_builds) { + // Use next_queuable to properly check if this deployment can run + if (next_queuable($next->server_id, $next->application_id, $next->commit, $next->pull_request_id)) { $next->update([ 'status' => ApplicationDeploymentStatus::IN_PROGRESS->value, ]); @@ -156,8 +158,8 @@ function next_after_cancel(?Server $server = null) ApplicationDeploymentJob::dispatch( application_deployment_queue_id: $next->id, ); + break; } - break; } } } From f18dff186d39a7c4d3ec4e325168d30b49ca8c53 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:25:23 +0200 Subject: [PATCH 11/50] refactor(remoteProcess): remove debugging statement from transfer_file_to_container function for cleaner code --- bootstrap/helpers/remoteProcess.php | 1 - 1 file changed, 1 deletion(-) diff --git a/bootstrap/helpers/remoteProcess.php b/bootstrap/helpers/remoteProcess.php index 8687bfaa5..7fa9671e3 100644 --- a/bootstrap/helpers/remoteProcess.php +++ b/bootstrap/helpers/remoteProcess.php @@ -130,7 +130,6 @@ function transfer_file_to_container(string $content, string $container_path, str return instant_remote_process_with_timeout($commands, $server, $throwError); } finally { - ray($temp_file); // Always cleanup local temp file if (file_exists($temp_file)) { unlink($temp_file); From d10e4fa38824f35bff8ad477838ef26f8e894fb9 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 13:26:35 +0200 Subject: [PATCH 12/50] refactor(deployment): streamline next deployment queuing logic by repositioning queue_next_deployment call --- app/Jobs/ApplicationDeploymentJob.php | 4 ++-- bootstrap/helpers/applications.php | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index 6059cb99a..a1a15eb40 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -2457,8 +2457,6 @@ private function run_post_deployment_command() private function next(string $status) { - queue_next_deployment($this->application); - // Never allow changing status from FAILED or CANCELLED_BY_USER to anything else if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::FAILED->value) { $this->application->environment->project->team?->notify(new DeploymentFailed($this->application, $this->deployment_uuid, $this->preview)); @@ -2473,6 +2471,8 @@ private function next(string $status) 'status' => $status, ]); + queue_next_deployment($this->application); + if ($status === ApplicationDeploymentStatus::FINISHED->value) { if (! $this->only_this_server) { $this->deploy_to_additional_destinations(); diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 6c4f8fd22..87c24dbc6 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -108,7 +108,6 @@ function queue_next_deployment(Application $application) ApplicationDeploymentJob::dispatch( application_deployment_queue_id: $next_deployment->id, ); - break; } } } @@ -158,7 +157,6 @@ function next_after_cancel(?Server $server = null) ApplicationDeploymentJob::dispatch( application_deployment_queue_id: $next->id, ); - break; } } } From 48d3b3d263682862ecd19c64098450340087f79a Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 14:31:29 +0200 Subject: [PATCH 13/50] refactor(deployment): add validation for pull request existence in deployment process to enhance error handling --- app/Http/Controllers/Api/DeployController.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/Http/Controllers/Api/DeployController.php b/app/Http/Controllers/Api/DeployController.php index b87420f72..c4d603392 100644 --- a/app/Http/Controllers/Api/DeployController.php +++ b/app/Http/Controllers/Api/DeployController.php @@ -225,6 +225,14 @@ private function by_uuids(string $uuid, int $teamId, bool $force = false, int $p foreach ($uuids as $uuid) { $resource = getResourceByUuid($uuid, $teamId); if ($resource) { + if ($pr !== 0) { + $preview = $resource->previews()->where('pull_request_id', $pr)->first(); + if (! $preview) { + $deployments->push(['message' => "Pull request {$pr} not found for this resource.", 'resource_uuid' => $uuid]); + + continue; + } + } ['message' => $return_message, 'deployment_uuid' => $deployment_uuid] = $this->deploy_resource($resource, $force, $pr); if ($deployment_uuid) { $deployments->push(['message' => $return_message, 'resource_uuid' => $uuid, 'deployment_uuid' => $deployment_uuid->toString()]); From 1c08d32b858ddca74342bcfca50436644d3a6a41 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:12:53 +0200 Subject: [PATCH 14/50] refactor(database): remove volume_configuration_dir and streamline configuration directory usage in MongoDB and PostgreSQL handlers --- app/Actions/Database/StartMongodb.php | 13 ++++--------- app/Actions/Database/StartPostgresql.php | 12 ++++-------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/app/Actions/Database/StartMongodb.php b/app/Actions/Database/StartMongodb.php index 0372cd64f..7135f1c70 100644 --- a/app/Actions/Database/StartMongodb.php +++ b/app/Actions/Database/StartMongodb.php @@ -18,8 +18,6 @@ class StartMongodb public string $configuration_dir; - public string $volume_configuration_dir; - private ?SslCertificate $ssl_certificate = null; public function handle(StandaloneMongodb $database) @@ -29,10 +27,7 @@ public function handle(StandaloneMongodb $database) $startCommand = 'mongod'; $container_name = $this->database->uuid; - $this->volume_configuration_dir = $this->configuration_dir = database_configuration_dir().'/'.$container_name; - if (isDev()) { - $this->volume_configuration_dir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/databases/'.$container_name; - } + $this->configuration_dir = database_configuration_dir().'/'.$container_name; $this->commands = [ "echo 'Starting database.'", @@ -178,7 +173,7 @@ public function handle(StandaloneMongodb $database) $docker_compose['services'][$container_name]['volumes'] ?? [], [[ 'type' => 'bind', - 'source' => $this->volume_configuration_dir.'/mongod.conf', + 'source' => $this->configuration_dir.'/mongod.conf', 'target' => '/etc/mongo/mongod.conf', 'read_only' => true, ]] @@ -192,7 +187,7 @@ public function handle(StandaloneMongodb $database) $docker_compose['services'][$container_name]['volumes'] ?? [], [[ 'type' => 'bind', - 'source' => $this->volume_configuration_dir.'/docker-entrypoint-initdb.d', + 'source' => $this->configuration_dir.'/docker-entrypoint-initdb.d', 'target' => '/docker-entrypoint-initdb.d', 'read_only' => true, ]] @@ -259,7 +254,7 @@ public function handle(StandaloneMongodb $database) $this->commands[] = [ 'transfer_file' => [ 'content' => $docker_compose, - 'destination' => "$this->volume_configuration_dir/docker-compose.yml", + 'destination' => "$this->configuration_dir/docker-compose.yml", ], ]; $readme = generate_readme_file($this->database->name, now()); diff --git a/app/Actions/Database/StartPostgresql.php b/app/Actions/Database/StartPostgresql.php index 80860bda2..75ca8ef10 100644 --- a/app/Actions/Database/StartPostgresql.php +++ b/app/Actions/Database/StartPostgresql.php @@ -20,8 +20,6 @@ class StartPostgresql public string $configuration_dir; - public string $volume_configuration_dir; - private ?SslCertificate $ssl_certificate = null; public function handle(StandalonePostgresql $database) @@ -29,10 +27,6 @@ public function handle(StandalonePostgresql $database) $this->database = $database; $container_name = $this->database->uuid; $this->configuration_dir = database_configuration_dir().'/'.$container_name; - $this->volume_configuration_dir = $this->configuration_dir; - if (isDev()) { - $this->volume_configuration_dir = '/var/lib/docker/volumes/coolify_dev_coolify_data/_data/databases/'.$container_name; - } $this->commands = [ "echo 'Starting database.'", @@ -195,7 +189,7 @@ public function handle(StandalonePostgresql $database) $docker_compose['services'][$container_name]['volumes'], [[ 'type' => 'bind', - 'source' => $this->volume_configuration_dir.'/custom-postgres.conf', + 'source' => $this->configuration_dir.'/custom-postgres.conf', 'target' => '/etc/postgresql/postgresql.conf', 'read_only' => true, ]] @@ -223,7 +217,7 @@ public function handle(StandalonePostgresql $database) $this->commands[] = [ 'transfer_file' => [ 'content' => $docker_compose, - 'destination' => "$this->volume_configuration_dir/docker-compose.yml", + 'destination' => "$this->configuration_dir/docker-compose.yml", ], ]; $readme = generate_readme_file($this->database->name, now()); @@ -236,6 +230,8 @@ public function handle(StandalonePostgresql $database) } $this->commands[] = "echo 'Database started.'"; + ray($this->commands); + return remote_process($this->commands, $database->destination->server, callEventOnFinish: 'DatabaseStatusChanged'); } From 2c8f5415f13244a844bb94995b67ca1408677527 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:13:08 +0200 Subject: [PATCH 15/50] feat(database-backup): enhance error handling and output management in DatabaseBackupJob --- app/Jobs/DatabaseBackupJob.php | 62 ++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 752d1f1ca..6ac9ae1e6 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -54,6 +54,10 @@ class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue public ?string $backup_output = null; + public ?string $error_output = null; + + public bool $s3_uploaded = false; + public ?string $postgres_password = null; public ?string $mongo_root_username = null; @@ -355,7 +359,6 @@ public function handle(): void // If local backup is disabled, delete the local file immediately after S3 upload if ($this->backup->disable_local_backup) { deleteBackupsLocally($this->backup_location, $this->server); - $this->add_to_backup_output('Local backup file deleted after S3 upload (disable_local_backup enabled).'); } } @@ -367,15 +370,34 @@ public function handle(): void 'size' => $size, ]); } catch (\Throwable $e) { - if ($this->backup_log) { - $this->backup_log->update([ - 'status' => 'failed', - 'message' => $this->backup_output, - 'size' => $size, - 'filename' => null, - ]); + // Check if backup actually failed or if it's just a post-backup issue + $actualBackupFailed = ! $this->s3_uploaded && $this->backup->save_s3; + + if ($actualBackupFailed || $size === 0) { + // Real backup failure + if ($this->backup_log) { + $this->backup_log->update([ + 'status' => 'failed', + 'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(), + 'size' => $size, + 'filename' => null, + ]); + } + $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database)); + } else { + // Backup succeeded but post-processing failed (cleanup, notification, etc.) + if ($this->backup_log) { + $this->backup_log->update([ + 'status' => 'success', + 'message' => $this->backup_output ? $this->backup_output."\nWarning: Post-backup cleanup encountered an issue: ".$e->getMessage() : 'Warning: '.$e->getMessage(), + 'size' => $size, + ]); + } + // Send success notification since the backup itself succeeded + $this->team->notify(new BackupSuccess($this->backup, $this->database, $database)); + // Log the post-backup issue + ray('Post-backup operation failed but backup was successful: '.$e->getMessage()); } - $this->team?->notify(new BackupFailed($this->backup, $this->database, $this->backup_output, $database)); } } if ($this->backup_log && $this->backup_log->status === 'success') { @@ -446,7 +468,7 @@ private function backup_standalone_mongodb(string $databaseWithCollections): voi $this->backup_output = null; } } catch (\Throwable $e) { - $this->add_to_backup_output($e->getMessage()); + $this->add_to_error_output($e->getMessage()); throw $e; } } @@ -472,7 +494,7 @@ private function backup_standalone_postgresql(string $database): void $this->backup_output = null; } } catch (\Throwable $e) { - $this->add_to_backup_output($e->getMessage()); + $this->add_to_error_output($e->getMessage()); throw $e; } } @@ -492,7 +514,7 @@ private function backup_standalone_mysql(string $database): void $this->backup_output = null; } } catch (\Throwable $e) { - $this->add_to_backup_output($e->getMessage()); + $this->add_to_error_output($e->getMessage()); throw $e; } } @@ -512,7 +534,7 @@ private function backup_standalone_mariadb(string $database): void $this->backup_output = null; } } catch (\Throwable $e) { - $this->add_to_backup_output($e->getMessage()); + $this->add_to_error_output($e->getMessage()); throw $e; } } @@ -526,6 +548,15 @@ private function add_to_backup_output($output): void } } + private function add_to_error_output($output): void + { + if ($this->error_output) { + $this->error_output = $this->error_output."\n".$output; + } else { + $this->error_output = $output; + } + } + private function calculate_size() { return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false); @@ -571,9 +602,10 @@ private function upload_to_s3(): void $commands[] = "docker exec backup-of-{$this->backup->uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/"; instant_remote_process($commands, $this->server); - $this->add_to_backup_output('Uploaded to S3.'); + $this->s3_uploaded = true; } catch (\Throwable $e) { - $this->add_to_backup_output($e->getMessage()); + $this->s3_uploaded = false; + $this->add_to_error_output($e->getMessage()); throw $e; } finally { $command = "docker rm -f backup-of-{$this->backup->uuid}"; From 49a294283626ff0e5c87c68332551fb7c0090b07 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 16:15:08 +0200 Subject: [PATCH 16/50] fix(deployment): add COOLIFY_* environment variables to Nixpacks build context for enhanced deployment configuration --- app/Jobs/ApplicationDeploymentJob.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/Jobs/ApplicationDeploymentJob.php b/app/Jobs/ApplicationDeploymentJob.php index a1a15eb40..35e479ff4 100644 --- a/app/Jobs/ApplicationDeploymentJob.php +++ b/app/Jobs/ApplicationDeploymentJob.php @@ -1600,6 +1600,12 @@ private function generate_nixpacks_env_variables() } } + // Add COOLIFY_* environment variables to Nixpacks build context + $coolify_envs = $this->generate_coolify_env_variables(); + $coolify_envs->each(function ($value, $key) { + $this->env_nixpacks_args->push("--env {$key}={$value}"); + }); + $this->env_nixpacks_args = $this->env_nixpacks_args->implode(' '); } From fc7770100bf7e47a34ec655e29f247f127013cf2 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 18:29:52 +0200 Subject: [PATCH 17/50] refactor(application-source): improve layout and accessibility of Git repository links in the application source view --- .../project/application/source.blade.php | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/resources/views/livewire/project/application/source.blade.php b/resources/views/livewire/project/application/source.blade.php index 9e746fadb..9d0d53f2e 100644 --- a/resources/views/livewire/project/application/source.blade.php +++ b/resources/views/livewire/project/application/source.blade.php @@ -5,25 +5,25 @@ @can('update', $application) Save @endcan - - +
+ Open Repository - - - @if (data_get($application, 'source.is_public') === false) - - + + @if (data_get($application, 'source.is_public') === false) + Open Git App - - - @endif - - Open Commits on Git + + @endif + + Open Commits on Git - - + +
Code source of your application.
@@ -34,11 +34,13 @@ class="font-bold text-warning">{{ data_get($application, 'source.name', 'No sour @endif
- +
- +
From f64622c764fab3af89592e3dd0c9df3b26e63572 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 19:34:49 +0200 Subject: [PATCH 18/50] refactor(models): remove 'is_readonly' attribute from multiple database models for consistency --- app/Models/StandaloneClickhouse.php | 1 - app/Models/StandaloneDragonfly.php | 1 - app/Models/StandaloneKeydb.php | 1 - app/Models/StandaloneMariadb.php | 1 - app/Models/StandaloneMongodb.php | 2 -- app/Models/StandaloneMysql.php | 1 - app/Models/StandalonePostgresql.php | 1 - app/Models/StandaloneRedis.php | 1 - resources/views/livewire/project/shared/storages/all.blade.php | 3 +-- 9 files changed, 1 insertion(+), 11 deletions(-) diff --git a/app/Models/StandaloneClickhouse.php b/app/Models/StandaloneClickhouse.php index 60a750a99..88142066f 100644 --- a/app/Models/StandaloneClickhouse.php +++ b/app/Models/StandaloneClickhouse.php @@ -28,7 +28,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/app/Models/StandaloneDragonfly.php b/app/Models/StandaloneDragonfly.php index 673851713..b7d22a2ce 100644 --- a/app/Models/StandaloneDragonfly.php +++ b/app/Models/StandaloneDragonfly.php @@ -28,7 +28,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/app/Models/StandaloneKeydb.php b/app/Models/StandaloneKeydb.php index e6562193b..807728a36 100644 --- a/app/Models/StandaloneKeydb.php +++ b/app/Models/StandaloneKeydb.php @@ -28,7 +28,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/app/Models/StandaloneMariadb.php b/app/Models/StandaloneMariadb.php index 1aa9d63c1..8d602c27d 100644 --- a/app/Models/StandaloneMariadb.php +++ b/app/Models/StandaloneMariadb.php @@ -29,7 +29,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/app/Models/StandaloneMongodb.php b/app/Models/StandaloneMongodb.php index 299ea75b2..f222b0e5c 100644 --- a/app/Models/StandaloneMongodb.php +++ b/app/Models/StandaloneMongodb.php @@ -24,7 +24,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); LocalPersistentVolume::create([ 'name' => 'mongodb-db-'.$database->uuid, @@ -32,7 +31,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/app/Models/StandaloneMysql.php b/app/Models/StandaloneMysql.php index f376c7644..e4693c76a 100644 --- a/app/Models/StandaloneMysql.php +++ b/app/Models/StandaloneMysql.php @@ -29,7 +29,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/app/Models/StandalonePostgresql.php b/app/Models/StandalonePostgresql.php index 0bca2f4a7..47c984ff7 100644 --- a/app/Models/StandalonePostgresql.php +++ b/app/Models/StandalonePostgresql.php @@ -29,7 +29,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/app/Models/StandaloneRedis.php b/app/Models/StandaloneRedis.php index 6a44ee714..79c6572ab 100644 --- a/app/Models/StandaloneRedis.php +++ b/app/Models/StandaloneRedis.php @@ -24,7 +24,6 @@ protected static function booted() 'host_path' => null, 'resource_id' => $database->id, 'resource_type' => $database->getMorphClass(), - 'is_readonly' => true, ]); }); static::forceDeleting(function ($database) { diff --git a/resources/views/livewire/project/shared/storages/all.blade.php b/resources/views/livewire/project/shared/storages/all.blade.php index 4ed1d1b52..45dad78b1 100644 --- a/resources/views/livewire/project/shared/storages/all.blade.php +++ b/resources/views/livewire/project/shared/storages/all.blade.php @@ -6,8 +6,7 @@ :resource="$resource" :isFirst="$loop->first" isService='true' /> @else + :resource="$resource" startedAt="{{ data_get($resource, 'started_at') }}" /> @endif @endforeach From d9ebf3b142421a28f07f1578cf844acc56b9af01 Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Wed, 10 Sep 2025 19:35:53 +0200 Subject: [PATCH 19/50] refactor(webhook): remove Webhook model and related logic; add migrations to drop webhooks and kubernetes tables --- app/Http/Controllers/Webhook/Stripe.php | 12 ------- app/Models/Kubernetes.php | 5 --- app/Models/Webhook.php | 15 --------- ...ly_from_local_persistent_volumes_table.php | 28 +++++++++++++++++ .../2025_09_10_173300_drop_webhooks_table.php | 31 +++++++++++++++++++ ...025_09_10_173402_drop_kubernetes_table.php | 28 +++++++++++++++++ 6 files changed, 87 insertions(+), 32 deletions(-) delete mode 100644 app/Models/Kubernetes.php delete mode 100644 app/Models/Webhook.php create mode 100644 database/migrations/2025_09_10_172952_remove_is_readonly_from_local_persistent_volumes_table.php create mode 100644 database/migrations/2025_09_10_173300_drop_webhooks_table.php create mode 100644 database/migrations/2025_09_10_173402_drop_kubernetes_table.php diff --git a/app/Http/Controllers/Webhook/Stripe.php b/app/Http/Controllers/Webhook/Stripe.php index 83ba16699..ae50aac42 100644 --- a/app/Http/Controllers/Webhook/Stripe.php +++ b/app/Http/Controllers/Webhook/Stripe.php @@ -4,15 +4,12 @@ use App\Http\Controllers\Controller; use App\Jobs\StripeProcessJob; -use App\Models\Webhook; use Exception; use Illuminate\Http\Request; use Illuminate\Support\Facades\Storage; class Stripe extends Controller { - protected $webhook; - public function events(Request $request) { try { @@ -40,19 +37,10 @@ public function events(Request $request) return response('Webhook received. Cool cool cool cool cool.', 200); } - $this->webhook = Webhook::create([ - 'type' => 'stripe', - 'payload' => $request->getContent(), - ]); StripeProcessJob::dispatch($event); return response('Webhook received. Cool cool cool cool cool.', 200); } catch (Exception $e) { - $this->webhook->update([ - 'status' => 'failed', - 'failure_reason' => $e->getMessage(), - ]); - return response($e->getMessage(), 400); } } diff --git a/app/Models/Kubernetes.php b/app/Models/Kubernetes.php deleted file mode 100644 index 174cb5bc8..000000000 --- a/app/Models/Kubernetes.php +++ /dev/null @@ -1,5 +0,0 @@ - 'string', - 'payload' => 'encrypted', - ]; -} diff --git a/database/migrations/2025_09_10_172952_remove_is_readonly_from_local_persistent_volumes_table.php b/database/migrations/2025_09_10_172952_remove_is_readonly_from_local_persistent_volumes_table.php new file mode 100644 index 000000000..31398bd35 --- /dev/null +++ b/database/migrations/2025_09_10_172952_remove_is_readonly_from_local_persistent_volumes_table.php @@ -0,0 +1,28 @@ +dropColumn('is_readonly'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::table('local_persistent_volumes', function (Blueprint $table) { + $table->boolean('is_readonly')->default(false); + }); + } +}; diff --git a/database/migrations/2025_09_10_173300_drop_webhooks_table.php b/database/migrations/2025_09_10_173300_drop_webhooks_table.php new file mode 100644 index 000000000..4cb1b4e70 --- /dev/null +++ b/database/migrations/2025_09_10_173300_drop_webhooks_table.php @@ -0,0 +1,31 @@ +id(); + $table->enum('status', ['pending', 'success', 'failed'])->default('pending'); + $table->string('type'); + $table->longText('payload'); + $table->longText('failure_reason')->nullable(); + $table->timestamps(); + }); + } +}; diff --git a/database/migrations/2025_09_10_173402_drop_kubernetes_table.php b/database/migrations/2025_09_10_173402_drop_kubernetes_table.php new file mode 100644 index 000000000..329ed0e7e --- /dev/null +++ b/database/migrations/2025_09_10_173402_drop_kubernetes_table.php @@ -0,0 +1,28 @@ +id(); + $table->string('uuid')->unique(); + $table->timestamps(); + }); + } +}; From 3e9dd6a7bfaaf7003afed543bad003a08f40aabe Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Wed, 10 Sep 2025 19:56:15 +0200 Subject: [PATCH 20/50] chore: remove webhooks table cleanup --- app/Console/Commands/CleanupDatabase.php | 8 -------- 1 file changed, 8 deletions(-) diff --git a/app/Console/Commands/CleanupDatabase.php b/app/Console/Commands/CleanupDatabase.php index 2ccb76529..347ea9419 100644 --- a/app/Console/Commands/CleanupDatabase.php +++ b/app/Console/Commands/CleanupDatabase.php @@ -64,13 +64,5 @@ public function handle() if ($this->option('yes')) { $scheduled_task_executions->delete(); } - - // Cleanup webhooks table - $webhooks = DB::table('webhooks')->where('created_at', '<', now()->subDays($keep_days)); - $count = $webhooks->count(); - echo "Delete $count entries from webhooks.\n"; - if ($this->option('yes')) { - $webhooks->delete(); - } } } From b140aa19969715ee1f8739ed9b664d920004ace5 Mon Sep 17 00:00:00 2001 From: peaklabs-dev <122374094+peaklabs-dev@users.noreply.github.com> Date: Wed, 10 Sep 2025 19:58:10 +0200 Subject: [PATCH 21/50] chore(deps): bump minio and Nixpacks version --- docker/coolify-helper/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/coolify-helper/Dockerfile b/docker/coolify-helper/Dockerfile index c66b8d67e..3ea3d8793 100644 --- a/docker/coolify-helper/Dockerfile +++ b/docker/coolify-helper/Dockerfile @@ -10,9 +10,9 @@ ARG DOCKER_BUILDX_VERSION=0.25.0 # https://github.com/buildpacks/pack/releases ARG PACK_VERSION=0.38.2 # https://github.com/railwayapp/nixpacks/releases -ARG NIXPACKS_VERSION=1.39.0 +ARG NIXPACKS_VERSION=1.40.0 # https://github.com/minio/mc/releases -ARG MINIO_VERSION=RELEASE.2025-03-12T17-29-24Z +ARG MINIO_VERSION=RELEASE.2025-08-13T08-35-41Z FROM minio/mc:${MINIO_VERSION} AS minio-client From e74da06465460d5f3446fe24e33156b8826655cf Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:38:08 +0200 Subject: [PATCH 22/50] refactor(clone): consolidate application cloning logic into a dedicated function for improved maintainability and readability --- app/Livewire/Project/CloneMe.php | 141 +------------- .../Project/Shared/ResourceOperations.php | 141 +------------- bootstrap/helpers/applications.php | 176 ++++++++++++++++++ 3 files changed, 180 insertions(+), 278 deletions(-) diff --git a/app/Livewire/Project/CloneMe.php b/app/Livewire/Project/CloneMe.php index be9de139f..a4f50ee06 100644 --- a/app/Livewire/Project/CloneMe.php +++ b/app/Livewire/Project/CloneMe.php @@ -2,7 +2,6 @@ namespace App\Livewire\Project; -use App\Actions\Application\StopApplication; use App\Actions\Database\StartDatabase; use App\Actions\Database\StopDatabase; use App\Actions\Service\StartService; @@ -128,144 +127,10 @@ public function clone(string $type) $databases = $this->environment->databases(); $services = $this->environment->services; foreach ($applications as $application) { - $applicationSettings = $application->settings; - - $uuid = (string) new Cuid2; - $url = $application->fqdn; - if ($this->server->proxyType() !== 'NONE' && $applicationSettings->is_container_label_readonly_enabled === true) { - $url = generateUrl(server: $this->server, random: $uuid); - } - - $newApplication = $application->replicate([ - 'id', - 'created_at', - 'updated_at', - 'additional_servers_count', - 'additional_networks_count', - ])->fill([ - 'uuid' => $uuid, - 'fqdn' => $url, - 'status' => 'exited', + $selectedDestination = $this->servers->flatMap(fn ($server) => $server->destinations)->where('id', $this->selectedDestination)->first(); + clone_application($application, $selectedDestination, [ 'environment_id' => $environment->id, - 'destination_id' => $this->selectedDestination, - ]); - $newApplication->save(); - - if ($newApplication->destination->server->proxyType() !== 'NONE' && $applicationSettings->is_container_label_readonly_enabled === true) { - $customLabels = str(implode('|coolify|', generateLabelsApplication($newApplication)))->replace('|coolify|', "\n"); - $newApplication->custom_labels = base64_encode($customLabels); - $newApplication->save(); - } - - $newApplication->settings()->delete(); - if ($applicationSettings) { - $newApplicationSettings = $applicationSettings->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'application_id' => $newApplication->id, - ]); - $newApplicationSettings->save(); - } - - $tags = $application->tags; - foreach ($tags as $tag) { - $newApplication->tags()->attach($tag->id); - } - - $scheduledTasks = $application->scheduled_tasks()->get(); - foreach ($scheduledTasks as $task) { - $newTask = $task->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'uuid' => (string) new Cuid2, - 'application_id' => $newApplication->id, - 'team_id' => currentTeam()->id, - ]); - $newTask->save(); - } - - $applicationPreviews = $application->previews()->get(); - foreach ($applicationPreviews as $preview) { - $newPreview = $preview->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'application_id' => $newApplication->id, - 'status' => 'exited', - ]); - $newPreview->save(); - } - - $persistentVolumes = $application->persistentStorages()->get(); - foreach ($persistentVolumes as $volume) { - $newName = ''; - if (str_starts_with($volume->name, $application->uuid)) { - $newName = str($volume->name)->replace($application->uuid, $newApplication->uuid); - } else { - $newName = $newApplication->uuid.'-'.$volume->name; - } - - $newPersistentVolume = $volume->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'name' => $newName, - 'resource_id' => $newApplication->id, - ]); - $newPersistentVolume->save(); - - if ($this->cloneVolumeData) { - try { - StopApplication::dispatch($application, false, false); - $sourceVolume = $volume->name; - $targetVolume = $newPersistentVolume->name; - $sourceServer = $application->destination->server; - $targetServer = $newApplication->destination->server; - - VolumeCloneJob::dispatch($sourceVolume, $targetVolume, $sourceServer, $targetServer, $newPersistentVolume); - - queue_application_deployment( - deployment_uuid: (string) new Cuid2, - application: $application, - server: $sourceServer, - destination: $application->destination, - no_questions_asked: true - ); - } catch (\Exception $e) { - \Log::error('Failed to copy volume data for '.$volume->name.': '.$e->getMessage()); - } - } - } - - $fileStorages = $application->fileStorages()->get(); - foreach ($fileStorages as $storage) { - $newStorage = $storage->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'resource_id' => $newApplication->id, - ]); - $newStorage->save(); - } - - $environmentVaribles = $application->environment_variables()->get(); - foreach ($environmentVaribles as $environmentVarible) { - $newEnvironmentVariable = $environmentVarible->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'resourceable_id' => $newApplication->id, - ]); - $newEnvironmentVariable->save(); - } + ], $this->cloneVolumeData); } foreach ($databases as $database) { diff --git a/app/Livewire/Project/Shared/ResourceOperations.php b/app/Livewire/Project/Shared/ResourceOperations.php index 28a6380d5..47b3534a2 100644 --- a/app/Livewire/Project/Shared/ResourceOperations.php +++ b/app/Livewire/Project/Shared/ResourceOperations.php @@ -2,7 +2,6 @@ namespace App\Livewire\Project\Shared; -use App\Actions\Application\StopApplication; use App\Actions\Database\StartDatabase; use App\Actions\Database\StopDatabase; use App\Actions\Service\StartService; @@ -61,145 +60,7 @@ public function cloneTo($destination_id) $server = $new_destination->server; if ($this->resource->getMorphClass() === \App\Models\Application::class) { - $name = 'clone-of-'.str($this->resource->name)->limit(20).'-'.$uuid; - $applicationSettings = $this->resource->settings; - $url = $this->resource->fqdn; - - if ($server->proxyType() !== 'NONE' && $applicationSettings->is_container_label_readonly_enabled === true) { - $url = generateUrl(server: $server, random: $uuid); - } - - $new_resource = $this->resource->replicate([ - 'id', - 'created_at', - 'updated_at', - 'additional_servers_count', - 'additional_networks_count', - ])->fill([ - 'uuid' => $uuid, - 'name' => $name, - 'fqdn' => $url, - 'status' => 'exited', - 'destination_id' => $new_destination->id, - ]); - $new_resource->save(); - - if ($new_resource->destination->server->proxyType() !== 'NONE' && $applicationSettings->is_container_label_readonly_enabled === true) { - $customLabels = str(implode('|coolify|', generateLabelsApplication($new_resource)))->replace('|coolify|', "\n"); - $new_resource->custom_labels = base64_encode($customLabels); - $new_resource->save(); - } - - $new_resource->settings()->delete(); - if ($applicationSettings) { - $newApplicationSettings = $applicationSettings->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'application_id' => $new_resource->id, - ]); - $newApplicationSettings->save(); - } - - $tags = $this->resource->tags; - foreach ($tags as $tag) { - $new_resource->tags()->attach($tag->id); - } - - $scheduledTasks = $this->resource->scheduled_tasks()->get(); - foreach ($scheduledTasks as $task) { - $newTask = $task->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'uuid' => (string) new Cuid2, - 'application_id' => $new_resource->id, - 'team_id' => currentTeam()->id, - ]); - $newTask->save(); - } - - $applicationPreviews = $this->resource->previews()->get(); - foreach ($applicationPreviews as $preview) { - $newPreview = $preview->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'application_id' => $new_resource->id, - 'status' => 'exited', - ]); - $newPreview->save(); - } - - $persistentVolumes = $this->resource->persistentStorages()->get(); - foreach ($persistentVolumes as $volume) { - $newName = ''; - if (str_starts_with($volume->name, $this->resource->uuid)) { - $newName = str($volume->name)->replace($this->resource->uuid, $new_resource->uuid); - } else { - $newName = $new_resource->uuid.'-'.str($volume->name)->afterLast('-'); - } - - $newPersistentVolume = $volume->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'name' => $newName, - 'resource_id' => $new_resource->id, - ]); - $newPersistentVolume->save(); - - if ($this->cloneVolumeData) { - try { - StopApplication::dispatch($this->resource, false, false); - $sourceVolume = $volume->name; - $targetVolume = $newPersistentVolume->name; - $sourceServer = $this->resource->destination->server; - $targetServer = $new_resource->destination->server; - - VolumeCloneJob::dispatch($sourceVolume, $targetVolume, $sourceServer, $targetServer, $newPersistentVolume); - - queue_application_deployment( - deployment_uuid: (string) new Cuid2, - application: $this->resource, - server: $sourceServer, - destination: $this->resource->destination, - no_questions_asked: true - ); - } catch (\Exception $e) { - \Log::error('Failed to copy volume data for '.$volume->name.': '.$e->getMessage()); - } - } - } - - $fileStorages = $this->resource->fileStorages()->get(); - foreach ($fileStorages as $storage) { - $newStorage = $storage->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'resource_id' => $new_resource->id, - ]); - $newStorage->save(); - } - - $environmentVaribles = $this->resource->environment_variables()->get(); - foreach ($environmentVaribles as $environmentVarible) { - $newEnvironmentVariable = $environmentVarible->replicate([ - 'id', - 'created_at', - 'updated_at', - ])->fill([ - 'resourceable_id' => $new_resource->id, - 'resourceable_type' => $new_resource->getMorphClass(), - ]); - $newEnvironmentVariable->save(); - } + $new_resource = clone_application($this->resource, $new_destination, ['uuid' => $uuid], $this->cloneVolumeData); $route = route('project.application.configuration', [ 'project_uuid' => $this->projectUuid, diff --git a/bootstrap/helpers/applications.php b/bootstrap/helpers/applications.php index 87c24dbc6..2ae641a2b 100644 --- a/bootstrap/helpers/applications.php +++ b/bootstrap/helpers/applications.php @@ -1,12 +1,15 @@ server; + + // Prepare name and URL + $name = $overrides['name'] ?? 'clone-of-'.str($source->name)->limit(20).'-'.$uuid; + $applicationSettings = $source->settings; + $url = $overrides['fqdn'] ?? $source->fqdn; + + if ($server->proxyType() !== 'NONE' && $applicationSettings->is_container_label_readonly_enabled === true) { + $url = generateUrl(server: $server, random: $uuid); + } + + // Clone the application + $newApplication = $source->replicate([ + 'id', + 'created_at', + 'updated_at', + 'additional_servers_count', + 'additional_networks_count', + ])->fill(array_merge([ + 'uuid' => $uuid, + 'name' => $name, + 'fqdn' => $url, + 'status' => 'exited', + 'destination_id' => $destination->id, + ], $overrides)); + $newApplication->save(); + + // Update custom labels if needed + if ($newApplication->destination->server->proxyType() !== 'NONE' && $applicationSettings->is_container_label_readonly_enabled === true) { + $customLabels = str(implode('|coolify|', generateLabelsApplication($newApplication)))->replace('|coolify|', "\n"); + $newApplication->custom_labels = base64_encode($customLabels); + $newApplication->save(); + } + + // Clone settings + $newApplication->settings()->delete(); + if ($applicationSettings) { + $newApplicationSettings = $applicationSettings->replicate([ + 'id', + 'created_at', + 'updated_at', + ])->fill([ + 'application_id' => $newApplication->id, + ]); + $newApplicationSettings->save(); + } + + // Clone tags + $tags = $source->tags; + foreach ($tags as $tag) { + $newApplication->tags()->attach($tag->id); + } + + // Clone scheduled tasks + $scheduledTasks = $source->scheduled_tasks()->get(); + foreach ($scheduledTasks as $task) { + $newTask = $task->replicate([ + 'id', + 'created_at', + 'updated_at', + ])->fill([ + 'uuid' => (string) new Cuid2, + 'application_id' => $newApplication->id, + 'team_id' => currentTeam()->id, + ]); + $newTask->save(); + } + + // Clone previews with FQDN regeneration + clone_application_previews($source, $newApplication); + + // Clone persistent volumes + $persistentVolumes = $source->persistentStorages()->get(); + foreach ($persistentVolumes as $volume) { + $newName = ''; + if (str_starts_with($volume->name, $source->uuid)) { + $newName = str($volume->name)->replace($source->uuid, $newApplication->uuid); + } else { + $newName = $newApplication->uuid.'-'.str($volume->name)->afterLast('-'); + } + + $newPersistentVolume = $volume->replicate([ + 'id', + 'created_at', + 'updated_at', + ])->fill([ + 'name' => $newName, + 'resource_id' => $newApplication->id, + ]); + $newPersistentVolume->save(); + + if ($cloneVolumeData) { + try { + StopApplication::dispatch($source, false, false); + $sourceVolume = $volume->name; + $targetVolume = $newPersistentVolume->name; + $sourceServer = $source->destination->server; + $targetServer = $newApplication->destination->server; + + VolumeCloneJob::dispatch($sourceVolume, $targetVolume, $sourceServer, $targetServer, $newPersistentVolume); + + queue_application_deployment( + deployment_uuid: (string) new Cuid2, + application: $source, + server: $sourceServer, + destination: $source->destination, + no_questions_asked: true + ); + } catch (\Exception $e) { + \Log::error('Failed to copy volume data for '.$volume->name.': '.$e->getMessage()); + } + } + } + + // Clone file storages + $fileStorages = $source->fileStorages()->get(); + foreach ($fileStorages as $storage) { + $newStorage = $storage->replicate([ + 'id', + 'created_at', + 'updated_at', + ])->fill([ + 'resource_id' => $newApplication->id, + ]); + $newStorage->save(); + } + + // Clone environment variables + $environmentVariables = $source->environment_variables()->get(); + foreach ($environmentVariables as $environmentVariable) { + $newEnvironmentVariable = $environmentVariable->replicate([ + 'id', + 'created_at', + 'updated_at', + ])->fill([ + 'resourceable_id' => $newApplication->id, + 'resourceable_type' => $newApplication->getMorphClass(), + ]); + $newEnvironmentVariable->save(); + } + + return $newApplication; +} + +function clone_application_previews(Application $sourceApplication, Application $targetApplication): void +{ + $applicationPreviews = $sourceApplication->previews()->get(); + foreach ($applicationPreviews as $preview) { + $newPreview = $preview->replicate([ + 'id', + 'created_at', + 'updated_at', + ])->fill([ + 'uuid' => (string) new Cuid2, + 'application_id' => $targetApplication->id, + 'status' => 'exited', + 'fqdn' => null, + 'docker_compose_domains' => null, + ]); + $newPreview->save(); + + // Regenerate FQDN for the cloned preview + if ($targetApplication->build_pack === 'dockercompose') { + $newPreview->generate_preview_fqdn_compose(); + } else { + $newPreview->generate_preview_fqdn(); + } + } +} From 4ac89f2ad3cf694b5282caccc31576168815c9ea Mon Sep 17 00:00:00 2001 From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com> Date: Thu, 11 Sep 2025 10:58:59 +0200 Subject: [PATCH 23/50] feat(application): display parsing version in development mode and clean up domain conflict modal markup --- .../livewire/project/application/general.blade.php | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/resources/views/livewire/project/application/general.blade.php b/resources/views/livewire/project/application/general.blade.php index 315385593..f2468c6b7 100644 --- a/resources/views/livewire/project/application/general.blade.php +++ b/resources/views/livewire/project/application/general.blade.php @@ -8,6 +8,9 @@

General

+ @if (isDev()) +
{{ $application->compose_parsing_version }}
+ @endif Save
General configuration for your application.
@@ -462,12 +465,9 @@ class="underline" href="https://coolify.io/docs/knowledge-base/docker/registry"
- - - + + + @script