Merge branch 'next' into s3-restore
This commit is contained in:
commit
680b9a2c10
3 changed files with 530 additions and 140 deletions
|
|
@ -976,7 +976,7 @@ private function push_to_docker_registry()
|
|||
} catch (Exception $e) {
|
||||
$this->application_deployment_queue->addLogEntry('Failed to push image to docker registry. Please check debug logs for more information.');
|
||||
if ($forceFail) {
|
||||
throw new DeploymentException($e->getMessage(), 69420);
|
||||
throw new DeploymentException(get_class($e).': '.$e->getMessage(), $e->getCode(), $e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1610,123 +1610,131 @@ private function laravel_finetunes()
|
|||
|
||||
private function rolling_update()
|
||||
{
|
||||
$this->checkForCancellation();
|
||||
if ($this->server->isSwarm()) {
|
||||
$this->application_deployment_queue->addLogEntry('Rolling update started.');
|
||||
$this->execute_remote_command(
|
||||
[
|
||||
executeInDocker($this->deployment_uuid, "docker stack deploy --detach=true --with-registry-auth -c {$this->workdir}{$this->docker_compose_location} {$this->application->uuid}"),
|
||||
],
|
||||
);
|
||||
$this->application_deployment_queue->addLogEntry('Rolling update completed.');
|
||||
} else {
|
||||
if ($this->use_build_server) {
|
||||
$this->write_deployment_configurations();
|
||||
$this->server = $this->original_server;
|
||||
}
|
||||
if (count($this->application->ports_mappings_array) > 0 || (bool) $this->application->settings->is_consistent_container_name_enabled || str($this->application->settings->custom_internal_name)->isNotEmpty() || $this->pull_request_id !== 0 || str($this->application->custom_docker_run_options)->contains('--ip') || str($this->application->custom_docker_run_options)->contains('--ip6')) {
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
if (count($this->application->ports_mappings_array) > 0) {
|
||||
$this->application_deployment_queue->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.');
|
||||
}
|
||||
if ((bool) $this->application->settings->is_consistent_container_name_enabled) {
|
||||
$this->application_deployment_queue->addLogEntry('Consistent container name feature enabled, rolling update is not supported.');
|
||||
}
|
||||
if (str($this->application->settings->custom_internal_name)->isNotEmpty()) {
|
||||
$this->application_deployment_queue->addLogEntry('Custom internal name is set, rolling update is not supported.');
|
||||
}
|
||||
if ($this->pull_request_id !== 0) {
|
||||
$this->application->settings->is_consistent_container_name_enabled = true;
|
||||
$this->application_deployment_queue->addLogEntry('Pull request deployment, rolling update is not supported.');
|
||||
}
|
||||
if (str($this->application->custom_docker_run_options)->contains('--ip') || str($this->application->custom_docker_run_options)->contains('--ip6')) {
|
||||
$this->application_deployment_queue->addLogEntry('Custom IP address is set, rolling update is not supported.');
|
||||
}
|
||||
$this->stop_running_container(force: true);
|
||||
$this->start_by_compose_file();
|
||||
} else {
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
try {
|
||||
$this->checkForCancellation();
|
||||
if ($this->server->isSwarm()) {
|
||||
$this->application_deployment_queue->addLogEntry('Rolling update started.');
|
||||
$this->start_by_compose_file();
|
||||
$this->health_check();
|
||||
$this->stop_running_container();
|
||||
$this->execute_remote_command(
|
||||
[
|
||||
executeInDocker($this->deployment_uuid, "docker stack deploy --detach=true --with-registry-auth -c {$this->workdir}{$this->docker_compose_location} {$this->application->uuid}"),
|
||||
],
|
||||
);
|
||||
$this->application_deployment_queue->addLogEntry('Rolling update completed.');
|
||||
} else {
|
||||
if ($this->use_build_server) {
|
||||
$this->write_deployment_configurations();
|
||||
$this->server = $this->original_server;
|
||||
}
|
||||
if (count($this->application->ports_mappings_array) > 0 || (bool) $this->application->settings->is_consistent_container_name_enabled || str($this->application->settings->custom_internal_name)->isNotEmpty() || $this->pull_request_id !== 0 || str($this->application->custom_docker_run_options)->contains('--ip') || str($this->application->custom_docker_run_options)->contains('--ip6')) {
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
if (count($this->application->ports_mappings_array) > 0) {
|
||||
$this->application_deployment_queue->addLogEntry('Application has ports mapped to the host system, rolling update is not supported.');
|
||||
}
|
||||
if ((bool) $this->application->settings->is_consistent_container_name_enabled) {
|
||||
$this->application_deployment_queue->addLogEntry('Consistent container name feature enabled, rolling update is not supported.');
|
||||
}
|
||||
if (str($this->application->settings->custom_internal_name)->isNotEmpty()) {
|
||||
$this->application_deployment_queue->addLogEntry('Custom internal name is set, rolling update is not supported.');
|
||||
}
|
||||
if ($this->pull_request_id !== 0) {
|
||||
$this->application->settings->is_consistent_container_name_enabled = true;
|
||||
$this->application_deployment_queue->addLogEntry('Pull request deployment, rolling update is not supported.');
|
||||
}
|
||||
if (str($this->application->custom_docker_run_options)->contains('--ip') || str($this->application->custom_docker_run_options)->contains('--ip6')) {
|
||||
$this->application_deployment_queue->addLogEntry('Custom IP address is set, rolling update is not supported.');
|
||||
}
|
||||
$this->stop_running_container(force: true);
|
||||
$this->start_by_compose_file();
|
||||
} else {
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
$this->application_deployment_queue->addLogEntry('Rolling update started.');
|
||||
$this->start_by_compose_file();
|
||||
$this->health_check();
|
||||
$this->stop_running_container();
|
||||
$this->application_deployment_queue->addLogEntry('Rolling update completed.');
|
||||
}
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
throw new DeploymentException('Rolling update failed ('.get_class($e).'): '.$e->getMessage(), $e->getCode(), $e);
|
||||
}
|
||||
}
|
||||
|
||||
private function health_check()
|
||||
{
|
||||
if ($this->server->isSwarm()) {
|
||||
// Implement healthcheck for swarm
|
||||
} else {
|
||||
if ($this->application->isHealthcheckDisabled() && $this->application->custom_healthcheck_found === false) {
|
||||
$this->newVersionIsHealthy = true;
|
||||
try {
|
||||
if ($this->server->isSwarm()) {
|
||||
// Implement healthcheck for swarm
|
||||
} else {
|
||||
if ($this->application->isHealthcheckDisabled() && $this->application->custom_healthcheck_found === false) {
|
||||
$this->newVersionIsHealthy = true;
|
||||
|
||||
return;
|
||||
}
|
||||
if ($this->application->custom_healthcheck_found) {
|
||||
$this->application_deployment_queue->addLogEntry('Custom healthcheck found in Dockerfile.');
|
||||
}
|
||||
if ($this->container_name) {
|
||||
$counter = 1;
|
||||
$this->application_deployment_queue->addLogEntry('Waiting for healthcheck to pass on the new container.');
|
||||
if ($this->full_healthcheck_url && ! $this->application->custom_healthcheck_found) {
|
||||
$this->application_deployment_queue->addLogEntry("Healthcheck URL (inside the container): {$this->full_healthcheck_url}");
|
||||
return;
|
||||
}
|
||||
$this->application_deployment_queue->addLogEntry("Waiting for the start period ({$this->application->health_check_start_period} seconds) before starting healthcheck.");
|
||||
$sleeptime = 0;
|
||||
while ($sleeptime < $this->application->health_check_start_period) {
|
||||
Sleep::for(1)->seconds();
|
||||
$sleeptime++;
|
||||
if ($this->application->custom_healthcheck_found) {
|
||||
$this->application_deployment_queue->addLogEntry('Custom healthcheck found in Dockerfile.');
|
||||
}
|
||||
while ($counter <= $this->application->health_check_retries) {
|
||||
$this->execute_remote_command(
|
||||
[
|
||||
"docker inspect --format='{{json .State.Health.Status}}' {$this->container_name}",
|
||||
'hidden' => true,
|
||||
'save' => 'health_check',
|
||||
'append' => false,
|
||||
],
|
||||
[
|
||||
"docker inspect --format='{{json .State.Health.Log}}' {$this->container_name}",
|
||||
'hidden' => true,
|
||||
'save' => 'health_check_logs',
|
||||
'append' => false,
|
||||
],
|
||||
);
|
||||
$this->application_deployment_queue->addLogEntry("Attempt {$counter} of {$this->application->health_check_retries} | Healthcheck status: {$this->saved_outputs->get('health_check')}");
|
||||
$health_check_logs = data_get(collect(json_decode($this->saved_outputs->get('health_check_logs')))->last(), 'Output', '(no logs)');
|
||||
if (empty($health_check_logs)) {
|
||||
$health_check_logs = '(no logs)';
|
||||
if ($this->container_name) {
|
||||
$counter = 1;
|
||||
$this->application_deployment_queue->addLogEntry('Waiting for healthcheck to pass on the new container.');
|
||||
if ($this->full_healthcheck_url && ! $this->application->custom_healthcheck_found) {
|
||||
$this->application_deployment_queue->addLogEntry("Healthcheck URL (inside the container): {$this->full_healthcheck_url}");
|
||||
}
|
||||
$health_check_return_code = data_get(collect(json_decode($this->saved_outputs->get('health_check_logs')))->last(), 'ExitCode', '(no return code)');
|
||||
if ($health_check_logs !== '(no logs)' || $health_check_return_code !== '(no return code)') {
|
||||
$this->application_deployment_queue->addLogEntry("Healthcheck logs: {$health_check_logs} | Return code: {$health_check_return_code}");
|
||||
}
|
||||
|
||||
if (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'healthy') {
|
||||
$this->newVersionIsHealthy = true;
|
||||
$this->application->update(['status' => 'running']);
|
||||
$this->application_deployment_queue->addLogEntry('New container is healthy.');
|
||||
break;
|
||||
}
|
||||
if (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') {
|
||||
$this->newVersionIsHealthy = false;
|
||||
$this->query_logs();
|
||||
break;
|
||||
}
|
||||
$counter++;
|
||||
$this->application_deployment_queue->addLogEntry("Waiting for the start period ({$this->application->health_check_start_period} seconds) before starting healthcheck.");
|
||||
$sleeptime = 0;
|
||||
while ($sleeptime < $this->application->health_check_interval) {
|
||||
while ($sleeptime < $this->application->health_check_start_period) {
|
||||
Sleep::for(1)->seconds();
|
||||
$sleeptime++;
|
||||
}
|
||||
}
|
||||
if (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'starting') {
|
||||
$this->query_logs();
|
||||
while ($counter <= $this->application->health_check_retries) {
|
||||
$this->execute_remote_command(
|
||||
[
|
||||
"docker inspect --format='{{json .State.Health.Status}}' {$this->container_name}",
|
||||
'hidden' => true,
|
||||
'save' => 'health_check',
|
||||
'append' => false,
|
||||
],
|
||||
[
|
||||
"docker inspect --format='{{json .State.Health.Log}}' {$this->container_name}",
|
||||
'hidden' => true,
|
||||
'save' => 'health_check_logs',
|
||||
'append' => false,
|
||||
],
|
||||
);
|
||||
$this->application_deployment_queue->addLogEntry("Attempt {$counter} of {$this->application->health_check_retries} | Healthcheck status: {$this->saved_outputs->get('health_check')}");
|
||||
$health_check_logs = data_get(collect(json_decode($this->saved_outputs->get('health_check_logs')))->last(), 'Output', '(no logs)');
|
||||
if (empty($health_check_logs)) {
|
||||
$health_check_logs = '(no logs)';
|
||||
}
|
||||
$health_check_return_code = data_get(collect(json_decode($this->saved_outputs->get('health_check_logs')))->last(), 'ExitCode', '(no return code)');
|
||||
if ($health_check_logs !== '(no logs)' || $health_check_return_code !== '(no return code)') {
|
||||
$this->application_deployment_queue->addLogEntry("Healthcheck logs: {$health_check_logs} | Return code: {$health_check_return_code}");
|
||||
}
|
||||
|
||||
if (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'healthy') {
|
||||
$this->newVersionIsHealthy = true;
|
||||
$this->application->update(['status' => 'running']);
|
||||
$this->application_deployment_queue->addLogEntry('New container is healthy.');
|
||||
break;
|
||||
}
|
||||
if (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'unhealthy') {
|
||||
$this->newVersionIsHealthy = false;
|
||||
$this->query_logs();
|
||||
break;
|
||||
}
|
||||
$counter++;
|
||||
$sleeptime = 0;
|
||||
while ($sleeptime < $this->application->health_check_interval) {
|
||||
Sleep::for(1)->seconds();
|
||||
$sleeptime++;
|
||||
}
|
||||
}
|
||||
if (str($this->saved_outputs->get('health_check'))->replace('"', '')->value() === 'starting') {
|
||||
$this->query_logs();
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception $e) {
|
||||
throw new DeploymentException('Health check failed ('.get_class($e).'): '.$e->getMessage(), $e->getCode(), $e);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3034,58 +3042,66 @@ private function graceful_shutdown_container(string $containerName)
|
|||
|
||||
private function stop_running_container(bool $force = false)
|
||||
{
|
||||
$this->application_deployment_queue->addLogEntry('Removing old containers.');
|
||||
if ($this->newVersionIsHealthy || $force) {
|
||||
if ($this->application->settings->is_consistent_container_name_enabled || str($this->application->settings->custom_internal_name)->isNotEmpty()) {
|
||||
$this->graceful_shutdown_container($this->container_name);
|
||||
} else {
|
||||
$containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id);
|
||||
if ($this->pull_request_id === 0) {
|
||||
$containers = $containers->filter(function ($container) {
|
||||
return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== addPreviewDeploymentSuffix($this->container_name, $this->pull_request_id);
|
||||
try {
|
||||
$this->application_deployment_queue->addLogEntry('Removing old containers.');
|
||||
if ($this->newVersionIsHealthy || $force) {
|
||||
if ($this->application->settings->is_consistent_container_name_enabled || str($this->application->settings->custom_internal_name)->isNotEmpty()) {
|
||||
$this->graceful_shutdown_container($this->container_name);
|
||||
} else {
|
||||
$containers = getCurrentApplicationContainerStatus($this->server, $this->application->id, $this->pull_request_id);
|
||||
if ($this->pull_request_id === 0) {
|
||||
$containers = $containers->filter(function ($container) {
|
||||
return data_get($container, 'Names') !== $this->container_name && data_get($container, 'Names') !== addPreviewDeploymentSuffix($this->container_name, $this->pull_request_id);
|
||||
});
|
||||
}
|
||||
$containers->each(function ($container) {
|
||||
$this->graceful_shutdown_container(data_get($container, 'Names'));
|
||||
});
|
||||
}
|
||||
$containers->each(function ($container) {
|
||||
$this->graceful_shutdown_container(data_get($container, 'Names'));
|
||||
});
|
||||
} else {
|
||||
if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile' || $this->application->build_pack === 'dockerimage') {
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
$this->application_deployment_queue->addLogEntry("WARNING: Dockerfile or Docker Image based deployment detected. The healthcheck needs a curl or wget command to check the health of the application. Please make sure that it is available in the image or turn off healthcheck on Coolify's UI.");
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
}
|
||||
$this->application_deployment_queue->addLogEntry('New container is not healthy, rolling back to the old container.');
|
||||
$this->failDeployment();
|
||||
$this->graceful_shutdown_container($this->container_name);
|
||||
}
|
||||
} else {
|
||||
if ($this->application->dockerfile || $this->application->build_pack === 'dockerfile' || $this->application->build_pack === 'dockerimage') {
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
$this->application_deployment_queue->addLogEntry("WARNING: Dockerfile or Docker Image based deployment detected. The healthcheck needs a curl or wget command to check the health of the application. Please make sure that it is available in the image or turn off healthcheck on Coolify's UI.");
|
||||
$this->application_deployment_queue->addLogEntry('----------------------------------------');
|
||||
}
|
||||
$this->application_deployment_queue->addLogEntry('New container is not healthy, rolling back to the old container.');
|
||||
$this->failDeployment();
|
||||
$this->graceful_shutdown_container($this->container_name);
|
||||
} catch (Exception $e) {
|
||||
throw new DeploymentException("Failed to stop running container: {$e->getMessage()}", $e->getCode(), $e);
|
||||
}
|
||||
}
|
||||
|
||||
private function start_by_compose_file()
|
||||
{
|
||||
// Ensure .env file exists before docker compose tries to load it (defensive programming)
|
||||
$this->execute_remote_command(
|
||||
["touch {$this->configuration_dir}/.env", 'hidden' => true],
|
||||
);
|
||||
|
||||
if ($this->application->build_pack === 'dockerimage') {
|
||||
$this->application_deployment_queue->addLogEntry('Pulling latest images from the registry.');
|
||||
try {
|
||||
// Ensure .env file exists before docker compose tries to load it (defensive programming)
|
||||
$this->execute_remote_command(
|
||||
[executeInDocker($this->deployment_uuid, "docker compose --project-name {$this->application->uuid} --project-directory {$this->workdir} pull"), 'hidden' => true],
|
||||
[executeInDocker($this->deployment_uuid, "{$this->coolify_variables} docker compose --project-name {$this->application->uuid} --project-directory {$this->workdir} up --build -d"), 'hidden' => true],
|
||||
["touch {$this->configuration_dir}/.env", 'hidden' => true],
|
||||
);
|
||||
} else {
|
||||
if ($this->use_build_server) {
|
||||
|
||||
if ($this->application->build_pack === 'dockerimage') {
|
||||
$this->application_deployment_queue->addLogEntry('Pulling latest images from the registry.');
|
||||
$this->execute_remote_command(
|
||||
["{$this->coolify_variables} docker compose --project-name {$this->application->uuid} --project-directory {$this->configuration_dir} -f {$this->configuration_dir}{$this->docker_compose_location} up --pull always --build -d", 'hidden' => true],
|
||||
[executeInDocker($this->deployment_uuid, "docker compose --project-name {$this->application->uuid} --project-directory {$this->workdir} pull"), 'hidden' => true],
|
||||
[executeInDocker($this->deployment_uuid, "{$this->coolify_variables} docker compose --project-name {$this->application->uuid} --project-directory {$this->workdir} up --build -d"), 'hidden' => true],
|
||||
);
|
||||
} else {
|
||||
$this->execute_remote_command(
|
||||
[executeInDocker($this->deployment_uuid, "{$this->coolify_variables} docker compose --project-name {$this->application->uuid} --project-directory {$this->workdir} -f {$this->workdir}{$this->docker_compose_location} up --build -d"), 'hidden' => true],
|
||||
);
|
||||
if ($this->use_build_server) {
|
||||
$this->execute_remote_command(
|
||||
["{$this->coolify_variables} docker compose --project-name {$this->application->uuid} --project-directory {$this->configuration_dir} -f {$this->configuration_dir}{$this->docker_compose_location} up --pull always --build -d", 'hidden' => true],
|
||||
);
|
||||
} else {
|
||||
$this->execute_remote_command(
|
||||
[executeInDocker($this->deployment_uuid, "{$this->coolify_variables} docker compose --project-name {$this->application->uuid} --project-directory {$this->workdir} -f {$this->workdir}{$this->docker_compose_location} up --build -d"), 'hidden' => true],
|
||||
);
|
||||
}
|
||||
}
|
||||
$this->application_deployment_queue->addLogEntry('New container started.');
|
||||
} catch (Exception $e) {
|
||||
throw new DeploymentException("Failed to start container: {$e->getMessage()}", $e->getCode(), $e);
|
||||
}
|
||||
$this->application_deployment_queue->addLogEntry('New container started.');
|
||||
}
|
||||
|
||||
private function analyzeBuildTimeVariables($variables)
|
||||
|
|
@ -3829,7 +3845,7 @@ private function completeDeployment(): void
|
|||
* Fail the deployment.
|
||||
* Sends failure notification and queues next deployment.
|
||||
*/
|
||||
private function failDeployment(): void
|
||||
protected function failDeployment(): void
|
||||
{
|
||||
$this->transitionToStatus(ApplicationDeploymentStatus::FAILED);
|
||||
}
|
||||
|
|
@ -3837,8 +3853,38 @@ private function failDeployment(): void
|
|||
public function failed(Throwable $exception): void
|
||||
{
|
||||
$this->failDeployment();
|
||||
|
||||
// Log comprehensive error information
|
||||
$errorMessage = $exception->getMessage() ?: 'Unknown error occurred';
|
||||
$errorCode = $exception->getCode();
|
||||
$errorClass = get_class($exception);
|
||||
|
||||
$this->application_deployment_queue->addLogEntry('========================================', 'stderr');
|
||||
$this->application_deployment_queue->addLogEntry("Deployment failed: {$errorMessage}", 'stderr');
|
||||
$this->application_deployment_queue->addLogEntry("Error type: {$errorClass}", 'stderr', hidden: true);
|
||||
$this->application_deployment_queue->addLogEntry("Error code: {$errorCode}", 'stderr', hidden: true);
|
||||
|
||||
// Log the exception file and line for debugging
|
||||
$this->application_deployment_queue->addLogEntry("Location: {$exception->getFile()}:{$exception->getLine()}", 'stderr', hidden: true);
|
||||
|
||||
// Log previous exceptions if they exist (for chained exceptions)
|
||||
$previous = $exception->getPrevious();
|
||||
if ($previous) {
|
||||
$this->application_deployment_queue->addLogEntry('Caused by:', 'stderr', hidden: true);
|
||||
$previousMessage = $previous->getMessage() ?: 'No message';
|
||||
$previousClass = get_class($previous);
|
||||
$this->application_deployment_queue->addLogEntry(" {$previousClass}: {$previousMessage}", 'stderr', hidden: true);
|
||||
$this->application_deployment_queue->addLogEntry(" at {$previous->getFile()}:{$previous->getLine()}", 'stderr', hidden: true);
|
||||
}
|
||||
|
||||
// Log first few lines of stack trace for debugging
|
||||
$trace = $exception->getTraceAsString();
|
||||
$traceLines = explode("\n", $trace);
|
||||
$this->application_deployment_queue->addLogEntry('Stack trace (first 5 lines):', 'stderr', hidden: true);
|
||||
foreach (array_slice($traceLines, 0, 5) as $traceLine) {
|
||||
$this->application_deployment_queue->addLogEntry(" {$traceLine}", 'stderr', hidden: true);
|
||||
}
|
||||
$this->application_deployment_queue->addLogEntry('========================================', 'stderr');
|
||||
|
||||
if ($this->application->build_pack !== 'dockercompose') {
|
||||
$code = $exception->getCode();
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ private function dispatchConnectionChecks(Collection $servers): void
|
|||
Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
|
||||
'server_id' => $server->id,
|
||||
'server_name' => $server->name,
|
||||
'error' => $e->getMessage(),
|
||||
'error' => get_class($e).': '.$e->getMessage(),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
|
@ -103,7 +103,7 @@ private function processScheduledTasks(Collection $servers): void
|
|||
Log::channel('scheduled-errors')->error('Error processing server tasks', [
|
||||
'server_id' => $server->id,
|
||||
'server_name' => $server->name,
|
||||
'error' => $e->getMessage(),
|
||||
'error' => get_class($e).': '.$e->getMessage(),
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
344
tests/Unit/ApplicationDeploymentErrorLoggingTest.php
Normal file
344
tests/Unit/ApplicationDeploymentErrorLoggingTest.php
Normal file
|
|
@ -0,0 +1,344 @@
|
|||
<?php
|
||||
|
||||
use App\Exceptions\DeploymentException;
|
||||
use App\Jobs\ApplicationDeploymentJob;
|
||||
use App\Models\Application;
|
||||
use App\Models\ApplicationDeploymentQueue;
|
||||
|
||||
/**
|
||||
* Test to verify that deployment errors are properly logged with comprehensive details.
|
||||
*
|
||||
* This test suite verifies the fix for issue #7113 where deployments fail without
|
||||
* clear error messages. The fix ensures that all deployment failures log:
|
||||
* - The exception message
|
||||
* - The exception type/class
|
||||
* - The exception code (if present)
|
||||
* - The file and line where the error occurred
|
||||
* - Previous exception details (if chained)
|
||||
* - Stack trace (first 5 lines)
|
||||
*/
|
||||
it('logs comprehensive error details when failed() is called', function () {
|
||||
// Create a mock exception with all properties
|
||||
$innerException = new \RuntimeException('Connection refused', 111);
|
||||
$exception = new DeploymentException(
|
||||
'Failed to start container',
|
||||
500,
|
||||
$innerException
|
||||
);
|
||||
|
||||
// Mock the application deployment queue
|
||||
$mockQueue = Mockery::mock(ApplicationDeploymentQueue::class);
|
||||
$logEntries = [];
|
||||
|
||||
// Capture all log entries
|
||||
$mockQueue->shouldReceive('addLogEntry')
|
||||
->withArgs(function ($message, $type = 'stdout', $hidden = false) use (&$logEntries) {
|
||||
$logEntries[] = ['message' => $message, 'type' => $type, 'hidden' => $hidden];
|
||||
|
||||
return true;
|
||||
})
|
||||
->atLeast()->once();
|
||||
|
||||
$mockQueue->shouldReceive('update')->andReturn(true);
|
||||
|
||||
// Mock Application and its relationships
|
||||
$mockApplication = Mockery::mock(Application::class);
|
||||
$mockApplication->shouldReceive('getAttribute')
|
||||
->with('build_pack')
|
||||
->andReturn('dockerfile');
|
||||
$mockApplication->shouldReceive('setAttribute')
|
||||
->with('build_pack', 'dockerfile')
|
||||
->andReturnSelf();
|
||||
$mockApplication->build_pack = 'dockerfile';
|
||||
|
||||
$mockSettings = Mockery::mock();
|
||||
$mockSettings->shouldReceive('getAttribute')
|
||||
->with('is_consistent_container_name_enabled')
|
||||
->andReturn(false);
|
||||
$mockSettings->shouldReceive('getAttribute')
|
||||
->with('custom_internal_name')
|
||||
->andReturn('');
|
||||
$mockSettings->shouldReceive('setAttribute')
|
||||
->andReturnSelf();
|
||||
$mockSettings->is_consistent_container_name_enabled = false;
|
||||
$mockSettings->custom_internal_name = '';
|
||||
|
||||
$mockApplication->shouldReceive('getAttribute')
|
||||
->with('settings')
|
||||
->andReturn($mockSettings);
|
||||
|
||||
// Use reflection to set private properties and call the failed() method
|
||||
$job = Mockery::mock(ApplicationDeploymentJob::class)->makePartial();
|
||||
$job->shouldAllowMockingProtectedMethods();
|
||||
|
||||
$reflection = new \ReflectionClass(ApplicationDeploymentJob::class);
|
||||
|
||||
$queueProperty = $reflection->getProperty('application_deployment_queue');
|
||||
$queueProperty->setAccessible(true);
|
||||
$queueProperty->setValue($job, $mockQueue);
|
||||
|
||||
$applicationProperty = $reflection->getProperty('application');
|
||||
$applicationProperty->setAccessible(true);
|
||||
$applicationProperty->setValue($job, $mockApplication);
|
||||
|
||||
$pullRequestProperty = $reflection->getProperty('pull_request_id');
|
||||
$pullRequestProperty->setAccessible(true);
|
||||
$pullRequestProperty->setValue($job, 0);
|
||||
|
||||
$containerNameProperty = $reflection->getProperty('container_name');
|
||||
$containerNameProperty->setAccessible(true);
|
||||
$containerNameProperty->setValue($job, 'test-container');
|
||||
|
||||
// Mock the failDeployment method to prevent errors
|
||||
$job->shouldReceive('failDeployment')->andReturn();
|
||||
$job->shouldReceive('execute_remote_command')->andReturn();
|
||||
|
||||
// Call the failed method
|
||||
$failedMethod = $reflection->getMethod('failed');
|
||||
$failedMethod->setAccessible(true);
|
||||
$failedMethod->invoke($job, $exception);
|
||||
|
||||
// Verify comprehensive error logging
|
||||
$errorMessages = array_column($logEntries, 'message');
|
||||
$errorMessageString = implode("\n", $errorMessages);
|
||||
|
||||
// Check that all critical information is logged
|
||||
expect($errorMessageString)->toContain('Deployment failed: Failed to start container');
|
||||
expect($errorMessageString)->toContain('Error type: App\Exceptions\DeploymentException');
|
||||
expect($errorMessageString)->toContain('Error code: 500');
|
||||
expect($errorMessageString)->toContain('Location:');
|
||||
expect($errorMessageString)->toContain('Caused by:');
|
||||
expect($errorMessageString)->toContain('RuntimeException: Connection refused');
|
||||
expect($errorMessageString)->toContain('Stack trace');
|
||||
|
||||
// Verify stderr type is used for error logging
|
||||
$stderrEntries = array_filter($logEntries, fn ($entry) => $entry['type'] === 'stderr');
|
||||
expect(count($stderrEntries))->toBeGreaterThan(0);
|
||||
|
||||
// Verify that the main error message is NOT hidden
|
||||
$mainErrorEntry = collect($logEntries)->first(fn ($entry) => str_contains($entry['message'], 'Deployment failed: Failed to start container'));
|
||||
expect($mainErrorEntry['hidden'])->toBeFalse();
|
||||
|
||||
// Verify that technical details ARE hidden
|
||||
$errorTypeEntry = collect($logEntries)->first(fn ($entry) => str_contains($entry['message'], 'Error type:'));
|
||||
expect($errorTypeEntry['hidden'])->toBeTrue();
|
||||
|
||||
$errorCodeEntry = collect($logEntries)->first(fn ($entry) => str_contains($entry['message'], 'Error code:'));
|
||||
expect($errorCodeEntry['hidden'])->toBeTrue();
|
||||
|
||||
$locationEntry = collect($logEntries)->first(fn ($entry) => str_contains($entry['message'], 'Location:'));
|
||||
expect($locationEntry['hidden'])->toBeTrue();
|
||||
|
||||
$stackTraceEntry = collect($logEntries)->first(fn ($entry) => str_contains($entry['message'], 'Stack trace'));
|
||||
expect($stackTraceEntry['hidden'])->toBeTrue();
|
||||
|
||||
$causedByEntry = collect($logEntries)->first(fn ($entry) => str_contains($entry['message'], 'Caused by:'));
|
||||
expect($causedByEntry['hidden'])->toBeTrue();
|
||||
});
|
||||
|
||||
it('handles exceptions with no message gracefully', function () {
|
||||
$exception = new \Exception;
|
||||
|
||||
$mockQueue = Mockery::mock(ApplicationDeploymentQueue::class);
|
||||
$logEntries = [];
|
||||
|
||||
$mockQueue->shouldReceive('addLogEntry')
|
||||
->withArgs(function ($message, $type = 'stdout', $hidden = false) use (&$logEntries) {
|
||||
$logEntries[] = ['message' => $message, 'type' => $type, 'hidden' => $hidden];
|
||||
|
||||
return true;
|
||||
})
|
||||
->atLeast()->once();
|
||||
|
||||
$mockQueue->shouldReceive('update')->andReturn(true);
|
||||
|
||||
$mockApplication = Mockery::mock(Application::class);
|
||||
$mockApplication->shouldReceive('getAttribute')
|
||||
->with('build_pack')
|
||||
->andReturn('dockerfile');
|
||||
$mockApplication->shouldReceive('setAttribute')
|
||||
->with('build_pack', 'dockerfile')
|
||||
->andReturnSelf();
|
||||
$mockApplication->build_pack = 'dockerfile';
|
||||
|
||||
$mockSettings = Mockery::mock();
|
||||
$mockSettings->shouldReceive('getAttribute')
|
||||
->with('is_consistent_container_name_enabled')
|
||||
->andReturn(false);
|
||||
$mockSettings->shouldReceive('getAttribute')
|
||||
->with('custom_internal_name')
|
||||
->andReturn('');
|
||||
$mockSettings->shouldReceive('setAttribute')
|
||||
->andReturnSelf();
|
||||
$mockSettings->is_consistent_container_name_enabled = false;
|
||||
$mockSettings->custom_internal_name = '';
|
||||
|
||||
$mockApplication->shouldReceive('getAttribute')
|
||||
->with('settings')
|
||||
->andReturn($mockSettings);
|
||||
|
||||
$job = Mockery::mock(ApplicationDeploymentJob::class)->makePartial();
|
||||
$job->shouldAllowMockingProtectedMethods();
|
||||
|
||||
$reflection = new \ReflectionClass(ApplicationDeploymentJob::class);
|
||||
|
||||
$queueProperty = $reflection->getProperty('application_deployment_queue');
|
||||
$queueProperty->setAccessible(true);
|
||||
$queueProperty->setValue($job, $mockQueue);
|
||||
|
||||
$applicationProperty = $reflection->getProperty('application');
|
||||
$applicationProperty->setAccessible(true);
|
||||
$applicationProperty->setValue($job, $mockApplication);
|
||||
|
||||
$pullRequestProperty = $reflection->getProperty('pull_request_id');
|
||||
$pullRequestProperty->setAccessible(true);
|
||||
$pullRequestProperty->setValue($job, 0);
|
||||
|
||||
$containerNameProperty = $reflection->getProperty('container_name');
|
||||
$containerNameProperty->setAccessible(true);
|
||||
$containerNameProperty->setValue($job, 'test-container');
|
||||
|
||||
$job->shouldReceive('failDeployment')->andReturn();
|
||||
$job->shouldReceive('execute_remote_command')->andReturn();
|
||||
|
||||
$failedMethod = $reflection->getMethod('failed');
|
||||
$failedMethod->setAccessible(true);
|
||||
$failedMethod->invoke($job, $exception);
|
||||
|
||||
$errorMessages = array_column($logEntries, 'message');
|
||||
$errorMessageString = implode("\n", $errorMessages);
|
||||
|
||||
// Should log "Unknown error occurred" for empty messages
|
||||
expect($errorMessageString)->toContain('Unknown error occurred');
|
||||
expect($errorMessageString)->toContain('Error type:');
|
||||
});
|
||||
|
||||
it('wraps exceptions in deployment methods with DeploymentException', function () {
|
||||
// Verify that our deployment methods wrap exceptions properly
|
||||
$originalException = new \RuntimeException('Container not found');
|
||||
|
||||
try {
|
||||
throw new DeploymentException('Failed to start container', 0, $originalException);
|
||||
} catch (DeploymentException $e) {
|
||||
expect($e->getMessage())->toBe('Failed to start container');
|
||||
expect($e->getPrevious())->toBe($originalException);
|
||||
expect($e->getPrevious()->getMessage())->toBe('Container not found');
|
||||
}
|
||||
});
|
||||
|
||||
it('logs error code 0 correctly', function () {
|
||||
// Verify that error code 0 is logged (previously skipped due to falsy check)
|
||||
$exception = new \Exception('Test error', 0);
|
||||
|
||||
$mockQueue = Mockery::mock(ApplicationDeploymentQueue::class);
|
||||
$logEntries = [];
|
||||
|
||||
$mockQueue->shouldReceive('addLogEntry')
|
||||
->withArgs(function ($message, $type = 'stdout', $hidden = false) use (&$logEntries) {
|
||||
$logEntries[] = ['message' => $message, 'type' => $type, 'hidden' => $hidden];
|
||||
|
||||
return true;
|
||||
})
|
||||
->atLeast()->once();
|
||||
|
||||
$mockQueue->shouldReceive('update')->andReturn(true);
|
||||
|
||||
$mockApplication = Mockery::mock(Application::class);
|
||||
$mockApplication->shouldReceive('getAttribute')
|
||||
->with('build_pack')
|
||||
->andReturn('dockerfile');
|
||||
$mockApplication->shouldReceive('setAttribute')
|
||||
->with('build_pack', 'dockerfile')
|
||||
->andReturnSelf();
|
||||
$mockApplication->build_pack = 'dockerfile';
|
||||
|
||||
$mockSettings = Mockery::mock();
|
||||
$mockSettings->shouldReceive('getAttribute')
|
||||
->with('is_consistent_container_name_enabled')
|
||||
->andReturn(false);
|
||||
$mockSettings->shouldReceive('getAttribute')
|
||||
->with('custom_internal_name')
|
||||
->andReturn('');
|
||||
$mockSettings->shouldReceive('setAttribute')
|
||||
->andReturnSelf();
|
||||
$mockSettings->is_consistent_container_name_enabled = false;
|
||||
$mockSettings->custom_internal_name = '';
|
||||
|
||||
$mockApplication->shouldReceive('getAttribute')
|
||||
->with('settings')
|
||||
->andReturn($mockSettings);
|
||||
|
||||
$job = Mockery::mock(ApplicationDeploymentJob::class)->makePartial();
|
||||
$job->shouldAllowMockingProtectedMethods();
|
||||
|
||||
$reflection = new \ReflectionClass(ApplicationDeploymentJob::class);
|
||||
|
||||
$queueProperty = $reflection->getProperty('application_deployment_queue');
|
||||
$queueProperty->setAccessible(true);
|
||||
$queueProperty->setValue($job, $mockQueue);
|
||||
|
||||
$applicationProperty = $reflection->getProperty('application');
|
||||
$applicationProperty->setAccessible(true);
|
||||
$applicationProperty->setValue($job, $mockApplication);
|
||||
|
||||
$pullRequestProperty = $reflection->getProperty('pull_request_id');
|
||||
$pullRequestProperty->setAccessible(true);
|
||||
$pullRequestProperty->setValue($job, 0);
|
||||
|
||||
$containerNameProperty = $reflection->getProperty('container_name');
|
||||
$containerNameProperty->setAccessible(true);
|
||||
$containerNameProperty->setValue($job, 'test-container');
|
||||
|
||||
$job->shouldReceive('failDeployment')->andReturn();
|
||||
$job->shouldReceive('execute_remote_command')->andReturn();
|
||||
|
||||
$failedMethod = $reflection->getMethod('failed');
|
||||
$failedMethod->setAccessible(true);
|
||||
$failedMethod->invoke($job, $exception);
|
||||
|
||||
$errorMessages = array_column($logEntries, 'message');
|
||||
$errorMessageString = implode("\n", $errorMessages);
|
||||
|
||||
// Should log error code 0 (not skip it)
|
||||
expect($errorMessageString)->toContain('Error code: 0');
|
||||
});
|
||||
|
||||
it('preserves original exception type in wrapped DeploymentException messages', function () {
|
||||
// Verify that when wrapping exceptions, the original exception type is included in the message
|
||||
$originalException = new \RuntimeException('Connection timeout');
|
||||
|
||||
// Test rolling update scenario
|
||||
$wrappedException = new DeploymentException(
|
||||
'Rolling update failed ('.get_class($originalException).'): '.$originalException->getMessage(),
|
||||
$originalException->getCode(),
|
||||
$originalException
|
||||
);
|
||||
|
||||
expect($wrappedException->getMessage())->toContain('RuntimeException');
|
||||
expect($wrappedException->getMessage())->toContain('Connection timeout');
|
||||
expect($wrappedException->getPrevious())->toBe($originalException);
|
||||
|
||||
// Test health check scenario
|
||||
$healthCheckException = new \InvalidArgumentException('Invalid health check URL');
|
||||
$wrappedHealthCheck = new DeploymentException(
|
||||
'Health check failed ('.get_class($healthCheckException).'): '.$healthCheckException->getMessage(),
|
||||
$healthCheckException->getCode(),
|
||||
$healthCheckException
|
||||
);
|
||||
|
||||
expect($wrappedHealthCheck->getMessage())->toContain('InvalidArgumentException');
|
||||
expect($wrappedHealthCheck->getMessage())->toContain('Invalid health check URL');
|
||||
expect($wrappedHealthCheck->getPrevious())->toBe($healthCheckException);
|
||||
|
||||
// Test docker registry push scenario
|
||||
$registryException = new \RuntimeException('Failed to authenticate');
|
||||
$wrappedRegistry = new DeploymentException(
|
||||
get_class($registryException).': '.$registryException->getMessage(),
|
||||
$registryException->getCode(),
|
||||
$registryException
|
||||
);
|
||||
|
||||
expect($wrappedRegistry->getMessage())->toContain('RuntimeException');
|
||||
expect($wrappedRegistry->getMessage())->toContain('Failed to authenticate');
|
||||
expect($wrappedRegistry->getPrevious())->toBe($registryException);
|
||||
});
|
||||
Loading…
Reference in a new issue