Fix deployment marked as failed after healthy container rolling update (#7583)

This commit is contained in:
Andras Bacsai 2025-12-11 10:01:00 +01:00 committed by GitHub
commit 07153de68d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 38 additions and 7 deletions

View file

@ -486,15 +486,38 @@ private function decide_what_to_do()
private function post_deployment()
{
GetContainersStatus::dispatch($this->server);
// Mark deployment as complete FIRST, before any other operations
// This ensures the deployment status is FINISHED even if subsequent operations fail
$this->completeDeployment();
// Then handle side effects - these should not fail the deployment
try {
GetContainersStatus::dispatch($this->server);
} catch (\Exception $e) {
\Log::warning('Failed to dispatch GetContainersStatus for deployment '.$this->deployment_uuid.': '.$e->getMessage());
}
if ($this->pull_request_id !== 0) {
if ($this->application->is_github_based()) {
ApplicationPullRequestUpdateJob::dispatch(application: $this->application, preview: $this->preview, deployment_uuid: $this->deployment_uuid, status: ProcessStatus::FINISHED);
try {
ApplicationPullRequestUpdateJob::dispatch(application: $this->application, preview: $this->preview, deployment_uuid: $this->deployment_uuid, status: ProcessStatus::FINISHED);
} catch (\Exception $e) {
\Log::warning('Failed to dispatch PR update for deployment '.$this->deployment_uuid.': '.$e->getMessage());
}
}
}
$this->run_post_deployment_command();
$this->application->isConfigurationChanged(true);
try {
$this->run_post_deployment_command();
} catch (\Exception $e) {
\Log::warning('Post deployment command failed for '.$this->deployment_uuid.': '.$e->getMessage());
}
try {
$this->application->isConfigurationChanged(true);
} catch (\Exception $e) {
\Log::warning('Failed to mark configuration as changed for deployment '.$this->deployment_uuid.': '.$e->getMessage());
}
}
private function deploy_simple_dockerfile()
@ -3934,13 +3957,17 @@ private function transitionToStatus(ApplicationDeploymentStatus $status): void
}
/**
* Check if deployment is in a terminal state (FAILED or CANCELLED).
* Check if deployment is in a terminal state (FINISHED, FAILED or CANCELLED).
* Terminal states cannot be changed.
*/
private function isInTerminalState(): bool
{
$this->application_deployment_queue->refresh();
if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::FINISHED->value) {
return true;
}
if ($this->application_deployment_queue->status === ApplicationDeploymentStatus::FAILED->value) {
return true;
}

View file

@ -140,9 +140,13 @@ public function execute_remote_command(...$commands)
// If we exhausted all retries and still failed
if (! $commandExecuted && $lastError) {
// Now we can set the status to FAILED since all retries have been exhausted
// But only if the deployment hasn't already been marked as FINISHED
if (isset($this->application_deployment_queue)) {
$this->application_deployment_queue->status = ApplicationDeploymentStatus::FAILED->value;
$this->application_deployment_queue->save();
$this->application_deployment_queue->refresh();
if ($this->application_deployment_queue->status !== ApplicationDeploymentStatus::FINISHED->value) {
$this->application_deployment_queue->status = ApplicationDeploymentStatus::FAILED->value;
$this->application_deployment_queue->save();
}
}
throw $lastError;
}