rosslh/coolify
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix Docker container race condition during upgrades (#7603)

Browse source
This commit is contained in:
Andras Bacsai 2025-12-12 17:38:30 +01:00 committed by GitHub
commit a3bc59dae2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 239 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
app/Actions/Server/UpdateCoolify.php
View file

@ -30,7 +30,6 @@ public function handle($manual_update = false)
if (! $this->server) {
return;
}
CleanupDocker::dispatch($this->server, false, false);
// Fetch fresh version from CDN instead of using cache
try {
@ -117,17 +116,12 @@ public function handle($manual_update = false)
private function update()
{
$helperImage = config('constants.coolify.helper_image');
$latest_version = getHelperVersion();
instant_remote_process(["docker pull -q {$helperImage}:{$latest_version}"], $this->server, false);
$image = config('constants.coolify.registry_url').'/coollabsio/coolify:'.$this->latestVersion;
instant_remote_process(["docker pull -q $image"], $this->server, false);
$latestHelperImageVersion = getHelperVersion();
$upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
remote_process([
"curl -fsSL {$upgradeScriptUrl} -o /data/coolify/source/upgrade.sh",
"bash /data/coolify/source/upgrade.sh $this->latestVersion",
"bash /data/coolify/source/upgrade.sh $this->latestVersion $latestHelperImageVersion",
], $this->server);
}
}

48
other/nightly/upgrade.sh
View file

@ -64,9 +64,45 @@ if [ -f /root/.docker/config.json ]; then
DOCKER_CONFIG_MOUNT="-v /root/.docker/config.json:/root/.docker/config.json"
fi
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
echo "docker-compose.custom.yml detected." >>"$LOGFILE"
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
else
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
fi
# Pull all required images before stopping containers
# This ensures we don't take down the system if image pull fails (rate limits, network issues, etc.)
echo "Pulling required Docker images..." >>"$LOGFILE"
docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify:${LATEST_IMAGE}" >>"$LOGFILE" 2>&1 || { echo "Failed to pull Coolify image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}" >>"$LOGFILE" 2>&1 || { echo "Failed to pull Coolify helper image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
docker pull postgres:15-alpine >>"$LOGFILE" 2>&1 || { echo "Failed to pull PostgreSQL image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
docker pull redis:7-alpine >>"$LOGFILE" 2>&1 || { echo "Failed to pull Redis image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
# Pull realtime image - version is hardcoded in docker-compose.prod.yml, extract it or use a known version
docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-realtime:1.0.10" >>"$LOGFILE" 2>&1 || { echo "Failed to pull Coolify realtime image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
echo "All images pulled successfully." >>"$LOGFILE"
# Stop and remove existing Coolify containers to prevent conflicts
# This handles both old installations (project "source") and new ones (project "coolify")
# Use nohup to ensure the script continues even if SSH connection is lost
echo "Starting container restart sequence (detached)..." >>"$LOGFILE"
nohup bash -c "
LOGFILE='$LOGFILE'
DOCKER_CONFIG_MOUNT='$DOCKER_CONFIG_MOUNT'
REGISTRY_URL='$REGISTRY_URL'
LATEST_HELPER_VERSION='$LATEST_HELPER_VERSION'
LATEST_IMAGE='$LATEST_IMAGE'
# Stop and remove containers
echo 'Stopping existing Coolify containers...' >>\"\$LOGFILE\"
for container in coolify coolify-db coolify-redis coolify-realtime; do
if docker ps -a --format '{{.Names}}' | grep -q \"^\${container}\$\"; then
docker stop \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
docker rm \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
echo \" - Removed container: \$container\" >>\"\$LOGFILE\"
fi
done
# Start new containers
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
echo 'docker-compose.custom.yml detected.' >>\"\$LOGFILE\"
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
else
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
fi
echo 'Upgrade completed.' >>\"\$LOGFILE\"
" >>"$LOGFILE" 2>&1 &

206
scripts/upgrade.sh
View file

@ -11,23 +11,63 @@ ENV_FILE="/data/coolify/source/.env"
DATE=$(date +%Y-%m-%d-%H-%M-%S)
LOGFILE="/data/coolify/source/upgrade-${DATE}.log"
# Helper function to log with timestamp
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >>"$LOGFILE"
}
# Helper function to log section headers
log_section() {
echo "" >>"$LOGFILE"
echo "============================================================" >>"$LOGFILE"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >>"$LOGFILE"
echo "============================================================" >>"$LOGFILE"
}
echo ""
echo "=========================================="
echo " Coolify Upgrade - ${DATE}"
echo "=========================================="
echo ""
# Initialize log file with header
echo "============================================================" >>"$LOGFILE"
echo "Coolify Upgrade Log" >>"$LOGFILE"
echo "Started: $(date '+%Y-%m-%d %H:%M:%S')" >>"$LOGFILE"
echo "Target Version: ${LATEST_IMAGE}" >>"$LOGFILE"
echo "Helper Version: ${LATEST_HELPER_VERSION}" >>"$LOGFILE"
echo "Registry URL: ${REGISTRY_URL}" >>"$LOGFILE"
echo "============================================================" >>"$LOGFILE"
log_section "Step 1/6: Downloading configuration files"
echo "1/6 Downloading latest configuration files..."
log "Downloading docker-compose.yml from ${CDN}/docker-compose.yml"
curl -fsSL -L $CDN/docker-compose.yml -o /data/coolify/source/docker-compose.yml
log "Downloading docker-compose.prod.yml from ${CDN}/docker-compose.prod.yml"
curl -fsSL -L $CDN/docker-compose.prod.yml -o /data/coolify/source/docker-compose.prod.yml
log "Downloading .env.production from ${CDN}/.env.production"
curl -fsSL -L $CDN/.env.production -o /data/coolify/source/.env.production
log "Configuration files downloaded successfully"
echo " Done."
# Backup existing .env file before making any changes
if [ "$SKIP_BACKUP" != "true" ]; then
if [ -f "$ENV_FILE" ]; then
echo "Creating backup of existing .env file to .env-$DATE" >>"$LOGFILE"
echo " Creating backup of .env file..."
log "Creating backup of .env file to .env-$DATE"
cp "$ENV_FILE" "$ENV_FILE-$DATE"
log "Backup created: ${ENV_FILE}-${DATE}"
else
echo "No existing .env file found to backup" >>"$LOGFILE"
log "WARNING: No existing .env file found to backup"
fi
fi
echo "Merging .env.production values into .env" >>"$LOGFILE"
log_section "Step 2/6: Updating environment configuration"
echo ""
echo "2/6 Updating environment configuration..."
log "Merging .env.production values into .env"
awk -F '=' '!seen[$1]++' "$ENV_FILE" /data/coolify/source/.env.production > "$ENV_FILE.tmp" && mv "$ENV_FILE.tmp" "$ENV_FILE"
echo ".env file merged successfully" >>"$LOGFILE"
log "Environment file merged successfully"
update_env_var() {
local key="$1"
@ -36,37 +76,179 @@ update_env_var() {
# If variable "key=" exists but has no value, update the value of the existing line
if grep -q "^${key}=$" "$ENV_FILE"; then
sed -i "s|^${key}=$|${key}=${value}|" "$ENV_FILE"
echo " - Updated value of ${key} as the current value was empty" >>"$LOGFILE"
log "Updated ${key} (was empty)"
# If variable "key=" doesn't exist, append it to the file with value
elif ! grep -q "^${key}=" "$ENV_FILE"; then
printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
echo " - Added ${key} with default value as the variable was missing" >>"$LOGFILE"
log "Added ${key} (was missing)"
fi
}
echo "Checking and updating environment variables if necessary..." >>"$LOGFILE"
log "Checking environment variables..."
update_env_var "PUSHER_APP_ID" "$(openssl rand -hex 32)"
update_env_var "PUSHER_APP_KEY" "$(openssl rand -hex 32)"
update_env_var "PUSHER_APP_SECRET" "$(openssl rand -hex 32)"
log "Environment variables check complete"
echo " Done."
# Make sure coolify network exists
# It is created when starting Coolify with docker compose
log "Checking Docker network 'coolify'..."
if ! docker network inspect coolify >/dev/null 2>&1; then
log "Network 'coolify' does not exist, creating..."
if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then
echo "Failed to create coolify network with ipv6. Trying without ipv6..."
log "Failed to create network with IPv6, trying without IPv6..."
docker network create --attachable coolify 2>/dev/null
log "Network 'coolify' created without IPv6"
else
log "Network 'coolify' created with IPv6 support"
fi
else
log "Network 'coolify' already exists"
fi
# Check if Docker config file exists
DOCKER_CONFIG_MOUNT=""
if [ -f /root/.docker/config.json ]; then
DOCKER_CONFIG_MOUNT="-v /root/.docker/config.json:/root/.docker/config.json"
log "Docker config mount enabled: /root/.docker/config.json"
fi
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
echo "docker-compose.custom.yml detected." >>"$LOGFILE"
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
log_section "Step 3/6: Pulling Docker images"
echo ""
echo "3/6 Pulling Docker images..."
echo " This may take a few minutes depending on your connection."
echo " - Pulling Coolify image..."
log "Pulling image: ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify:${LATEST_IMAGE}"
if docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify:${LATEST_IMAGE}" >>"$LOGFILE" 2>&1; then
log "Successfully pulled Coolify image"
else
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
log "ERROR: Failed to pull Coolify image"
echo " ERROR: Failed to pull Coolify image. Aborting upgrade."
exit 1
fi
echo " - Pulling Coolify helper image..."
log "Pulling image: ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"
if docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}" >>"$LOGFILE" 2>&1; then
log "Successfully pulled Coolify helper image"
else
log "ERROR: Failed to pull Coolify helper image"
echo " ERROR: Failed to pull helper image. Aborting upgrade."
exit 1
fi
echo " - Pulling PostgreSQL image..."
log "Pulling image: postgres:15-alpine"
if docker pull postgres:15-alpine >>"$LOGFILE" 2>&1; then
log "Successfully pulled PostgreSQL image"
else
log "ERROR: Failed to pull PostgreSQL image"
echo " ERROR: Failed to pull PostgreSQL image. Aborting upgrade."
exit 1
fi
echo " - Pulling Redis image..."
log "Pulling image: redis:7-alpine"
if docker pull redis:7-alpine >>"$LOGFILE" 2>&1; then
log "Successfully pulled Redis image"
else
log "ERROR: Failed to pull Redis image"
echo " ERROR: Failed to pull Redis image. Aborting upgrade."
exit 1
fi
echo " - Pulling Coolify realtime image..."
log "Pulling image: ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-realtime:1.0.10"
if docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-realtime:1.0.10" >>"$LOGFILE" 2>&1; then
log "Successfully pulled Coolify realtime image"
else
log "ERROR: Failed to pull Coolify realtime image"
echo " ERROR: Failed to pull realtime image. Aborting upgrade."
exit 1
fi
log "All images pulled successfully"
echo " All images pulled successfully."
log_section "Step 4/6: Stopping and restarting containers"
echo ""
echo "4/6 Stopping containers and starting new ones..."
echo " This step will restart all Coolify containers."
echo " Check the log file for details: ${LOGFILE}"
# From this point forward, we need to ensure the script continues even if
# the SSH connection is lost (which happens when coolify container stops)
# We use a subshell with nohup to ensure completion
log "Starting container restart sequence (detached)..."
nohup bash -c "
LOGFILE='$LOGFILE'
DOCKER_CONFIG_MOUNT='$DOCKER_CONFIG_MOUNT'
REGISTRY_URL='$REGISTRY_URL'
LATEST_HELPER_VERSION='$LATEST_HELPER_VERSION'
LATEST_IMAGE='$LATEST_IMAGE'
log() {
echo \"[\$(date '+%Y-%m-%d %H:%M:%S')] \$1\" >>\"\$LOGFILE\"
}
# Stop and remove containers
for container in coolify coolify-db coolify-redis coolify-realtime; do
if docker ps -a --format '{{.Names}}' | grep -q \"^\${container}\$\"; then
log \"Stopping container: \${container}\"
docker stop \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
log \"Removing container: \${container}\"
docker rm \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
log \"Container \${container} stopped and removed\"
else
log \"Container \${container} not found (skipping)\"
fi
done
log \"Container cleanup complete\"
# Start new containers
echo '' >>\"\$LOGFILE\"
echo '============================================================' >>\"\$LOGFILE\"
log 'Step 5/6: Starting new containers'
echo '============================================================' >>\"\$LOGFILE\"
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
log 'Using custom docker-compose.yml'
log 'Running docker compose up with custom configuration...'
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
else
log 'Using standard docker-compose configuration'
log 'Running docker compose up...'
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
fi
log 'Docker compose up completed'
# Final log entry
echo '' >>\"\$LOGFILE\"
echo '============================================================' >>\"\$LOGFILE\"
log 'Step 6/6: Upgrade complete'
echo '============================================================' >>\"\$LOGFILE\"
log 'Coolify upgrade completed successfully'
log \"Version: \${LATEST_IMAGE}\"
echo '' >>\"\$LOGFILE\"
echo '============================================================' >>\"\$LOGFILE\"
echo \"Upgrade completed: \$(date '+%Y-%m-%d %H:%M:%S')\" >>\"\$LOGFILE\"
echo '============================================================' >>\"\$LOGFILE\"
" >>"$LOGFILE" 2>&1 &
# Give the background process a moment to start
sleep 2
log "Container restart sequence started in background (PID: $!)"
echo ""
echo "5/6 Containers are being restarted in the background..."
echo "6/6 Upgrade process initiated!"
echo ""
echo "=========================================="
echo " Coolify upgrade to ${LATEST_IMAGE} in progress"
echo "=========================================="
echo ""
echo " The upgrade will continue in the background."
echo " Coolify will be available again shortly."
echo " Log file: ${LOGFILE}"