ac36a59088
Store s3_storage_id on scheduled volume backup executions so retention and recovery use the S3 that received the upload. Extract DeleteScheduledVolumeBackup for UI and resource deletion, and isolate database backup retention failures so cleanup errors do not fail the job.
878 lines
43 KiB
PHP
878 lines
43 KiB
PHP
<?php
|
|
|
|
namespace App\Jobs;
|
|
|
|
use App\Events\BackupCreated;
|
|
use App\Models\S3Storage;
|
|
use App\Models\ScheduledDatabaseBackup;
|
|
use App\Models\ScheduledDatabaseBackupExecution;
|
|
use App\Models\Server;
|
|
use App\Models\ServiceDatabase;
|
|
use App\Models\StandaloneClickhouse;
|
|
use App\Models\StandaloneMariadb;
|
|
use App\Models\StandaloneMongodb;
|
|
use App\Models\StandaloneMysql;
|
|
use App\Models\StandalonePostgresql;
|
|
use App\Models\Team;
|
|
use App\Notifications\Database\BackupFailed;
|
|
use App\Notifications\Database\BackupSuccess;
|
|
use App\Notifications\Database\BackupSuccessWithS3Warning;
|
|
use App\Rules\SafeWebhookUrl;
|
|
use App\Support\ClickhouseBackupCommand;
|
|
use Carbon\Carbon;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Illuminate\Support\Str;
|
|
use Throwable;
|
|
|
|
class DatabaseBackupJob implements ShouldBeEncrypted, ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
public $maxExceptions = 1;
|
|
|
|
public ?Team $team = null;
|
|
|
|
public Server $server;
|
|
|
|
public StandalonePostgresql|StandaloneMongodb|StandaloneMysql|StandaloneMariadb|StandaloneClickhouse|ServiceDatabase $database;
|
|
|
|
public ?string $container_name = null;
|
|
|
|
public ?string $directory_name = null;
|
|
|
|
public ?ScheduledDatabaseBackupExecution $backup_log = null;
|
|
|
|
public string $backup_status = 'failed';
|
|
|
|
public ?string $backup_location = null;
|
|
|
|
public string $backup_dir;
|
|
|
|
public string $backup_file;
|
|
|
|
public int $size = 0;
|
|
|
|
public ?string $backup_output = null;
|
|
|
|
public ?string $error_output = null;
|
|
|
|
public bool $s3_uploaded = false;
|
|
|
|
public ?string $postgres_password = null;
|
|
|
|
public ?string $mongo_root_username = null;
|
|
|
|
public ?string $mongo_root_password = null;
|
|
|
|
public ?S3Storage $s3 = null;
|
|
|
|
public $timeout = 3600;
|
|
|
|
public ?string $backup_log_uuid = null;
|
|
|
|
public function __construct(public ScheduledDatabaseBackup $backup)
|
|
{
|
|
$this->onQueue(crons_queue());
|
|
$this->timeout = $backup->timeout ?? 3600;
|
|
}
|
|
|
|
public function middleware(): array
|
|
{
|
|
$expireAfter = ($this->backup->timeout ?? 3600) + 300;
|
|
|
|
return [(new WithoutOverlapping('database-backup-'.$this->backup->id))->expireAfter($expireAfter)->dontRelease()];
|
|
}
|
|
|
|
public function handle(): void
|
|
{
|
|
try {
|
|
$databasesToBackup = null;
|
|
|
|
$this->team = Team::find($this->backup->team_id);
|
|
if (! $this->team) {
|
|
$this->backup->delete();
|
|
|
|
return;
|
|
}
|
|
if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
|
|
$this->database = data_get($this->backup, 'database');
|
|
$this->server = $this->database->service->server;
|
|
$this->s3 = $this->backup->s3;
|
|
} else {
|
|
$this->database = data_get($this->backup, 'database');
|
|
$this->server = $this->database->destination->server;
|
|
$this->s3 = $this->backup->s3;
|
|
}
|
|
if (is_null($this->server)) {
|
|
throw new \Exception('Server not found?!');
|
|
}
|
|
if (is_null($this->database)) {
|
|
throw new \Exception('Database not found?!');
|
|
}
|
|
|
|
$this->markStaleExecutionsAsFailed();
|
|
|
|
BackupCreated::dispatch($this->team->id);
|
|
|
|
$status = str(data_get($this->database, 'status'));
|
|
if (! $status->startsWith('running') && $this->database->id !== 0) {
|
|
Log::info('DatabaseBackupJob skipped: database not running', [
|
|
'backup_id' => $this->backup->id,
|
|
'database_id' => $this->database->id,
|
|
'status' => (string) $status,
|
|
]);
|
|
|
|
return;
|
|
}
|
|
if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
|
|
$databaseType = $this->database->databaseType();
|
|
$serviceUuid = $this->database->service->uuid;
|
|
$serviceName = str($this->database->service->name)->slug();
|
|
if (str($databaseType)->contains('postgres')) {
|
|
$this->container_name = "{$this->database->name}-$serviceUuid";
|
|
$this->directory_name = $serviceName.'-'.$this->container_name;
|
|
$commands[] = "docker exec $this->container_name env | grep POSTGRES_";
|
|
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
|
|
$envs = str($envs)->explode("\n");
|
|
|
|
$user = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('POSTGRES_USER=');
|
|
})->first();
|
|
if ($user) {
|
|
$this->database->postgres_user = str($user)->after('POSTGRES_USER=')->value();
|
|
} else {
|
|
$this->database->postgres_user = 'postgres';
|
|
}
|
|
|
|
$db = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('POSTGRES_DB=');
|
|
})->first();
|
|
|
|
if ($db) {
|
|
$databasesToBackup = str($db)->after('POSTGRES_DB=')->value();
|
|
} else {
|
|
$databasesToBackup = $this->database->postgres_user;
|
|
}
|
|
$this->postgres_password = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('POSTGRES_PASSWORD=');
|
|
})->first();
|
|
if ($this->postgres_password) {
|
|
$this->postgres_password = str($this->postgres_password)->after('POSTGRES_PASSWORD=')->value();
|
|
}
|
|
} elseif (str($databaseType)->contains('mysql')) {
|
|
$this->container_name = "{$this->database->name}-$serviceUuid";
|
|
$this->directory_name = $serviceName.'-'.$this->container_name;
|
|
$commands[] = "docker exec $this->container_name env | grep MYSQL_";
|
|
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
|
|
$envs = str($envs)->explode("\n");
|
|
|
|
$rootPassword = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MYSQL_ROOT_PASSWORD=');
|
|
})->first();
|
|
if ($rootPassword) {
|
|
$this->database->mysql_root_password = str($rootPassword)->after('MYSQL_ROOT_PASSWORD=')->value();
|
|
}
|
|
|
|
$db = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MYSQL_DATABASE=');
|
|
})->first();
|
|
|
|
if ($db) {
|
|
$databasesToBackup = str($db)->after('MYSQL_DATABASE=')->value();
|
|
} else {
|
|
throw new \Exception('MYSQL_DATABASE not found');
|
|
}
|
|
} elseif (str($databaseType)->contains('mariadb')) {
|
|
$this->container_name = "{$this->database->name}-$serviceUuid";
|
|
$this->directory_name = $serviceName.'-'.$this->container_name;
|
|
$commands[] = "docker exec $this->container_name env";
|
|
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
|
|
$envs = str($envs)->explode("\n");
|
|
$rootPassword = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MARIADB_ROOT_PASSWORD=');
|
|
})->first();
|
|
if ($rootPassword) {
|
|
$this->database->mariadb_root_password = str($rootPassword)->after('MARIADB_ROOT_PASSWORD=')->value();
|
|
} else {
|
|
$rootPassword = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MYSQL_ROOT_PASSWORD=');
|
|
})->first();
|
|
if ($rootPassword) {
|
|
$this->database->mariadb_root_password = str($rootPassword)->after('MYSQL_ROOT_PASSWORD=')->value();
|
|
}
|
|
}
|
|
|
|
$db = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MARIADB_DATABASE=');
|
|
})->first();
|
|
|
|
if ($db) {
|
|
$databasesToBackup = str($db)->after('MARIADB_DATABASE=')->value();
|
|
} else {
|
|
$db = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MYSQL_DATABASE=');
|
|
})->first();
|
|
|
|
if ($db) {
|
|
$databasesToBackup = str($db)->after('MYSQL_DATABASE=')->value();
|
|
} else {
|
|
throw new \Exception('MARIADB_DATABASE or MYSQL_DATABASE not found');
|
|
}
|
|
}
|
|
} elseif (str($databaseType)->contains('mongo')) {
|
|
$databasesToBackup = ['*'];
|
|
$this->container_name = "{$this->database->name}-$serviceUuid";
|
|
$this->directory_name = $serviceName.'-'.$this->container_name;
|
|
|
|
// Try to extract MongoDB credentials from environment variables
|
|
try {
|
|
$commands = [];
|
|
$commands[] = "docker exec $this->container_name env | grep MONGO_INITDB_";
|
|
$envs = instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
|
|
|
|
if (filled($envs)) {
|
|
$envs = str($envs)->explode("\n");
|
|
$rootPassword = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MONGO_INITDB_ROOT_PASSWORD=');
|
|
})->first();
|
|
if ($rootPassword) {
|
|
$this->mongo_root_password = str($rootPassword)->after('MONGO_INITDB_ROOT_PASSWORD=')->value();
|
|
}
|
|
$rootUsername = $envs->filter(function ($env) {
|
|
return str($env)->startsWith('MONGO_INITDB_ROOT_USERNAME=');
|
|
})->first();
|
|
if ($rootUsername) {
|
|
$this->mongo_root_username = str($rootUsername)->after('MONGO_INITDB_ROOT_USERNAME=')->value();
|
|
}
|
|
}
|
|
|
|
} catch (Throwable $e) {
|
|
// Continue without env vars - will be handled in backup_standalone_mongodb method
|
|
}
|
|
}
|
|
} else {
|
|
$databaseName = str($this->database->name)->slug()->value();
|
|
$this->container_name = $this->database->uuid;
|
|
$this->directory_name = $databaseName.'-'.$this->container_name;
|
|
$databaseType = $this->database->type();
|
|
$databasesToBackup = data_get($this->backup, 'databases_to_backup');
|
|
}
|
|
if (blank($databasesToBackup)) {
|
|
if (str($databaseType)->contains('postgres')) {
|
|
$databasesToBackup = [$this->database->postgres_db];
|
|
} elseif (str($databaseType)->contains('mongo')) {
|
|
$databasesToBackup = ['*'];
|
|
} elseif (str($databaseType)->contains('mysql')) {
|
|
$databasesToBackup = [$this->database->mysql_database];
|
|
} elseif (str($databaseType)->contains('mariadb')) {
|
|
$databasesToBackup = [$this->database->mariadb_database];
|
|
} elseif ($this->database instanceof StandaloneClickhouse) {
|
|
$databasesToBackup = [$this->database->clickhouse_db];
|
|
} else {
|
|
return;
|
|
}
|
|
} else {
|
|
if (str($databaseType)->contains('postgres')) {
|
|
// Format: db1,db2,db3
|
|
$databasesToBackup = explode(',', $databasesToBackup);
|
|
$databasesToBackup = array_map('trim', $databasesToBackup);
|
|
} elseif (str($databaseType)->contains('mongo')) {
|
|
// Format: db1:collection1,collection2|db2:collection3,collection4
|
|
// Only explode if it's a string, not if it's already an array
|
|
if (is_string($databasesToBackup)) {
|
|
$databasesToBackup = explode('|', $databasesToBackup);
|
|
$databasesToBackup = array_map('trim', $databasesToBackup);
|
|
}
|
|
} elseif (str($databaseType)->contains('mysql')) {
|
|
// Format: db1,db2,db3
|
|
$databasesToBackup = explode(',', $databasesToBackup);
|
|
$databasesToBackup = array_map('trim', $databasesToBackup);
|
|
} elseif (str($databaseType)->contains('mariadb')) {
|
|
// Format: db1,db2,db3
|
|
$databasesToBackup = explode(',', $databasesToBackup);
|
|
$databasesToBackup = array_map('trim', $databasesToBackup);
|
|
} elseif ($this->database instanceof StandaloneClickhouse) {
|
|
// Format: db1,db2,db3
|
|
$databasesToBackup = explode(',', $databasesToBackup);
|
|
$databasesToBackup = array_map('trim', $databasesToBackup);
|
|
} else {
|
|
return;
|
|
}
|
|
}
|
|
$this->backup_dir = backup_dir().'/databases/'.str($this->team->name)->slug().'-'.$this->team->id.'/'.$this->directory_name;
|
|
if ($this->database->name === 'coolify-db') {
|
|
$databasesToBackup = ['coolify'];
|
|
$this->directory_name = $this->container_name = 'coolify-db';
|
|
$ip = Str::slug($this->server->ip);
|
|
$this->backup_dir = backup_dir().'/coolify'."/coolify-db-$ip";
|
|
}
|
|
foreach ($databasesToBackup as $database) {
|
|
// Generate unique UUID for each database backup execution
|
|
$attempts = 0;
|
|
do {
|
|
$this->backup_log_uuid = new_public_id();
|
|
$exists = ScheduledDatabaseBackupExecution::where('uuid', $this->backup_log_uuid)->exists();
|
|
$attempts++;
|
|
if ($attempts >= 3 && $exists) {
|
|
throw new \Exception('Unable to generate unique UUID for backup execution after 3 attempts');
|
|
}
|
|
} while ($exists);
|
|
|
|
$size = 0;
|
|
$localBackupSucceeded = false;
|
|
$s3UploadError = null;
|
|
|
|
// Step 1: Create local backup
|
|
try {
|
|
if (str($databaseType)->contains('postgres')) {
|
|
$this->backup_file = "/pg-dump-$database-".Carbon::now()->timestamp.'.dmp';
|
|
if ($this->backup->dump_all) {
|
|
$this->backup_file = '/pg-dump-all-'.Carbon::now()->timestamp.'.gz';
|
|
}
|
|
$this->backup_location = $this->backup_dir.$this->backup_file;
|
|
$this->backup_log = ScheduledDatabaseBackupExecution::create([
|
|
'uuid' => $this->backup_log_uuid,
|
|
'database_name' => $database,
|
|
'filename' => $this->backup_location,
|
|
'scheduled_database_backup_id' => $this->backup->id,
|
|
'local_storage_deleted' => false,
|
|
]);
|
|
$this->backup_standalone_postgresql($database);
|
|
} elseif (str($databaseType)->contains('mongo')) {
|
|
if ($database === '*') {
|
|
$database = 'all';
|
|
$databaseName = 'all';
|
|
} else {
|
|
if (str($database)->contains(':')) {
|
|
$databaseName = str($database)->before(':');
|
|
} else {
|
|
$databaseName = $database;
|
|
}
|
|
}
|
|
$this->backup_file = "/mongo-dump-$databaseName-".Carbon::now()->timestamp.'.tar.gz';
|
|
$this->backup_location = $this->backup_dir.$this->backup_file;
|
|
$this->backup_log = ScheduledDatabaseBackupExecution::create([
|
|
'uuid' => $this->backup_log_uuid,
|
|
'database_name' => $databaseName,
|
|
'filename' => $this->backup_location,
|
|
'scheduled_database_backup_id' => $this->backup->id,
|
|
'local_storage_deleted' => false,
|
|
]);
|
|
$this->backup_standalone_mongodb($database);
|
|
} elseif (str($databaseType)->contains('mysql')) {
|
|
$this->backup_file = "/mysql-dump-$database-".Carbon::now()->timestamp.'.dmp';
|
|
if ($this->backup->dump_all) {
|
|
$this->backup_file = '/mysql-dump-all-'.Carbon::now()->timestamp.'.gz';
|
|
}
|
|
$this->backup_location = $this->backup_dir.$this->backup_file;
|
|
$this->backup_log = ScheduledDatabaseBackupExecution::create([
|
|
'uuid' => $this->backup_log_uuid,
|
|
'database_name' => $database,
|
|
'filename' => $this->backup_location,
|
|
'scheduled_database_backup_id' => $this->backup->id,
|
|
'local_storage_deleted' => false,
|
|
]);
|
|
$this->backup_standalone_mysql($database);
|
|
} elseif (str($databaseType)->contains('mariadb')) {
|
|
$this->backup_file = "/mariadb-dump-$database-".Carbon::now()->timestamp.'.dmp';
|
|
if ($this->backup->dump_all) {
|
|
$this->backup_file = '/mariadb-dump-all-'.Carbon::now()->timestamp.'.gz';
|
|
}
|
|
$this->backup_location = $this->backup_dir.$this->backup_file;
|
|
$this->backup_log = ScheduledDatabaseBackupExecution::create([
|
|
'uuid' => $this->backup_log_uuid,
|
|
'database_name' => $database,
|
|
'filename' => $this->backup_location,
|
|
'scheduled_database_backup_id' => $this->backup->id,
|
|
'local_storage_deleted' => false,
|
|
]);
|
|
$this->backup_standalone_mariadb($database);
|
|
} elseif ($this->database instanceof StandaloneClickhouse) {
|
|
$this->backup_file = '/clickhouse-backup-'.Carbon::now()->timestamp."-{$this->backup_log_uuid}.zip";
|
|
$this->backup_location = $this->backup_dir.$this->backup_file;
|
|
$this->backup_log = ScheduledDatabaseBackupExecution::create([
|
|
'uuid' => $this->backup_log_uuid,
|
|
'database_name' => $database,
|
|
'filename' => $this->backup_location,
|
|
'scheduled_database_backup_id' => $this->backup->id,
|
|
'local_storage_deleted' => false,
|
|
]);
|
|
$this->backup_standalone_clickhouse($database);
|
|
} else {
|
|
throw new \Exception('Unsupported database type');
|
|
}
|
|
|
|
$size = $this->calculate_size();
|
|
|
|
// Verify local backup succeeded
|
|
if ($size > 0) {
|
|
$localBackupSucceeded = true;
|
|
} else {
|
|
throw new \Exception('Local backup file is empty or was not created');
|
|
}
|
|
} catch (Throwable $e) {
|
|
// Local backup failed
|
|
if ($this->database instanceof StandaloneClickhouse) {
|
|
deleteBackupsLocally($this->backup_location, $this->server);
|
|
}
|
|
if ($this->backup_log) {
|
|
$this->backup_log->update([
|
|
'status' => 'failed',
|
|
'message' => $this->error_output ?? $this->backup_output ?? $e->getMessage(),
|
|
'size' => $size,
|
|
'filename' => null,
|
|
's3_uploaded' => null,
|
|
]);
|
|
}
|
|
try {
|
|
$this->team?->notify(new BackupFailed($this->backup, $this->database, $this->error_output ?? $this->backup_output ?? $e->getMessage(), $database));
|
|
} catch (Throwable $notifyException) {
|
|
Log::channel('scheduled-errors')->warning('Failed to send backup failure notification', [
|
|
'backup_id' => $this->backup->uuid,
|
|
'database' => $database,
|
|
'error' => $notifyException->getMessage(),
|
|
]);
|
|
}
|
|
|
|
continue;
|
|
}
|
|
|
|
// Step 2: Upload to S3 if enabled (independent of local backup)
|
|
$localStorageDeleted = false;
|
|
if ($this->backup->save_s3 && $localBackupSucceeded) {
|
|
try {
|
|
$this->upload_to_s3();
|
|
|
|
// If local backup is disabled, delete the local file immediately after S3 upload
|
|
if ($this->backup->disable_local_backup) {
|
|
deleteBackupsLocally($this->backup_location, $this->server);
|
|
$localStorageDeleted = true;
|
|
}
|
|
} catch (Throwable $e) {
|
|
// S3 upload failed but local backup succeeded
|
|
$s3UploadError = $e->getMessage();
|
|
}
|
|
}
|
|
|
|
// Step 3: Update status and send notifications based on results
|
|
if ($localBackupSucceeded) {
|
|
$message = $this->backup_output;
|
|
|
|
if ($s3UploadError) {
|
|
$message = $message
|
|
? $message."\n\nWarning: S3 upload failed: ".$s3UploadError
|
|
: 'Warning: S3 upload failed: '.$s3UploadError;
|
|
}
|
|
|
|
$this->backup_log->update([
|
|
'status' => 'success',
|
|
'message' => $message,
|
|
'size' => $size,
|
|
's3_uploaded' => $this->backup->save_s3 ? $this->s3_uploaded : null,
|
|
'local_storage_deleted' => $localStorageDeleted,
|
|
]);
|
|
|
|
// Send appropriate notification (wrapped in try-catch so notification
|
|
// failures never affect backup status — see GitHub issue #9088)
|
|
try {
|
|
if ($s3UploadError) {
|
|
$this->team->notify(new BackupSuccessWithS3Warning($this->backup, $this->database, $database, $s3UploadError));
|
|
} else {
|
|
$this->team->notify(new BackupSuccess($this->backup, $this->database, $database));
|
|
}
|
|
} catch (Throwable $e) {
|
|
Log::channel('scheduled-errors')->warning('Failed to send backup success notification', [
|
|
'backup_id' => $this->backup->uuid,
|
|
'database' => $database,
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
if ($this->backup_log && $this->backup_log->status === 'success') {
|
|
$this->removeExpiredBackups();
|
|
}
|
|
} catch (Throwable $e) {
|
|
throw $e;
|
|
} finally {
|
|
if ($this->team) {
|
|
BackupCreated::dispatch($this->team->id);
|
|
}
|
|
if ($this->backup_log) {
|
|
$this->backup_log->update([
|
|
'finished_at' => Carbon::now()->toImmutable(),
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
|
|
private function removeExpiredBackups(): void
|
|
{
|
|
try {
|
|
removeOldBackups($this->backup);
|
|
} catch (Throwable $exception) {
|
|
Log::channel('scheduled-errors')->warning('Database backup retention cleanup failed', [
|
|
'backup_id' => $this->backup->id,
|
|
'execution_id' => $this->backup_log?->id,
|
|
'error' => $exception->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
private function backup_standalone_mongodb(string $databaseWithCollections): void
|
|
{
|
|
try {
|
|
$url = $this->database->internal_db_url;
|
|
if (blank($url)) {
|
|
// For service-based MongoDB, try to build URL from environment variables
|
|
if (filled($this->mongo_root_username) && filled($this->mongo_root_password)) {
|
|
// Use container name instead of server IP for service-based MongoDB
|
|
// URL-encode credentials to prevent URI injection
|
|
$encodedUser = rawurlencode($this->mongo_root_username);
|
|
$encodedPass = rawurlencode($this->mongo_root_password);
|
|
$url = "mongodb://{$encodedUser}:{$encodedPass}@{$this->container_name}:27017";
|
|
} else {
|
|
// If no environment variables are available, throw an exception
|
|
throw new \Exception('MongoDB credentials not found. Ensure MONGO_INITDB_ROOT_USERNAME and MONGO_INITDB_ROOT_PASSWORD environment variables are available in the container.');
|
|
}
|
|
}
|
|
Log::info('MongoDB backup URL configured', ['has_url' => filled($url), 'using_env_vars' => blank($this->database->internal_db_url)]);
|
|
$escapedUrl = escapeshellarg($url);
|
|
if ($databaseWithCollections === 'all') {
|
|
$commands[] = 'mkdir -p '.$this->backup_dir;
|
|
if (str($this->database->image)->startsWith('mongo:4')) {
|
|
$commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --archive > $this->backup_location";
|
|
} else {
|
|
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --gzip --archive > $this->backup_location";
|
|
}
|
|
} else {
|
|
if (str($databaseWithCollections)->contains(':')) {
|
|
$databaseName = str($databaseWithCollections)->before(':');
|
|
$collectionsToExclude = str($databaseWithCollections)->after(':')->explode(',');
|
|
} else {
|
|
$databaseName = $databaseWithCollections;
|
|
$collectionsToExclude = collect();
|
|
}
|
|
$commands[] = 'mkdir -p '.$this->backup_dir;
|
|
|
|
// Validate and escape database name to prevent command injection
|
|
validateShellSafePath($databaseName, 'database name');
|
|
$escapedDatabaseName = escapeshellarg($databaseName);
|
|
|
|
if ($collectionsToExclude->count() === 0) {
|
|
if (str($this->database->image)->startsWith('mongo:4')) {
|
|
$commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --archive > $this->backup_location";
|
|
} else {
|
|
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --db $escapedDatabaseName --gzip --archive > $this->backup_location";
|
|
}
|
|
} else {
|
|
// Validate and escape each collection name
|
|
$escapedCollections = $collectionsToExclude->map(function ($collection) {
|
|
$collection = trim($collection);
|
|
validateShellSafePath($collection, 'collection name');
|
|
|
|
return escapeshellarg($collection);
|
|
});
|
|
|
|
if (str($this->database->image)->startsWith('mongo:4')) {
|
|
$commands[] = "docker exec $this->container_name mongodump --uri=$escapedUrl --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
|
|
} else {
|
|
$commands[] = "docker exec $this->container_name mongodump --authenticationDatabase=admin --uri=$escapedUrl --db $escapedDatabaseName --gzip --excludeCollection ".$escapedCollections->implode(' --excludeCollection ')." --archive > $this->backup_location";
|
|
}
|
|
}
|
|
}
|
|
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
|
|
$this->backup_output = trim($this->backup_output);
|
|
if ($this->backup_output === '') {
|
|
$this->backup_output = null;
|
|
}
|
|
} catch (Throwable $e) {
|
|
$this->add_to_error_output($e->getMessage());
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
private function backup_standalone_postgresql(string $database): void
|
|
{
|
|
try {
|
|
$commands[] = 'mkdir -p '.$this->backup_dir;
|
|
$backupCommand = 'docker exec';
|
|
if ($this->postgres_password) {
|
|
$backupCommand .= ' -e PGPASSWORD='.escapeshellarg($this->postgres_password);
|
|
}
|
|
$escapedUsername = escapeshellarg($this->database->postgres_user);
|
|
if ($this->backup->dump_all) {
|
|
$backupCommand .= " $this->container_name pg_dumpall --username $escapedUsername | gzip > $this->backup_location";
|
|
} else {
|
|
// Validate and escape database name to prevent command injection
|
|
validateShellSafePath($database, 'database name');
|
|
$escapedDatabase = escapeshellarg($database);
|
|
$backupCommand .= " $this->container_name pg_dump --format=custom --no-acl --no-owner --username $escapedUsername $escapedDatabase > $this->backup_location";
|
|
}
|
|
|
|
$commands[] = $backupCommand;
|
|
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
|
|
$this->backup_output = trim($this->backup_output);
|
|
if ($this->backup_output === '') {
|
|
$this->backup_output = null;
|
|
}
|
|
} catch (Throwable $e) {
|
|
$this->add_to_error_output($e->getMessage());
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
private function backup_standalone_mysql(string $database): void
|
|
{
|
|
try {
|
|
$commands[] = 'mkdir -p '.$this->backup_dir;
|
|
$escapedPassword = escapeshellarg($this->database->mysql_root_password);
|
|
if ($this->backup->dump_all) {
|
|
$commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress | gzip > $this->backup_location";
|
|
} else {
|
|
// Validate and escape database name to prevent command injection
|
|
validateShellSafePath($database, 'database name');
|
|
$escapedDatabase = escapeshellarg($database);
|
|
$commands[] = "docker exec $this->container_name mysqldump -u root -p$escapedPassword $escapedDatabase > $this->backup_location";
|
|
}
|
|
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
|
|
$this->backup_output = trim($this->backup_output);
|
|
if ($this->backup_output === '') {
|
|
$this->backup_output = null;
|
|
}
|
|
} catch (Throwable $e) {
|
|
$this->add_to_error_output($e->getMessage());
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
private function backup_standalone_mariadb(string $database): void
|
|
{
|
|
try {
|
|
$commands[] = 'mkdir -p '.$this->backup_dir;
|
|
$escapedPassword = escapeshellarg($this->database->mariadb_root_password);
|
|
if ($this->backup->dump_all) {
|
|
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword --all-databases --single-transaction --quick --lock-tables=false --compress > $this->backup_location";
|
|
} else {
|
|
// Validate and escape database name to prevent command injection
|
|
validateShellSafePath($database, 'database name');
|
|
$escapedDatabase = escapeshellarg($database);
|
|
$commands[] = "docker exec $this->container_name mariadb-dump -u root -p$escapedPassword $escapedDatabase > $this->backup_location";
|
|
}
|
|
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
|
|
$this->backup_output = trim($this->backup_output);
|
|
if ($this->backup_output === '') {
|
|
$this->backup_output = null;
|
|
}
|
|
} catch (Throwable $e) {
|
|
$this->add_to_error_output($e->getMessage());
|
|
throw $e;
|
|
}
|
|
}
|
|
|
|
private function backup_standalone_clickhouse(string $database): void
|
|
{
|
|
$archiveName = ltrim($this->backup_file, '/');
|
|
|
|
try {
|
|
$commands = ClickhouseBackupCommand::make(
|
|
containerName: $this->container_name,
|
|
database: $database,
|
|
archiveName: $archiveName,
|
|
backupDirectory: $this->backup_dir,
|
|
);
|
|
|
|
$this->backup_output = instant_remote_process($commands, $this->server, true, false, $this->timeout, disableMultiplexing: true);
|
|
$this->backup_output = trim($this->backup_output);
|
|
if ($this->backup_output === '') {
|
|
$this->backup_output = null;
|
|
}
|
|
} catch (Throwable $e) {
|
|
$this->add_to_error_output($e->getMessage());
|
|
throw $e;
|
|
} finally {
|
|
$cleanupCommand = ClickhouseBackupCommand::cleanup($this->container_name, $archiveName);
|
|
instant_remote_process([$cleanupCommand], $this->server, false, false, null, disableMultiplexing: true);
|
|
}
|
|
}
|
|
|
|
private function add_to_backup_output($output): void
|
|
{
|
|
if ($this->backup_output) {
|
|
$this->backup_output = $this->backup_output."\n".$output;
|
|
} else {
|
|
$this->backup_output = $output;
|
|
}
|
|
}
|
|
|
|
private function add_to_error_output($output): void
|
|
{
|
|
if ($this->error_output) {
|
|
$this->error_output = $this->error_output."\n".$output;
|
|
} else {
|
|
$this->error_output = $output;
|
|
}
|
|
}
|
|
|
|
private function calculate_size()
|
|
{
|
|
return instant_remote_process(["du -b $this->backup_location | cut -f1"], $this->server, false, false, null, disableMultiplexing: true);
|
|
}
|
|
|
|
private function upload_to_s3(): void
|
|
{
|
|
if (is_null($this->s3)) {
|
|
$previousS3StorageId = $this->backup->s3_storage_id;
|
|
|
|
$this->backup->update([
|
|
'save_s3' => false,
|
|
's3_storage_id' => null,
|
|
]);
|
|
|
|
throw new \Exception('S3 storage configuration is missing or has been deleted (S3 storage ID: '.($previousS3StorageId ?? 'null').'). S3 backup has been disabled for this schedule.');
|
|
}
|
|
|
|
try {
|
|
$key = $this->s3->key;
|
|
$secret = $this->s3->secret;
|
|
// $region = $this->s3->region;
|
|
$bucket = $this->s3->bucket;
|
|
$endpoint = $this->s3->endpoint;
|
|
$this->s3->testConnection(shouldSave: true);
|
|
if (data_get($this->backup, 'database_type') === ServiceDatabase::class) {
|
|
$network = $this->database->service->destination->network;
|
|
} else {
|
|
$network = $this->database->destination->network;
|
|
}
|
|
$safeNetwork = escapeshellarg($network);
|
|
|
|
$fullImageName = $this->getFullImageName();
|
|
|
|
$containerExists = instant_remote_process(["docker ps -a -q -f name=backup-of-{$this->backup_log_uuid}"], $this->server, false, false, null, disableMultiplexing: true);
|
|
if (filled($containerExists)) {
|
|
instant_remote_process(["docker rm -f backup-of-{$this->backup_log_uuid}"], $this->server, false, false, null, disableMultiplexing: true);
|
|
}
|
|
|
|
if (isDev()) {
|
|
if ($this->database->name === 'coolify-db') {
|
|
$backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/coolify/coolify-db-'.$this->server->ip.$this->backup_file;
|
|
$commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
|
|
} else {
|
|
$backup_location_from = '/var/lib/docker/volumes/coolify_dev_backups_data/_data/databases/'.str($this->team->name)->slug().'-'.$this->team->id.'/'.$this->directory_name.$this->backup_file;
|
|
$commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $backup_location_from:$this->backup_location:ro {$fullImageName}";
|
|
}
|
|
} else {
|
|
$commands[] = "docker run -d --network {$safeNetwork} --name backup-of-{$this->backup_log_uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}";
|
|
}
|
|
|
|
// Escape S3 credentials to prevent command injection
|
|
$escapedEndpoint = escapeshellarg($endpoint);
|
|
$escapedKey = escapeshellarg($key);
|
|
$escapedSecret = escapeshellarg($secret);
|
|
$escapedBackupLocation = escapeshellarg($this->backup_location);
|
|
$escapedS3Destination = escapeshellarg("temporary/{$bucket}{$this->backup_dir}/");
|
|
$resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($endpoint))
|
|
->map(fn (string $resolveOption): string => '--resolve '.escapeshellarg($resolveOption))
|
|
->implode(' ');
|
|
$resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions;
|
|
|
|
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set{$resolveOptions} temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}";
|
|
$commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp {$escapedBackupLocation} {$escapedS3Destination}";
|
|
instant_remote_process($commands, $this->server, true, false, null, disableMultiplexing: true);
|
|
|
|
$this->s3_uploaded = true;
|
|
} catch (Throwable $e) {
|
|
$this->s3_uploaded = false;
|
|
$this->add_to_error_output($e->getMessage());
|
|
throw $e;
|
|
} finally {
|
|
$command = "docker rm -f backup-of-{$this->backup_log_uuid}";
|
|
instant_remote_process([$command], $this->server, true, false, null, disableMultiplexing: true);
|
|
}
|
|
}
|
|
|
|
private function getFullImageName(): string
|
|
{
|
|
$helperImage = coolifyHelperImage();
|
|
$latestVersion = getHelperVersion();
|
|
|
|
return "{$helperImage}:{$latestVersion}";
|
|
}
|
|
|
|
private function markStaleExecutionsAsFailed(): void
|
|
{
|
|
try {
|
|
$timeoutSeconds = ($this->backup->timeout ?? 3600) * 2;
|
|
|
|
$staleExecutions = $this->backup->executions()
|
|
->where('status', 'running')
|
|
->where('created_at', '<', now()->subSeconds($timeoutSeconds))
|
|
->get();
|
|
|
|
foreach ($staleExecutions as $execution) {
|
|
$execution->update([
|
|
'status' => 'failed',
|
|
'message' => 'Marked as failed - backup execution exceeded maximum allowed time',
|
|
'finished_at' => now(),
|
|
]);
|
|
}
|
|
} catch (Throwable $e) {
|
|
Log::channel('scheduled-errors')->warning('Failed to clean up stale backup executions', [
|
|
'backup_id' => $this->backup->uuid,
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
public function failed(?Throwable $exception): void
|
|
{
|
|
Log::channel('scheduled-errors')->error('DatabaseBackup permanently failed', [
|
|
'job' => 'DatabaseBackupJob',
|
|
'backup_id' => $this->backup->uuid,
|
|
'database' => $this->database?->name ?? 'unknown',
|
|
'database_type' => get_class($this->database ?? new \stdClass),
|
|
'server' => $this->server?->name ?? 'unknown',
|
|
'total_attempts' => $this->attempts(),
|
|
'error' => $exception?->getMessage(),
|
|
'trace' => $exception?->getTraceAsString(),
|
|
]);
|
|
|
|
$log = ScheduledDatabaseBackupExecution::where('uuid', $this->backup_log_uuid)->first();
|
|
|
|
if ($log) {
|
|
// Don't overwrite a successful backup status — a post-backup error
|
|
// (e.g. notification failure) should not retroactively mark the backup
|
|
// as failed (see GitHub issue #9088)
|
|
if ($log->status !== 'success') {
|
|
$log->update([
|
|
'status' => 'failed',
|
|
'message' => 'Job permanently failed after '.$this->attempts().' attempts: '.($exception?->getMessage() ?? 'Unknown error'),
|
|
'size' => 0,
|
|
'filename' => null,
|
|
'finished_at' => Carbon::now(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
// Notify team about permanent failure (only if backup didn't already succeed)
|
|
if ($this->team && $log?->status !== 'success') {
|
|
$databaseName = $log?->database_name ?? 'unknown';
|
|
$output = $this->backup_output ?? $exception?->getMessage() ?? 'Unknown error';
|
|
try {
|
|
$this->team->notify(new BackupFailed($this->backup, $this->database, $output, $databaseName));
|
|
} catch (Throwable $e) {
|
|
Log::channel('scheduled-errors')->warning('Failed to send backup permanent failure notification', [
|
|
'backup_id' => $this->backup->uuid,
|
|
'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
}
|