diff --git a/app/Events/RestoreJobFinished.php b/app/Events/RestoreJobFinished.php index d3adb7798..cc4be8029 100644 --- a/app/Events/RestoreJobFinished.php +++ b/app/Events/RestoreJobFinished.php @@ -17,17 +17,23 @@ public function __construct($data) $tmpPath = data_get($data, 'tmpPath'); $container = data_get($data, 'container'); $serverId = data_get($data, 'serverId'); - if (filled($scriptPath) && filled($tmpPath) && filled($container) && filled($serverId)) { - if (str($tmpPath)->startsWith('/tmp/') - && str($scriptPath)->startsWith('/tmp/') - && ! str($tmpPath)->contains('..') - && ! str($scriptPath)->contains('..') - && strlen($tmpPath) > 5 // longer than just "/tmp/" - && strlen($scriptPath) > 5 - ) { - $commands[] = "docker exec {$container} sh -c 'rm {$scriptPath}'"; - $commands[] = "docker exec {$container} sh -c 'rm {$tmpPath}'"; - instant_remote_process($commands, Server::find($serverId), throwError: true); + + if (filled($container) && filled($serverId)) { + $commands = []; + + if (isSafeTmpPath($scriptPath)) { + $commands[] = "docker exec {$container} sh -c 'rm ".escapeshellarg($scriptPath)." 2>/dev/null || true'"; + } + + if (isSafeTmpPath($tmpPath)) { + $commands[] = "docker exec {$container} sh -c 'rm ".escapeshellarg($tmpPath)." 2>/dev/null || true'"; + } + + if (! empty($commands)) { + $server = Server::find($serverId); + if ($server) { + instant_remote_process($commands, $server, throwError: false); + } } } } diff --git a/app/Events/S3RestoreJobFinished.php b/app/Events/S3RestoreJobFinished.php new file mode 100644 index 000000000..e1f844558 --- /dev/null +++ b/app/Events/S3RestoreJobFinished.php @@ -0,0 +1,56 @@ +/dev/null || true'; + } + + // Clean up server temp file if still exists (should already be cleaned) + if (isSafeTmpPath($serverTmpPath)) { + $commands[] = 'rm -f '.escapeshellarg($serverTmpPath).' 2>/dev/null || true'; + } + + // Clean up any remaining files in database container (may already be cleaned) + if (filled($container)) { + if (isSafeTmpPath($containerTmpPath)) { + $commands[] = 'docker exec '.escapeshellarg($container).' rm -f '.escapeshellarg($containerTmpPath).' 2>/dev/null || true'; + } + if (isSafeTmpPath($scriptPath)) { + $commands[] = 'docker exec '.escapeshellarg($container).' rm -f '.escapeshellarg($scriptPath).' 2>/dev/null || true'; + } + } + + if (! empty($commands)) { + $server = Server::find($serverId); + if ($server) { + instant_remote_process($commands, $server, throwError: false); + } + } + } + } +} diff --git a/app/Jobs/CoolifyTask.php b/app/Jobs/CoolifyTask.php index d6dc6fa05..ce535e036 100755 --- a/app/Jobs/CoolifyTask.php +++ b/app/Jobs/CoolifyTask.php @@ -90,5 +90,22 @@ public function failed(?\Throwable $exception): void 'failed_at' => now()->toIso8601String(), ]); $this->activity->save(); + + // Dispatch cleanup event on failure (same as on success) + if ($this->call_event_on_finish) { + try { + $eventClass = "App\\Events\\$this->call_event_on_finish"; + if (! is_null($this->call_event_data)) { + event(new $eventClass($this->call_event_data)); + } else { + event(new $eventClass($this->activity->causer_id)); + } + Log::info('Cleanup event dispatched after job failure', [ + 'event' => $this->call_event_on_finish, + ]); + } catch (\Throwable $e) { + Log::error('Error dispatching cleanup event on failure: '.$e->getMessage()); + } + } } } diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index 8766a1afc..45ac6eb7d 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -639,7 +639,13 @@ private function upload_to_s3(): void } else { $commands[] = "docker run -d --network {$network} --name backup-of-{$this->backup_log_uuid} --rm -v $this->backup_location:$this->backup_location:ro {$fullImageName}"; } - $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set temporary {$endpoint} {$key} \"{$secret}\""; + + // Escape S3 credentials to prevent command injection + $escapedEndpoint = escapeshellarg($endpoint); + $escapedKey = escapeshellarg($key); + $escapedSecret = escapeshellarg($secret); + + $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc alias set temporary {$escapedEndpoint} {$escapedKey} {$escapedSecret}"; $commands[] = "docker exec backup-of-{$this->backup_log_uuid} mc cp $this->backup_location temporary/$bucket{$this->backup_dir}/"; instant_remote_process($commands, $this->server); diff --git a/app/Livewire/Project/Database/Import.php b/app/Livewire/Project/Database/Import.php index 7d6ac3131..01ddb7f5d 100644 --- a/app/Livewire/Project/Database/Import.php +++ b/app/Livewire/Project/Database/Import.php @@ -2,6 +2,7 @@ namespace App\Livewire\Project\Database; +use App\Models\S3Storage; use App\Models\Server; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Facades\Auth; @@ -54,6 +55,15 @@ class Import extends Component public string $mongodbRestoreCommand = 'mongorestore --authenticationDatabase=admin --username $MONGO_INITDB_ROOT_USERNAME --password $MONGO_INITDB_ROOT_PASSWORD --uri mongodb://localhost:27017 --gzip --archive='; + // S3 Restore properties + public $availableS3Storages = []; + + public ?int $s3StorageId = null; + + public string $s3Path = ''; + + public ?int $s3FileSize = null; + public function getListeners() { $userId = Auth::id(); @@ -65,11 +75,9 @@ public function getListeners() public function mount() { - if (isDev()) { - $this->customLocation = '/data/coolify/pg-dump-all-1736245863.gz'; - } $this->parameters = get_route_parameters(); $this->getContainers(); + $this->loadAvailableS3Storages(); } public function updatedDumpAll($value) @@ -179,22 +187,22 @@ public function runImport() try { $this->importRunning = true; $this->importCommands = []; - if (filled($this->customLocation)) { - $backupFileName = '/tmp/restore_'.$this->resource->uuid; - $this->importCommands[] = "docker cp {$this->customLocation} {$this->container}:{$backupFileName}"; - $tmpPath = $backupFileName; - } else { - $backupFileName = "upload/{$this->resource->uuid}/restore"; - $path = Storage::path($backupFileName); - if (! Storage::exists($backupFileName)) { - $this->dispatch('error', 'The file does not exist or has been deleted.'); + $backupFileName = "upload/{$this->resource->uuid}/restore"; - return; - } + // Check if an uploaded file exists first (takes priority over custom location) + if (Storage::exists($backupFileName)) { + $path = Storage::path($backupFileName); $tmpPath = '/tmp/'.basename($backupFileName).'_'.$this->resource->uuid; instant_scp($path, $tmpPath, $this->server); Storage::delete($backupFileName); $this->importCommands[] = "docker cp {$tmpPath} {$this->container}:{$tmpPath}"; + } elseif (filled($this->customLocation)) { + $tmpPath = '/tmp/restore_'.$this->resource->uuid; + $this->importCommands[] = "docker cp {$this->customLocation} {$this->container}:{$tmpPath}"; + } else { + $this->dispatch('error', 'The file does not exist or has been deleted.'); + + return; } // Copy the restore command to a script file @@ -248,7 +256,10 @@ public function runImport() 'container' => $this->container, 'serverId' => $this->server->id, ]); + + // Dispatch activity to the monitor and open slide-over $this->dispatch('activityMonitor', $activity->id); + $this->dispatch('databaserestore'); } } catch (\Throwable $e) { return handleError($e, $this); @@ -257,4 +268,232 @@ public function runImport() $this->importCommands = []; } } + + public function loadAvailableS3Storages() + { + try { + $this->availableS3Storages = S3Storage::ownedByCurrentTeam(['id', 'name', 'description']) + ->where('is_usable', true) + ->get(); + } catch (\Throwable $e) { + $this->availableS3Storages = collect(); + } + } + + public function updatedS3Path($value) + { + // Reset validation state when path changes + $this->s3FileSize = null; + + // Ensure path starts with a slash + if ($value !== null && $value !== '') { + $this->s3Path = str($value)->trim()->start('/')->value(); + } + } + + public function updatedS3StorageId() + { + // Reset validation state when storage changes + $this->s3FileSize = null; + } + + public function checkS3File() + { + if (! $this->s3StorageId) { + $this->dispatch('error', 'Please select an S3 storage.'); + + return; + } + + if (blank($this->s3Path)) { + $this->dispatch('error', 'Please provide an S3 path.'); + + return; + } + + try { + $s3Storage = S3Storage::ownedByCurrentTeam()->findOrFail($this->s3StorageId); + + // Test connection + $s3Storage->testConnection(); + + // Build S3 disk configuration + $disk = Storage::build([ + 'driver' => 's3', + 'region' => $s3Storage->region, + 'key' => $s3Storage->key, + 'secret' => $s3Storage->secret, + 'bucket' => $s3Storage->bucket, + 'endpoint' => $s3Storage->endpoint, + 'use_path_style_endpoint' => true, + ]); + + // Clean the path (remove leading slash if present) + $cleanPath = ltrim($this->s3Path, '/'); + + // Check if file exists + if (! $disk->exists($cleanPath)) { + $this->dispatch('error', 'File not found in S3. Please check the path.'); + + return; + } + + // Get file size + $this->s3FileSize = $disk->size($cleanPath); + + $this->dispatch('success', 'File found in S3. Size: '.formatBytes($this->s3FileSize)); + } catch (\Throwable $e) { + $this->s3FileSize = null; + + return handleError($e, $this); + } + } + + public function restoreFromS3() + { + $this->authorize('update', $this->resource); + + if (! $this->s3StorageId || blank($this->s3Path)) { + $this->dispatch('error', 'Please select S3 storage and provide a path first.'); + + return; + } + + if (is_null($this->s3FileSize)) { + $this->dispatch('error', 'Please check the file first by clicking "Check File".'); + + return; + } + + try { + $this->importRunning = true; + + $s3Storage = S3Storage::ownedByCurrentTeam()->findOrFail($this->s3StorageId); + + $key = $s3Storage->key; + $secret = $s3Storage->secret; + $bucket = $s3Storage->bucket; + $endpoint = $s3Storage->endpoint; + + // Clean the S3 path + $cleanPath = ltrim($this->s3Path, '/'); + + // Get helper image + $helperImage = config('constants.coolify.helper_image'); + $latestVersion = getHelperVersion(); + $fullImageName = "{$helperImage}:{$latestVersion}"; + + // Get the database destination network + $destinationNetwork = $this->resource->destination->network ?? 'coolify'; + + // Generate unique names for this operation + $containerName = "s3-restore-{$this->resource->uuid}"; + $helperTmpPath = '/tmp/'.basename($cleanPath); + $serverTmpPath = "/tmp/s3-restore-{$this->resource->uuid}-".basename($cleanPath); + $containerTmpPath = "/tmp/restore_{$this->resource->uuid}-".basename($cleanPath); + $scriptPath = "/tmp/restore_{$this->resource->uuid}.sh"; + + // Prepare all commands in sequence + $commands = []; + + // 1. Clean up any existing helper container and temp files from previous runs + $commands[] = "docker rm -f {$containerName} 2>/dev/null || true"; + $commands[] = "rm -f {$serverTmpPath} 2>/dev/null || true"; + $commands[] = "docker exec {$this->container} rm -f {$containerTmpPath} {$scriptPath} 2>/dev/null || true"; + + // 2. Start helper container on the database network + $commands[] = "docker run -d --network {$destinationNetwork} --name {$containerName} {$fullImageName} sleep 3600"; + + // 3. Configure S3 access in helper container + $escapedEndpoint = escapeshellarg($endpoint); + $escapedKey = escapeshellarg($key); + $escapedSecret = escapeshellarg($secret); + $commands[] = "docker exec {$containerName} mc alias set s3temp {$escapedEndpoint} {$escapedKey} {$escapedSecret}"; + + // 4. Check file exists in S3 + $commands[] = "docker exec {$containerName} mc stat s3temp/{$bucket}/{$cleanPath}"; + + // 5. Download from S3 to helper container (progress shown by default) + $commands[] = "docker exec {$containerName} mc cp s3temp/{$bucket}/{$cleanPath} {$helperTmpPath}"; + + // 6. Copy from helper to server, then immediately to database container + $commands[] = "docker cp {$containerName}:{$helperTmpPath} {$serverTmpPath}"; + $commands[] = "docker cp {$serverTmpPath} {$this->container}:{$containerTmpPath}"; + + // 7. Cleanup helper container and server temp file immediately (no longer needed) + $commands[] = "docker rm -f {$containerName} 2>/dev/null || true"; + $commands[] = "rm -f {$serverTmpPath} 2>/dev/null || true"; + + // 8. Build and execute restore command inside database container + $restoreCommand = $this->buildRestoreCommand($containerTmpPath); + + $restoreCommandBase64 = base64_encode($restoreCommand); + $commands[] = "echo \"{$restoreCommandBase64}\" | base64 -d > {$scriptPath}"; + $commands[] = "chmod +x {$scriptPath}"; + $commands[] = "docker cp {$scriptPath} {$this->container}:{$scriptPath}"; + + // 9. Execute restore and cleanup temp files immediately after completion + $commands[] = "docker exec {$this->container} sh -c '{$scriptPath} && rm -f {$containerTmpPath} {$scriptPath}'"; + $commands[] = "docker exec {$this->container} sh -c 'echo \"Import finished with exit code $?\"'"; + + // Execute all commands with cleanup event (as safety net for edge cases) + $activity = remote_process($commands, $this->server, ignore_errors: true, callEventOnFinish: 'S3RestoreJobFinished', callEventData: [ + 'containerName' => $containerName, + 'serverTmpPath' => $serverTmpPath, + 'scriptPath' => $scriptPath, + 'containerTmpPath' => $containerTmpPath, + 'container' => $this->container, + 'serverId' => $this->server->id, + ]); + + // Dispatch activity to the monitor and open slide-over + $this->dispatch('activityMonitor', $activity->id); + $this->dispatch('databaserestore'); + $this->dispatch('info', 'Restoring database from S3. Progress will be shown in the activity monitor...'); + } catch (\Throwable $e) { + $this->importRunning = false; + + return handleError($e, $this); + } + } + + public function buildRestoreCommand(string $tmpPath): string + { + switch ($this->resource->getMorphClass()) { + case \App\Models\StandaloneMariadb::class: + $restoreCommand = $this->mariadbRestoreCommand; + if ($this->dumpAll) { + $restoreCommand .= " && (gunzip -cf {$tmpPath} 2>/dev/null || cat {$tmpPath}) | mariadb -u root -p\$MARIADB_ROOT_PASSWORD"; + } else { + $restoreCommand .= " < {$tmpPath}"; + } + break; + case \App\Models\StandaloneMysql::class: + $restoreCommand = $this->mysqlRestoreCommand; + if ($this->dumpAll) { + $restoreCommand .= " && (gunzip -cf {$tmpPath} 2>/dev/null || cat {$tmpPath}) | mysql -u root -p\$MYSQL_ROOT_PASSWORD"; + } else { + $restoreCommand .= " < {$tmpPath}"; + } + break; + case \App\Models\StandalonePostgresql::class: + $restoreCommand = $this->postgresqlRestoreCommand; + if ($this->dumpAll) { + $restoreCommand .= " && (gunzip -cf {$tmpPath} 2>/dev/null || cat {$tmpPath}) | psql -U \$POSTGRES_USER postgres"; + } else { + $restoreCommand .= " {$tmpPath}"; + } + break; + case \App\Models\StandaloneMongodb::class: + $restoreCommand = $this->mongodbRestoreCommand; + if ($this->dumpAll === false) { + $restoreCommand .= "{$tmpPath}"; + } + break; + default: + $restoreCommand = ''; + } + + return $restoreCommand; + } } diff --git a/app/Livewire/Settings/Index.php b/app/Livewire/Settings/Index.php index 96f13b173..7a96eabb2 100644 --- a/app/Livewire/Settings/Index.php +++ b/app/Livewire/Settings/Index.php @@ -44,6 +44,8 @@ class Index extends Component public bool $forceSaveDomains = false; + public $buildActivityId = null; + public function render() { return view('livewire.settings.index'); @@ -151,4 +153,37 @@ public function submit() return handleError($e, $this); } } + + public function buildHelperImage() + { + try { + if (! isDev()) { + $this->dispatch('error', 'Building helper image is only available in development mode.'); + + return; + } + + $version = $this->dev_helper_version ?: config('constants.coolify.helper_version'); + if (empty($version)) { + $this->dispatch('error', 'Please specify a version to build.'); + + return; + } + + $buildCommand = "docker build -t ghcr.io/coollabsio/coolify-helper:{$version} -f docker/coolify-helper/Dockerfile ."; + + $activity = remote_process( + command: [$buildCommand], + server: $this->server, + type: 'build-helper-image' + ); + + $this->buildActivityId = $activity->id; + $this->dispatch('activityMonitor', $activity->id); + + $this->dispatch('success', "Building coolify-helper:{$version}..."); + } catch (\Exception $e) { + return handleError($e, $this); + } + } } diff --git a/app/Livewire/Storage/Form.php b/app/Livewire/Storage/Form.php index d97550693..63d9ce3da 100644 --- a/app/Livewire/Storage/Form.php +++ b/app/Livewire/Storage/Form.php @@ -120,8 +120,15 @@ public function testConnection() $this->storage->testConnection(shouldSave: true); + // Update component property to reflect the new validation status + $this->isUsable = $this->storage->is_usable; + return $this->dispatch('success', 'Connection is working.', 'Tested with "ListObjectsV2" action.'); } catch (\Throwable $e) { + // Refresh model and sync to get the latest state + $this->storage->refresh(); + $this->isUsable = $this->storage->is_usable; + $this->dispatch('error', 'Failed to create storage.', $e->getMessage()); } } diff --git a/app/Livewire/Storage/Show.php b/app/Livewire/Storage/Show.php index bdea9a3b0..fdf3d0d28 100644 --- a/app/Livewire/Storage/Show.php +++ b/app/Livewire/Storage/Show.php @@ -3,10 +3,13 @@ namespace App\Livewire\Storage; use App\Models\S3Storage; +use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; class Show extends Component { + use AuthorizesRequests; + public $storage = null; public function mount() @@ -15,6 +18,7 @@ public function mount() if (! $this->storage) { abort(404); } + $this->authorize('view', $this->storage); } public function render() diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index de27bbca6..47652eb35 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -3,6 +3,7 @@ namespace App\Models; use App\Traits\HasSafeStringAttribute; +use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; use Illuminate\Notifications\Messages\MailMessage; use Illuminate\Support\Facades\Storage; @@ -41,6 +42,19 @@ public function awsUrl() return "{$this->endpoint}/{$this->bucket}"; } + protected function path(): Attribute + { + return Attribute::make( + set: function (?string $value) { + if ($value === null || $value === '') { + return null; + } + + return str($value)->trim()->start('/')->value(); + } + ); + } + public function testConnection(bool $shouldSave = false) { try { diff --git a/bootstrap/helpers/shared.php b/bootstrap/helpers/shared.php index 8a278476e..1b23247fa 100644 --- a/bootstrap/helpers/shared.php +++ b/bootstrap/helpers/shared.php @@ -3154,6 +3154,118 @@ function generateDockerComposeServiceName(mixed $services, int $pullRequestId = return $collection; } +function formatBytes(?int $bytes, int $precision = 2): string +{ + if ($bytes === null || $bytes === 0) { + return '0 B'; + } + + // Handle negative numbers + if ($bytes < 0) { + return '0 B'; + } + + $units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']; + $base = 1024; + $exponent = floor(log($bytes) / log($base)); + $exponent = min($exponent, count($units) - 1); + + $value = $bytes / pow($base, $exponent); + + return round($value, $precision).' '.$units[$exponent]; +} + +/** + * Validates that a file path is safely within the /tmp/ directory. + * Protects against path traversal attacks by resolving the real path + * and verifying it stays within /tmp/. + * + * Note: On macOS, /tmp is often a symlink to /private/tmp, which is handled. + */ +function isSafeTmpPath(?string $path): bool +{ + if (blank($path)) { + return false; + } + + // URL decode to catch encoded traversal attempts + $decodedPath = urldecode($path); + + // Minimum length check - /tmp/x is 6 chars + if (strlen($decodedPath) < 6) { + return false; + } + + // Must start with /tmp/ + if (! str($decodedPath)->startsWith('/tmp/')) { + return false; + } + + // Quick check for obvious traversal attempts + if (str($decodedPath)->contains('..')) { + return false; + } + + // Check for null bytes (directory traversal technique) + if (str($decodedPath)->contains("\0")) { + return false; + } + + // Remove any trailing slashes for consistent validation + $normalizedPath = rtrim($decodedPath, '/'); + + // Normalize the path by removing redundant separators and resolving . and .. + // We'll do this manually since realpath() requires the path to exist + $parts = explode('/', $normalizedPath); + $resolvedParts = []; + + foreach ($parts as $part) { + if ($part === '' || $part === '.') { + // Skip empty parts (from //) and current directory references + continue; + } elseif ($part === '..') { + // Parent directory - this should have been caught earlier but double-check + return false; + } else { + $resolvedParts[] = $part; + } + } + + $resolvedPath = '/'.implode('/', $resolvedParts); + + // Final check: resolved path must start with /tmp/ + // And must have at least one component after /tmp/ + if (! str($resolvedPath)->startsWith('/tmp/') || $resolvedPath === '/tmp') { + return false; + } + + // Resolve the canonical /tmp path (handles symlinks like /tmp -> /private/tmp on macOS) + $canonicalTmpPath = realpath('/tmp'); + if ($canonicalTmpPath === false) { + // If /tmp doesn't exist, something is very wrong, but allow non-existing paths + $canonicalTmpPath = '/tmp'; + } + + // Calculate dirname once to avoid redundant calls + $dirPath = dirname($resolvedPath); + + // If the directory exists, resolve it via realpath to catch symlink attacks + if (is_dir($dirPath)) { + // For existing paths, resolve to absolute path to catch symlinks + $realDir = realpath($dirPath); + if ($realDir === false) { + return false; + } + + // Check if the real directory is within /tmp (or its canonical path) + if (! str($realDir)->startsWith('/tmp') && ! str($realDir)->startsWith($canonicalTmpPath)) { + return false; + } + } + + return true; +} + /** * Transform colon-delimited status format to human-readable parentheses format. * diff --git a/resources/views/livewire/project/database/backup-executions.blade.php b/resources/views/livewire/project/database/backup-executions.blade.php index 30eef5976..c56908f50 100644 --- a/resources/views/livewire/project/database/backup-executions.blade.php +++ b/resources/views/livewire/project/database/backup-executions.blade.php @@ -68,22 +68,21 @@ class="flex flex-col gap-4">
Upload a backup file or specify a file path on the server
+Download and restore a backup from S3 storage
+