diff --git a/RELEASE.md b/RELEASE.md index bc159b040..1df585fc5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -15,20 +15,35 @@ ## Table of Contents ## Release Process -1. **Development on `next` or Feature Branches** - - Improvements, fixes, and new features are developed on the `next` branch or separate feature branches. +1. **Prepare the Release** + - Develop changes on `next` or feature branches. + - Set the release version in `config/constants.php` and `versions.json`. Both values must match the planned Git tag without the `v` prefix (for example, `4.2.0` for tag `v4.2.0`). + - Verify the changelog and required tests before merging. -2. **Merging to `main`** - - Once ready, changes are merged from the `next` branch into the `main` branch (via a pull request). +2. **Build the Release Commit** + - Merge the release commit into `v4.x` through a pull request. + - The `Production Build (v4)` workflow builds AMD64 and ARM64 images and publishes them to Docker Hub and GHCR using immutable architecture tags. + - After both builds complete, the workflow creates the multi-architecture `sha-` manifest in both registries. + - This workflow does not update a semantic version tag or `latest`. -3. **Building the Release** - - After merging to `main`, GitHub Actions automatically builds release images for all architectures and pushes them to the GitHub Container Registry and Docker Hub with the specific version tag and the `latest` tag. +3. **Wait for the SHA Image** + - Confirm the complete `Production Build (v4)` workflow, including its `merge-manifest` job, succeeded. + - Do not publish the release before the multi-architecture SHA image exists in both registries. -4. **Creating a GitHub Release** - - A new GitHub release is manually created with details of the changes made in the version. +4. **Create and Publish the GitHub Release** + - Create a GitHub release with a semantic version tag such as `v4.2.0`, targeting the exact commit that produced the SHA image. + - Mark beta or other test releases as prereleases. Publish production versions as stable releases. + - Publishing the release starts the `Release Coolify` workflow. It verifies that the Git tag matches `config/constants.php`, then promotes the existing SHA image without rebuilding it. + - The workflow assigns the semantic version tag in Docker Hub and GHCR. Stable releases also update `latest`; prereleases do not. -5. **Updating the CDN** - - To make a new version publicly available, the version information on the CDN needs to be updated manually. After that the new version number will be available at [https://cdn.coollabs.io/coolify/versions.json](https://cdn.coollabs.io/coolify/versions.json). +5. **Verify the Promotion** + - Confirm the `Release Coolify` workflow succeeded. + - Verify the semantic version image has the same manifest digest as `sha-` in Docker Hub and GHCR. + - For stable releases, also verify `latest` points to the promoted release manifest. + +6. **Update the CDN** + - To make a new version available to self-hosted instances, update the version information on the CDN manually. + - Confirm the new version is available at [https://cdn.coollabs.io/coolify/versions.json](https://cdn.coollabs.io/coolify/versions.json). > [!NOTE] > The CDN update may not occur immediately after the GitHub release. It can take hours or even days due to additional testing, stability checks, or potential hotfixes. **The update becomes available only after the CDN is updated. After the CDN is updated, a discord announcement will be made in the Production Release channel.** @@ -36,7 +51,7 @@ ## Release Process ## Version Types
- Stable (coming soon) + Stable - **Stable** - The production version suitable for stable, production environments (recommended). diff --git a/app/Actions/Shared/DeleteScheduledVolumeBackup.php b/app/Actions/Shared/DeleteScheduledVolumeBackup.php new file mode 100644 index 000000000..55972ab52 --- /dev/null +++ b/app/Actions/Shared/DeleteScheduledVolumeBackup.php @@ -0,0 +1,71 @@ +id), $backup->timeout + 300); + + if (! $lock->get()) { + throw new \RuntimeException('Wait for the queued or running storage backup to finish before deleting this schedule.'); + } + + try { + if ($backup->executions() + ->where(fn ($query) => $query + ->where('status', 'running') + ->orWhere('stop_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->exists()) { + throw new \RuntimeException('Wait for the running storage backup and recovery operations to finish before deleting this schedule.'); + } + + $localFilenames = $backup->executions() + ->where('local_storage_deleted', false) + ->pluck('filename') + ->filter() + ->all(); + + if ($localFilenames !== []) { + $server ??= $backup->server(); + if (! $server) { + throw new \RuntimeException('The server is unavailable, so local backup archives cannot be deleted.'); + } + + deleteBackupsLocally($localFilenames, $server, throwError: true); + } + + $s3Executions = $backup->executions() + ->with('s3') + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false) + ->get(); + + foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { + $s3 = $executions->first()->s3; + if (! $s3) { + throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + } + + $filenames = $executions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsS3($filenames, $s3); + } + } + + $backup->delete(); + } finally { + $lock->release(); + } + } +} diff --git a/app/Http/Controllers/Api/ApplicationsController.php b/app/Http/Controllers/Api/ApplicationsController.php index 80c3d4ac2..c2592d7f9 100644 --- a/app/Http/Controllers/Api/ApplicationsController.php +++ b/app/Http/Controllers/Api/ApplicationsController.php @@ -4910,6 +4910,8 @@ public function delete_storage(Request $request): JsonResponse ], 422); } + $storage->abortIfScheduledBackupsExist(); + if ($storage instanceof LocalFileVolume) { $storage->deleteStorageOnServer(); } diff --git a/app/Http/Controllers/Api/DatabasesController.php b/app/Http/Controllers/Api/DatabasesController.php index 4c3b0ed43..41af38a72 100644 --- a/app/Http/Controllers/Api/DatabasesController.php +++ b/app/Http/Controllers/Api/DatabasesController.php @@ -4484,6 +4484,8 @@ public function delete_storage(Request $request): JsonResponse ], 422); } + $storage->abortIfScheduledBackupsExist(); + if ($storage instanceof LocalFileVolume) { $storage->deleteStorageOnServer(); } diff --git a/app/Http/Controllers/Api/ServicesController.php b/app/Http/Controllers/Api/ServicesController.php index c87f02156..9945dfcf9 100644 --- a/app/Http/Controllers/Api/ServicesController.php +++ b/app/Http/Controllers/Api/ServicesController.php @@ -2911,6 +2911,8 @@ public function delete_storage(Request $request): JsonResponse ], 422); } + $storage->abortIfScheduledBackupsExist(); + if ($storage instanceof LocalFileVolume) { $storage->deleteStorageOnServer(); } diff --git a/app/Http/Controllers/Api/VolumeBackupsController.php b/app/Http/Controllers/Api/VolumeBackupsController.php new file mode 100644 index 000000000..e30cdad38 --- /dev/null +++ b/app/Http/Controllers/Api/VolumeBackupsController.php @@ -0,0 +1,445 @@ + []]], + requestBody: new OA\RequestBody(required: true, content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleRequest')), + tags: ['Applications'], + parameters: [ + new OA\Parameter(name: 'uuid', in: 'path', required: true, description: 'UUID of the application.', schema: new OA\Schema(type: 'string')), + new OA\Parameter(name: 'storage_uuid', in: 'path', required: true, description: 'UUID of the persistent volume or directory storage.', schema: new OA\Schema(type: 'string')), + ], + responses: [ + new OA\Response(response: 200, description: 'Backup schedule replaced.', content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleResponse')), + new OA\Response(response: 201, description: 'Backup schedule created.', content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleResponse')), + new OA\Response(response: 400, ref: '#/components/responses/400'), + new OA\Response(response: 401, ref: '#/components/responses/401'), + new OA\Response(response: 403, description: 'Forbidden.'), + new OA\Response(response: 404, ref: '#/components/responses/404'), + new OA\Response(response: 422, ref: '#/components/responses/422'), + ], + )] + #[OA\Put( + summary: 'Set database storage backup schedule', + description: 'Create or replace the backup schedule for a database persistent volume or directory storage.', + path: '/databases/{uuid}/storages/{storage_uuid}/backups', + operationId: 'set-database-storage-backup-schedule', + security: [['bearerAuth' => []]], + requestBody: new OA\RequestBody(required: true, content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleRequest')), + tags: ['Databases'], + parameters: [ + new OA\Parameter(name: 'uuid', in: 'path', required: true, description: 'UUID of the database.', schema: new OA\Schema(type: 'string')), + new OA\Parameter(name: 'storage_uuid', in: 'path', required: true, description: 'UUID of the persistent volume or directory storage.', schema: new OA\Schema(type: 'string')), + ], + responses: [ + new OA\Response(response: 200, description: 'Backup schedule replaced.', content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleResponse')), + new OA\Response(response: 201, description: 'Backup schedule created.', content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleResponse')), + new OA\Response(response: 400, ref: '#/components/responses/400'), + new OA\Response(response: 401, ref: '#/components/responses/401'), + new OA\Response(response: 403, description: 'Forbidden.'), + new OA\Response(response: 404, ref: '#/components/responses/404'), + new OA\Response(response: 422, ref: '#/components/responses/422'), + ], + )] + #[OA\Put( + summary: 'Set service storage backup schedule', + description: 'Create or replace the backup schedule for a service persistent volume or directory storage.', + path: '/services/{uuid}/storages/{storage_uuid}/backups', + operationId: 'set-service-storage-backup-schedule', + security: [['bearerAuth' => []]], + requestBody: new OA\RequestBody(required: true, content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleRequest')), + tags: ['Services'], + parameters: [ + new OA\Parameter(name: 'uuid', in: 'path', required: true, description: 'UUID of the service.', schema: new OA\Schema(type: 'string')), + new OA\Parameter(name: 'storage_uuid', in: 'path', required: true, description: 'UUID of the persistent volume or directory storage.', schema: new OA\Schema(type: 'string')), + ], + responses: [ + new OA\Response(response: 200, description: 'Backup schedule replaced.', content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleResponse')), + new OA\Response(response: 201, description: 'Backup schedule created.', content: new OA\JsonContent(ref: '#/components/schemas/VolumeBackupScheduleResponse')), + new OA\Response(response: 400, ref: '#/components/responses/400'), + new OA\Response(response: 401, ref: '#/components/responses/401'), + new OA\Response(response: 403, description: 'Forbidden.'), + new OA\Response(response: 404, ref: '#/components/responses/404'), + new OA\Response(response: 422, ref: '#/components/responses/422'), + ], + )] + public function upsert(Request $request): JsonResponse + { + $teamId = getTeamIdFromToken(); + if (is_null($teamId)) { + return invalidTokenResponse(); + } + + $invalidRequest = validateIncomingRequest($request); + if ($invalidRequest instanceof JsonResponse) { + return $invalidRequest; + } + + $resourceType = $request->route('resource_type'); + $resource = $this->findResource($resourceType, $request->route('uuid'), $teamId); + if (! $resource) { + return response()->json([ + 'message' => match ($resourceType) { + 'application' => 'Application not found.', + 'database' => 'Database not found.', + 'service' => 'Service not found.', + default => 'Resource not found.', + }, + ], 404); + } + + $this->authorize('update', $resource); + + $storage = $this->findStorage($resource, $request->route('storage_uuid')); + if (! $storage) { + return response()->json(['message' => 'Storage not found.'], 404); + } + + ['errors' => $errors, 's3Storage' => $s3Storage, 'saveToS3' => $saveToS3] = $this->validateUpsertRequest($request, $storage, $teamId); + + if ($errors->isNotEmpty()) { + return response()->json([ + 'message' => 'Validation failed.', + 'errors' => $errors, + ], 422); + } + + return $this->persistSchedule($request, $storage, $teamId, $s3Storage, $saveToS3, $resourceType, $resource); + } + + /** + * @return array{errors: MessageBag, s3Storage: S3Storage|null, saveToS3: bool} + */ + private function validateUpsertRequest( + Request $request, + LocalPersistentVolume|LocalFileVolume $storage, + int|string $teamId, + ): array { + $validator = customApiValidator($request->all(), [ + 'frequency' => 'required|string|max:255', + 'enabled' => 'boolean', + 'save_s3' => 'boolean', + 'disable_local_backup' => 'boolean', + 'stop_during_backup' => 'boolean', + 's3_storage_uuid' => 'nullable|string', + 'retention_amount_locally' => 'integer|min:0|max:10000', + 'retention_days_locally' => 'integer|min:0|max:2147483647', + 'retention_max_storage_locally' => 'numeric|min:0|max:9999999999', + 'retention_amount_s3' => 'integer|min:0|max:10000', + 'retention_days_s3' => 'integer|min:0|max:2147483647', + 'retention_max_storage_s3' => 'numeric|min:0|max:9999999999', + 'timeout' => 'integer|min:60|max:36000', + ]); + $errors = $validator->errors(); + $allowedFields = [ + 'frequency', + 'enabled', + 'save_s3', + 'disable_local_backup', + 'stop_during_backup', + 's3_storage_uuid', + 'retention_amount_locally', + 'retention_days_locally', + 'retention_max_storage_locally', + 'retention_amount_s3', + 'retention_days_s3', + 'retention_max_storage_s3', + 'timeout', + ]; + + foreach (array_diff(array_keys($request->all()), $allowedFields) as $field) { + $errors->add($field, 'This field is not allowed.'); + } + + if (! $errors->has('frequency') && ! validate_cron_expression($request->string('frequency')->toString())) { + $errors->add('frequency', 'The frequency must be a valid cron or human expression.'); + } + + $saveToS3 = $request->boolean('save_s3'); + if ($request->boolean('disable_local_backup') && ! $saveToS3) { + $errors->add('disable_local_backup', 'Local backups can only be disabled when S3 backups are enabled.'); + } + + $s3Storage = null; + if ($saveToS3) { + $s3Storage = S3Storage::query() + ->where('team_id', $teamId) + ->where('is_usable', true) + ->where('uuid', $request->input('s3_storage_uuid')) + ->first(); + + if (! $s3Storage) { + $errors->add('s3_storage_uuid', 'Select a usable S3 storage owned by your team.'); + } + } + + if ($storage instanceof LocalFileVolume && (! $storage->is_directory || $storage->is_host_file)) { + $errors->add('storage_uuid', 'Only directory file storages can be backed up.'); + } + + return [ + 'errors' => $errors, + 's3Storage' => $s3Storage, + 'saveToS3' => $saveToS3, + ]; + } + + private function persistSchedule( + Request $request, + LocalPersistentVolume|LocalFileVolume $storage, + int|string $teamId, + ?S3Storage $s3Storage, + bool $saveToS3, + string $resourceType, + Model $resource, + ): JsonResponse { + $backup = $storage->scheduledBackups()->updateOrCreate([], [ + 'team_id' => $teamId, + 'frequency' => $request->string('frequency')->toString(), + 'enabled' => $request->boolean('enabled', true), + 'save_s3' => $saveToS3, + 'disable_local_backup' => $saveToS3 && $request->boolean('disable_local_backup'), + 'stop_during_backup' => $request->boolean('stop_during_backup'), + 's3_storage_id' => $s3Storage?->id, + 'retention_amount_locally' => $request->integer('retention_amount_locally', 7), + 'retention_days_locally' => $request->integer('retention_days_locally'), + 'retention_max_storage_locally' => $request->float('retention_max_storage_locally'), + 'retention_amount_s3' => $request->integer('retention_amount_s3', 7), + 'retention_days_s3' => $request->integer('retention_days_s3'), + 'retention_max_storage_s3' => $request->float('retention_max_storage_s3'), + 'timeout' => $request->integer('timeout', 3600), + ]); + $created = $backup->wasRecentlyCreated; + + auditLog('api.volume_backup.schedule_set', [ + 'team_id' => $teamId, + 'resource_type' => $resourceType, + 'resource_uuid' => $resource->uuid, + 'storage_uuid' => $storage->uuid, + 'backup_uuid' => $backup->uuid, + ]); + + return response()->json($this->responseData($backup, $storage, $s3Storage, $created), $created ? 201 : 200); + } + + #[OA\Delete( + summary: 'Delete application storage backup schedule', + description: 'Delete the backup schedule and its local and S3 archives for an application storage.', + path: '/applications/{uuid}/storages/{storage_uuid}/backups', + operationId: 'delete-application-storage-backup-schedule', + security: [['bearerAuth' => []]], + tags: ['Applications'], + parameters: [ + new OA\Parameter(name: 'uuid', in: 'path', required: true, schema: new OA\Schema(type: 'string')), + new OA\Parameter(name: 'storage_uuid', in: 'path', required: true, schema: new OA\Schema(type: 'string')), + ], + responses: [ + new OA\Response(response: 200, description: 'Backup schedule and archives deleted.'), + new OA\Response(response: 401, ref: '#/components/responses/401'), + new OA\Response(response: 403, description: 'Forbidden.'), + new OA\Response(response: 404, ref: '#/components/responses/404'), + new OA\Response(response: 409, description: 'Backup or recovery operation is still running.'), + ], + )] + #[OA\Delete( + summary: 'Delete database storage backup schedule', + description: 'Delete the backup schedule and its local and S3 archives for a database storage.', + path: '/databases/{uuid}/storages/{storage_uuid}/backups', + operationId: 'delete-database-storage-backup-schedule', + security: [['bearerAuth' => []]], + tags: ['Databases'], + parameters: [ + new OA\Parameter(name: 'uuid', in: 'path', required: true, schema: new OA\Schema(type: 'string')), + new OA\Parameter(name: 'storage_uuid', in: 'path', required: true, schema: new OA\Schema(type: 'string')), + ], + responses: [ + new OA\Response(response: 200, description: 'Backup schedule and archives deleted.'), + new OA\Response(response: 401, ref: '#/components/responses/401'), + new OA\Response(response: 403, description: 'Forbidden.'), + new OA\Response(response: 404, ref: '#/components/responses/404'), + new OA\Response(response: 409, description: 'Backup or recovery operation is still running.'), + ], + )] + #[OA\Delete( + summary: 'Delete service storage backup schedule', + description: 'Delete the backup schedule and its local and S3 archives for a service storage.', + path: '/services/{uuid}/storages/{storage_uuid}/backups', + operationId: 'delete-service-storage-backup-schedule', + security: [['bearerAuth' => []]], + tags: ['Services'], + parameters: [ + new OA\Parameter(name: 'uuid', in: 'path', required: true, schema: new OA\Schema(type: 'string')), + new OA\Parameter(name: 'storage_uuid', in: 'path', required: true, schema: new OA\Schema(type: 'string')), + ], + responses: [ + new OA\Response(response: 200, description: 'Backup schedule and archives deleted.'), + new OA\Response(response: 401, ref: '#/components/responses/401'), + new OA\Response(response: 403, description: 'Forbidden.'), + new OA\Response(response: 404, ref: '#/components/responses/404'), + new OA\Response(response: 409, description: 'Backup or recovery operation is still running.'), + ], + )] + public function destroy(Request $request): JsonResponse + { + $teamId = getTeamIdFromToken(); + if (is_null($teamId)) { + return invalidTokenResponse(); + } + + $resourceType = $request->route('resource_type'); + $resource = $this->findResource($resourceType, $request->route('uuid'), $teamId); + if (! $resource) { + return response()->json(['message' => 'Resource not found.'], 404); + } + + $this->authorize('update', $resource); + + $storage = $this->findStorage($resource, $request->route('storage_uuid')); + if (! $storage) { + return response()->json(['message' => 'Storage not found.'], 404); + } + + $backup = $storage->scheduledBackups()->first(); + if (! $backup) { + return response()->json(['message' => 'Storage backup schedule not found.'], 404); + } + + try { + DeleteScheduledVolumeBackup::run($backup); + } catch (RuntimeException $exception) { + return response()->json(['message' => $exception->getMessage()], 409); + } + + auditLog('api.volume_backup.schedule_deleted', [ + 'team_id' => $teamId, + 'resource_type' => $resourceType, + 'resource_uuid' => $resource->uuid, + 'storage_uuid' => $storage->uuid, + 'backup_uuid' => $backup->uuid, + ]); + + return response()->json(['message' => 'Storage backup schedule and archives deleted.']); + } + + private function findResource(string $resourceType, string $uuid, int|string $teamId): ?Model + { + return match ($resourceType) { + 'application' => Application::ownedByCurrentTeamAPI($teamId)->where('uuid', $uuid)->first(), + 'database' => queryDatabaseByUuidWithinTeam($uuid, $teamId), + 'service' => Service::query()->whereRelation('environment.project.team', 'id', $teamId)->where('uuid', $uuid)->first(), + default => null, + }; + } + + private function findStorage(Model $resource, string $storageUuid): LocalPersistentVolume|LocalFileVolume|null + { + if ($resource instanceof Service) { + foreach ($resource->applications->concat($resource->databases) as $serviceResource) { + $storage = $this->findStorage($serviceResource, $storageUuid); + if ($storage) { + return $storage; + } + } + + return null; + } + + $storage = $resource->persistentStorages()->where('uuid', $storageUuid)->first(); + + return $storage ?? $resource->fileStorages()->where('uuid', $storageUuid)->first(); + } + + private function responseData( + ScheduledVolumeBackup $backup, + LocalPersistentVolume|LocalFileVolume $storage, + ?S3Storage $s3Storage, + bool $created, + ): array { + return [ + 'uuid' => $backup->uuid, + 'message' => $created ? 'Storage backup schedule created.' : 'Storage backup schedule updated.', + 'storage_uuid' => $storage->uuid, + 'storage_type' => $storage instanceof LocalFileVolume ? 'directory' : 'persistent', + 'frequency' => $backup->frequency, + 'enabled' => $backup->enabled, + 'save_s3' => $backup->save_s3, + 'disable_local_backup' => $backup->disable_local_backup, + 'stop_during_backup' => $backup->stop_during_backup, + 's3_storage_uuid' => $s3Storage?->uuid, + 'retention_amount_locally' => $backup->retention_amount_locally, + 'retention_days_locally' => $backup->retention_days_locally, + 'retention_max_storage_locally' => $backup->retention_max_storage_locally, + 'retention_amount_s3' => $backup->retention_amount_s3, + 'retention_days_s3' => $backup->retention_days_s3, + 'retention_max_storage_s3' => $backup->retention_max_storage_s3, + 'timeout' => $backup->timeout, + ]; + } +} diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php index f25158d34..c80da0cab 100644 --- a/app/Jobs/DatabaseBackupJob.php +++ b/app/Jobs/DatabaseBackupJob.php @@ -497,7 +497,7 @@ public function handle(): void } } if ($this->backup_log && $this->backup_log->status === 'success') { - removeOldBackups($this->backup); + $this->removeExpiredBackups(); } } catch (Throwable $e) { throw $e; @@ -513,6 +513,19 @@ public function handle(): void } } + private function removeExpiredBackups(): void + { + try { + removeOldBackups($this->backup); + } catch (Throwable $exception) { + Log::channel('scheduled-errors')->warning('Database backup retention cleanup failed', [ + 'backup_id' => $this->backup->id, + 'execution_id' => $this->backup_log?->id, + 'error' => $exception->getMessage(), + ]); + } + } + private function backup_standalone_mongodb(string $databaseWithCollections): void { try { diff --git a/app/Jobs/DeleteResourceJob.php b/app/Jobs/DeleteResourceJob.php index 825604910..656e5c409 100644 --- a/app/Jobs/DeleteResourceJob.php +++ b/app/Jobs/DeleteResourceJob.php @@ -7,7 +7,10 @@ use App\Actions\Server\CleanupDocker; use App\Actions\Service\DeleteService; use App\Actions\Service\StopService; +use App\Actions\Shared\DeleteScheduledVolumeBackup; +use App\Enums\ApplicationDeploymentStatus; use App\Models\Application; +use App\Models\ApplicationDeploymentQueue; use App\Models\ApplicationPreview; use App\Models\Service; use App\Models\StandaloneClickhouse; @@ -42,6 +45,10 @@ public function __construct( public function handle() { + if (! $this->resource instanceof ApplicationPreview) { + $this->deleteScheduledVolumeBackups(); + } + try { // Handle ApplicationPreview instances separately if ($this->resource instanceof ApplicationPreview) { @@ -113,6 +120,24 @@ public function handle() } } + private function deleteScheduledVolumeBackups(): void + { + $server = data_get($this->resource, 'server') ?? data_get($this->resource, 'destination.server'); + $resources = $this->resource instanceof Service + ? $this->resource->applications()->get()->concat($this->resource->databases()->get()) + : collect([$this->resource]); + + foreach ($resources as $resource) { + $storages = $resource->persistentStorages()->get()->concat($resource->fileStorages()->get()); + + foreach ($storages as $storage) { + foreach ($storage->scheduledBackups()->get() as $backup) { + DeleteScheduledVolumeBackup::run($backup, $server); + } + } + } + } + private function deleteApplicationPreview() { $application = $this->resource->application; @@ -125,11 +150,11 @@ private function deleteApplicationPreview() } // Cancel any active deployments for this PR (same logic as API cancel_deployment) - $activeDeployments = \App\Models\ApplicationDeploymentQueue::where('application_id', $application->id) + $activeDeployments = ApplicationDeploymentQueue::where('application_id', $application->id) ->where('pull_request_id', $pull_request_id) ->whereIn('status', [ - \App\Enums\ApplicationDeploymentStatus::QUEUED->value, - \App\Enums\ApplicationDeploymentStatus::IN_PROGRESS->value, + ApplicationDeploymentStatus::QUEUED->value, + ApplicationDeploymentStatus::IN_PROGRESS->value, ]) ->get(); @@ -137,7 +162,7 @@ private function deleteApplicationPreview() try { // Mark deployment as cancelled $activeDeployment->update([ - 'status' => \App\Enums\ApplicationDeploymentStatus::CANCELLED_BY_USER->value, + 'status' => ApplicationDeploymentStatus::CANCELLED_BY_USER->value, ]); // Add cancellation log entry diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php index 46bc89d42..6e2fab14a 100644 --- a/app/Jobs/ScheduledJobManager.php +++ b/app/Jobs/ScheduledJobManager.php @@ -2,8 +2,11 @@ namespace App\Jobs; +use App\Actions\Shared\DeleteScheduledVolumeBackup; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledTask; +use App\Models\ScheduledVolumeBackup; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\Team; use Cron\CronExpression; @@ -109,6 +112,16 @@ public function handle(): void ]); } + try { + $this->recoverStoppedVolumeBackupContainers(); + $this->processScheduledVolumeBackups(); + } catch (\Exception $e) { + Log::channel('scheduled-errors')->error('Failed to process scheduled volume backups', [ + 'error' => $e->getMessage(), + 'trace' => $e->getTraceAsString(), + ]); + } + // Process Docker cleanups - don't let failures stop the job manager try { $this->processDockerCleanups(); @@ -341,6 +354,112 @@ private function processScheduledTask(ScheduledTask $task, ?Server $precheckedSe } } + private function processScheduledVolumeBackups(): void + { + ScheduledVolumeBackup::query() + ->with(['backupable', 'team.subscription']) + ->where('enabled', true) + ->chunkById(self::CHUNK_SIZE, function ($backups): void { + foreach ($backups as $backup) { + $this->processScheduledVolumeBackup($backup); + } + }); + } + + private function recoverStoppedVolumeBackupContainers(): void + { + ScheduledVolumeBackupExecution::query() + ->where(fn (Builder $query) => $query + ->where('stop_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->chunkById(self::CHUNK_SIZE, function ($executions): void { + foreach ($executions as $execution) { + VolumeBackupRecoveryJob::dispatch($execution); + } + }); + } + + private function processScheduledVolumeBackup(ScheduledVolumeBackup $backup): void + { + try { + $server = $backup->server(); + + if ($backup->executions() + ->where(fn (Builder $query) => $query + ->where('stop_recovery_pending', true) + ->orWhere('s3_cleanup_pending', true)) + ->exists()) { + $this->skippedCount++; + $this->logSkip('volume_backup', 'container_recovery_pending', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + ]); + + return; + } + + if (! $backup->backupable) { + DeleteScheduledVolumeBackup::run($backup, $server); + $this->skippedCount++; + $this->logSkip('volume_backup', 'resource_deleted', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + ]); + + return; + } + + if (! $server) { + $this->skippedCount++; + $this->logSkip('volume_backup', 'server_missing', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + ]); + + return; + } + + if (! $server->isFunctional()) { + $this->skippedCount++; + $this->logSkip('volume_backup', 'server_not_functional', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + 'server_id' => $server->id, + ]); + + return; + } + + if (isCloud() && $backup->team_id !== 0 && ! data_get($backup, 'team.subscription.stripe_invoice_paid', false)) { + $this->skippedCount++; + $this->logSkip('volume_backup', 'subscription_unpaid', [ + 'backup_id' => $backup->id, + 'team_id' => $backup->team_id, + 'server_id' => $server->id, + ]); + + return; + } + + if ($this->shouldDispatch($backup->frequency, $server, "scheduled-volume-backup:{$backup->id}")) { + VolumeBackupJob::dispatch($backup); + $this->dispatchedCount++; + Log::channel('scheduled')->info('Volume backup dispatched', [ + 'backup_id' => $backup->id, + 'backupable_type' => $backup->backupable_type, + 'backupable_id' => $backup->backupable_id, + 'team_id' => $backup->team_id, + 'server_id' => $server->id, + ]); + } + } catch (\Exception $e) { + Log::channel('scheduled-errors')->error('Error processing volume backup', [ + 'backup_id' => $backup->id, + 'error' => $e->getMessage(), + ]); + } + } + private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string { if (blank(data_get($backup, 'database'))) { diff --git a/app/Jobs/VolumeBackupJob.php b/app/Jobs/VolumeBackupJob.php new file mode 100644 index 000000000..a2c2360d0 --- /dev/null +++ b/app/Jobs/VolumeBackupJob.php @@ -0,0 +1,443 @@ +onQueue(crons_queue()); + $this->timeout = $backup->timeout ?? 3600; + } + + public function middleware(): array + { + return [ + (new WithoutOverlapping('volume-backup-'.$this->backup->id)) + ->shared() + ->expireAfter($this->timeout + 60) + ->dontRelease(), + ]; + } + + public static function lockKey(int $backupId): string + { + return 'laravel-queue-overlap:volume-backup-'.$backupId; + } + + public function handle(): void + { + $this->backup->loadMissing(['backupable.resource', 'team', 's3']); + $server = $this->backup->server(); + $target = $this->backup->backupable; + $team = $this->backup->team; + + if (! $server || ! $target || ! $team) { + throw new \RuntimeException('The storage backup resource, team, or server no longer exists.'); + } + + $this->execution = $this->backup->executions()->create([ + 's3_storage_id' => $this->backup->save_s3 ? $this->backup->s3_storage_id : null, + ]); + BackupCreated::dispatch($team->id); + + $backupDirectory = backup_dir().'/volumes/'.str($team->name)->slug().'-'.$team->id.'/'.$target->uuid; + $filename = str($this->backup->targetType())->lower().'-'.str($this->backup->targetName())->slug().'-'.Carbon::now()->timestamp.'.tar.gz'; + $backupLocation = $backupDirectory.'/'.$filename; + $this->execution->update(['filename' => $backupLocation]); + + try { + $source = $this->backup->sourcePath(); + $containerName = 'volume-backup-'.$this->execution->uuid; + $image = coolifyHelperImage().':'.getHelperVersion(); + $verifySourceCommand = $target instanceof LocalPersistentVolume && blank($target->host_path) + ? 'docker volume inspect '.escapeshellarg($source).' >/dev/null' + : 'test -d '.escapeshellarg($source); + + $archiveCommand = 'docker run --rm --name '.escapeshellarg($containerName) + .' -v '.escapeshellarg($source.':/volume:ro') + .' '.escapeshellarg($image) + .' tar -czf - -C /volume . > '.escapeshellarg($backupLocation); + + if ($this->backup->stop_during_backup) { + $containers = $this->containersUsingVolume($source, $server); + if ($containers !== []) { + $this->execution->update(['stop_recovery_pending' => true]); + $archiveCommand = $this->archiveWithStoppedContainers( + $archiveCommand, + $containers, + VolumeBackupRecoveryJob::stateFile($this->execution), + ); + } + } + + instant_remote_process([ + $verifySourceCommand, + 'mkdir -p '.escapeshellarg($backupDirectory), + $archiveCommand, + ], $server, timeout: $this->timeout, disableMultiplexing: true); + $this->execution->update([ + 'stop_container_ids' => null, + 'stop_recovery_pending' => false, + ]); + + $size = (int) instant_remote_process( + ['du -b '.escapeshellarg($backupLocation).' | cut -f1'], + $server, + disableMultiplexing: true, + ); + + if ($size <= 0) { + throw new \RuntimeException('The storage backup archive is empty or was not created.'); + } + + $warning = null; + $s3Uploaded = null; + $s3CleanupPending = false; + $localStorageDeleted = false; + + if ($this->backup->save_s3) { + $s3CleanupPending = true; + $this->execution->update(['s3_cleanup_pending' => true]); + + try { + $this->uploadToS3($backupLocation, $backupDirectory, $server); + $s3Uploaded = true; + $s3CleanupPending = false; + } catch (Throwable $exception) { + $s3Uploaded = false; + $warning = 'S3 upload failed: '.$exception->getMessage(); + + try { + VolumeBackupRecoveryJob::cleanupS3Upload($this->execution); + $s3CleanupPending = false; + } catch (Throwable $cleanupException) { + VolumeBackupRecoveryJob::dispatch($this->execution); + $warning .= ' Partial S3 upload cleanup failed: '.$cleanupException->getMessage(); + } + } + + if ($s3Uploaded && $this->backup->disable_local_backup) { + try { + deleteBackupsLocally($backupLocation, $server, throwError: true); + $localStorageDeleted = true; + } catch (Throwable $exception) { + $warning = 'S3 upload succeeded, but the local archive could not be deleted: '.$exception->getMessage(); + } + } + } + + $this->execution->update([ + 'status' => 'success', + 'message' => $warning, + 'size' => $size, + 'filename' => $backupLocation, + 's3_uploaded' => $s3Uploaded, + 's3_cleanup_pending' => $s3CleanupPending, + 'local_storage_deleted' => $localStorageDeleted, + ]); + + try { + $this->removeExpiredBackups($server); + } catch (Throwable $exception) { + Log::channel('scheduled-errors')->warning('Volume backup retention cleanup failed', [ + 'backup_id' => $this->backup->id, + 'execution_id' => $this->execution->id, + 'error' => $exception->getMessage(), + ]); + } + } catch (Throwable $exception) { + $recoveryError = $this->recoverIncompleteBackup($this->execution); + $archiveDeleted = false; + + try { + deleteBackupsLocally($backupLocation, $server, throwError: true); + $archiveDeleted = true; + } catch (Throwable $cleanupException) { + $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage(); + } + + $s3CleanupPending = $this->execution->fresh()->s3_cleanup_pending; + + $this->execution->update([ + 'status' => 'failed', + 'message' => $exception->getMessage().$recoveryError, + 'filename' => $archiveDeleted && ! $s3CleanupPending ? null : $backupLocation, + 'local_storage_deleted' => $archiveDeleted, + ]); + + throw $exception; + } finally { + $this->execution->update(['finished_at' => now()]); + BackupCreated::dispatch($team->id); + } + } + + public function failed(?Throwable $exception): void + { + $execution = $this->execution ?? $this->backup->executions() + ->where('status', 'running') + ->latest('id') + ->first(); + + if ($execution) { + $recoveryError = $this->recoverIncompleteBackup($execution); + $server = $this->backup->server(); + $filename = $execution->filename; + $localStorageDeleted = $execution->local_storage_deleted; + + if ($server && filled($filename)) { + try { + deleteBackupsLocally($filename, $server, throwError: true); + $localStorageDeleted = true; + if (! $execution->fresh()->s3_cleanup_pending) { + $filename = null; + } + } catch (Throwable $cleanupException) { + $recoveryError .= ' Archive cleanup failed: '.$cleanupException->getMessage(); + } + } + + $execution->update([ + 'status' => 'failed', + 'message' => ($exception?->getMessage() ?? 'Volume backup timed out or was terminated.').$recoveryError, + 'finished_at' => now(), + 'filename' => $filename, + 'local_storage_deleted' => $localStorageDeleted, + ]); + } + } + + /** + * @return array + */ + private function containersUsingVolume(string $source, Server $server): array + { + $output = instant_remote_process( + ["containers=\$(docker ps -q) || exit 1; for container in \$containers; do paused=\$(docker inspect --format '{{.State.Paused}}' \"\$container\") || exit 1; [ \"\$paused\" = true ] && continue; mounts=\$(docker inspect --format '{{range .Mounts}}{{println .Source}}{{println .Name}}{{end}}' \"\$container\") || exit 1; if printf '%s\\n' \"\$mounts\" | grep -Fqx -- ".escapeshellarg($source).'; then echo "$container"; fi; done'], + $server, + disableMultiplexing: true, + ); + $containers = collect(preg_split('/\s+/', trim((string) $output))) + ->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1) + ->values() + ->all(); + + return $containers; + } + + /** + * @param array $containers + */ + private function archiveWithStoppedContainers(string $archiveCommand, array $containers, string $stateFile): string + { + if ($containers === []) { + return $archiveCommand; + } + + $containerList = implode(' ', $containers); + $script = "set -eu\n" + .'state_file='.escapeshellarg($stateFile)."\n" + .": > \"\$state_file\"\n" + ."cleanup() { status=\$?; trap - EXIT HUP INT TERM; while IFS= read -r container; do docker start \"\$container\" >/dev/null || status=1; done < \"\$state_file\"; [ \$status -eq 0 ] && rm -f \"\$state_file\"; exit \$status; }\n" + ."trap cleanup EXIT\n" + ."trap 'exit 129' HUP\n" + ."trap 'exit 130' INT\n" + ."trap 'exit 143' TERM\n" + ."for container in {$containerList}; do echo \"\$container\" >> \"\$state_file\"; docker stop \"\$container\" >/dev/null; done\n" + .$archiveCommand; + + return 'sh -c '.escapeshellarg($script); + } + + private function recoverIncompleteBackup(ScheduledVolumeBackupExecution $execution): string + { + if (! $execution->stop_recovery_pending && ! $execution->s3_cleanup_pending) { + return ''; + } + + try { + VolumeBackupRecoveryJob::recover($execution); + + return ''; + } catch (Throwable $exception) { + VolumeBackupRecoveryJob::dispatch($execution); + + return ' Container recovery failed: '.$exception->getMessage(); + } + } + + private function uploadToS3(string $backupLocation, string $backupDirectory, Server $server): void + { + $s3 = $this->backup->s3; + + if (! $s3) { + $this->backup->update(['save_s3' => false, 's3_storage_id' => null]); + + throw new \RuntimeException('The selected S3 storage no longer exists. S3 backup has been disabled.'); + } + + $s3->testConnection(shouldSave: true); + $containerName = 'volume-upload-'.$this->execution->uuid; + $image = coolifyHelperImage().':'.getHelperVersion(); + $resolveOptions = collect(SafeWebhookUrl::minioClientResolveOptions($s3->endpoint)) + ->map(fn (string $option): string => '--resolve '.escapeshellarg($option)) + ->implode(' '); + $resolveOptions = $resolveOptions === '' ? '' : ' '.$resolveOptions; + + try { + instant_remote_process([ + 'docker rm -f '.escapeshellarg($containerName).' >/dev/null 2>&1 || true', + 'docker run -d --name '.escapeshellarg($containerName).' --rm -v ' + .escapeshellarg($backupLocation.':'.$backupLocation.':ro').' '.escapeshellarg($image), + 'docker exec '.escapeshellarg($containerName).' mc alias set'.$resolveOptions.' temporary ' + .escapeshellarg($s3->endpoint).' '.escapeshellarg($s3->key).' '.escapeshellarg($s3->secret), + 'docker exec '.escapeshellarg($containerName).' mc cp '.escapeshellarg($backupLocation).' ' + .escapeshellarg('temporary/'.$s3->bucket.$backupDirectory.'/'), + ], $server, timeout: $this->timeout, disableMultiplexing: true); + } finally { + instant_remote_process( + ['docker rm -f '.escapeshellarg($containerName)], + $server, + throwError: false, + disableMultiplexing: true, + ); + } + } + + private function removeExpiredBackups(Server $server): void + { + if ($this->hasRetentionLimits( + $this->backup->retention_amount_locally, + $this->backup->retention_days_locally, + $this->backup->retention_max_storage_locally, + )) { + $localExecutions = $this->backup->executions() + ->where('status', 'success') + ->where('local_storage_deleted', false) + ->get(); + $localExecutions = $this->executionsOutsideRetention( + $localExecutions, + $this->backup->retention_amount_locally, + $this->backup->retention_days_locally, + $this->backup->retention_max_storage_locally, + ); + + $filenames = $localExecutions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsLocally($filenames, $server, throwError: true); + $this->backup->executions()->whereKey($localExecutions->pluck('id')->all()) + ->update(['local_storage_deleted' => true]); + } + } + + if ($this->backup->save_s3 && $this->backup->s3 && $this->hasRetentionLimits( + $this->backup->retention_amount_s3, + $this->backup->retention_days_s3, + $this->backup->retention_max_storage_s3, + )) { + $s3Executions = $this->backup->executions() + ->with('s3') + ->where('status', 'success') + ->where('s3_uploaded', true) + ->where('s3_storage_deleted', false) + ->get(); + $s3Executions = $this->executionsOutsideRetention( + $s3Executions, + $this->backup->retention_amount_s3, + $this->backup->retention_days_s3, + $this->backup->retention_max_storage_s3, + ); + + foreach ($s3Executions->groupBy('s3_storage_id') as $executions) { + $s3 = $executions->first()->s3; + if (! $s3) { + throw new \RuntimeException('The S3 storage used by an existing backup is unavailable.'); + } + + $filenames = $executions->pluck('filename')->filter()->all(); + if ($filenames !== []) { + deleteBackupsS3($filenames, $s3); + $this->backup->executions()->whereKey($executions->pluck('id')->all()) + ->update(['s3_storage_deleted' => true]); + } + } + } + + $this->backup->executions() + ->where('local_storage_deleted', true) + ->where(function (Builder $query): void { + $query->where('s3_storage_deleted', true)->orWhereNull('s3_uploaded'); + }) + ->delete(); + } + + private function hasRetentionLimits(int $amount, int $days, float $maxStorageGb): bool + { + return $amount > 0 || $days > 0 || $maxStorageGb > 0; + } + + private function executionsOutsideRetention(Collection $executions, int $amount, int $days, float $maxStorageGb): Collection + { + if ($amount === 0 && $days === 0 && $maxStorageGb == 0) { + return collect(); + } + + $executionsToDelete = collect(); + + if ($amount > 0) { + $executionsToDelete = $executionsToDelete->merge($executions->skip($amount)); + } + + if ($days > 0) { + $oldestAllowedDate = now()->subDays($days); + $executionsToDelete = $executionsToDelete->merge( + $executions->filter(fn (ScheduledVolumeBackupExecution $execution): bool => $execution->created_at->isBefore($oldestAllowedDate)), + ); + } + + if ($maxStorageGb > 0) { + $maxStorageBytes = $maxStorageGb * 1024 ** 3; + $totalSize = 0; + + foreach ($executions as $index => $execution) { + $totalSize += (int) $execution->size; + + if ($index > 0 && $totalSize > $maxStorageBytes) { + $executionsToDelete = $executionsToDelete->merge($executions->slice($index)); + + break; + } + } + } + + return $executionsToDelete->unique('id')->values(); + } +} diff --git a/app/Jobs/VolumeBackupRecoveryJob.php b/app/Jobs/VolumeBackupRecoveryJob.php new file mode 100644 index 000000000..a3465b192 --- /dev/null +++ b/app/Jobs/VolumeBackupRecoveryJob.php @@ -0,0 +1,117 @@ +onQueue(crons_queue()); + } + + public function middleware(): array + { + return [ + (new WithoutOverlapping('volume-backup-'.$this->execution->scheduled_volume_backup_id)) + ->shared() + ->expireAfter(300) + ->dontRelease(), + (new WithoutOverlapping('volume-backup-recovery-'.$this->execution->id)) + ->expireAfter(300) + ->dontRelease(), + ]; + } + + public function handle(): void + { + self::recover($this->execution); + } + + public static function recover(ScheduledVolumeBackupExecution $execution): void + { + $execution->loadMissing('scheduledVolumeBackup.backupable.resource'); + + if ($execution->stop_recovery_pending) { + self::recoverContainers($execution); + } + + if ($execution->s3_cleanup_pending) { + self::cleanupS3Upload($execution); + } + } + + private static function recoverContainers(ScheduledVolumeBackupExecution $execution): void + { + $server = $execution->scheduledVolumeBackup?->server(); + + if (! $server) { + throw new \RuntimeException('The server is unavailable for container recovery.'); + } + + $stateFile = self::stateFile($execution); + $output = instant_remote_process( + ['cat '.escapeshellarg($stateFile).' 2>/dev/null || true'], + $server, + disableMultiplexing: true, + ); + $containers = collect(preg_split('/\s+/', trim((string) $output))) + ->filter(fn (string $container): bool => preg_match('/^[a-f0-9]{6,64}$/i', $container) === 1) + ->values() + ->all(); + $execution->update(['stop_container_ids' => $containers]); + + $remainingFile = $stateFile.'.remaining'; + $script = 'status=0; : > '.escapeshellarg($remainingFile).'; ' + .'if [ -f '.escapeshellarg($stateFile).' ]; then while IFS= read -r container; do ' + .'[ -z "$container" ] && continue; running=$(docker inspect --format \'{{.State.Running}}\' "$container" 2>/dev/null) ' + .'|| { echo "$container" >> '.escapeshellarg($remainingFile).'; status=1; continue; }; ' + .'if [ "$running" != true ] && ! docker start "$container" >/dev/null; then echo "$container" >> ' + .escapeshellarg($remainingFile).'; status=1; fi; done < '.escapeshellarg($stateFile).'; fi; ' + .'if [ -s '.escapeshellarg($remainingFile).' ]; then mv '.escapeshellarg($remainingFile).' '.escapeshellarg($stateFile) + .'; else rm -f '.escapeshellarg($stateFile).' '.escapeshellarg($remainingFile).'; fi; exit $status'; + + instant_remote_process(['sh -c '.escapeshellarg($script)], $server, disableMultiplexing: true); + $execution->update([ + 'stop_container_ids' => null, + 'stop_recovery_pending' => false, + ]); + } + + public static function cleanupS3Upload(ScheduledVolumeBackupExecution $execution): void + { + $execution->loadMissing('s3'); + $s3 = $execution->s3; + + if (! $s3 || blank($execution->filename)) { + throw new \RuntimeException('The S3 storage or backup filename is unavailable for upload cleanup.'); + } + + deleteBackupsS3($execution->filename, $s3); + $execution->update([ + 's3_cleanup_pending' => false, + 's3_storage_deleted' => true, + ]); + } + + public static function stateFile(ScheduledVolumeBackupExecution $execution): string + { + return '/tmp/coolify-volume-backup-'.$execution->uuid.'.stopped'; + } +} diff --git a/app/Livewire/Project/Application/Backup/Create.php b/app/Livewire/Project/Application/Backup/Create.php new file mode 100644 index 000000000..4e46ba779 --- /dev/null +++ b/app/Livewire/Project/Application/Backup/Create.php @@ -0,0 +1,145 @@ + ['required', 'string', 'regex:/^(volume|directory):[1-9][0-9]*$/'], + 'frequency' => ['required', 'string'], + ]; + } + + public function mount(): void + { + $this->authorize('view', $this->application); + $volumes = $this->application->persistentStorages() + ->orderBy('name') + ->get() + ->map(fn (LocalPersistentVolume $volume): array => [ + 'key' => 'volume:'.$volume->id, + 'type' => 'Volume', + 'name' => $volume->name, + ]); + $directories = $this->application->fileStorages() + ->where('is_directory', true) + ->where('is_host_file', false) + ->orderBy('fs_path') + ->get() + ->map(fn (LocalFileVolume $directory): array => [ + 'key' => 'directory:'.$directory->id, + 'type' => 'Directory', + 'name' => $directory->fs_path, + ]); + $this->targets = $volumes->concat($directories)->values(); + $this->targetLocked = $this->selectedTargetKey !== null; + $this->targetKey = $this->selectedTargetKey ?? data_get($this->targets->first(), 'key'); + $this->loadSelectedBackup(); + } + + public function updatedTargetKey(): void + { + $this->loadSelectedBackup(); + } + + public function submit(): void + { + $this->authorize('update', $this->application); + $this->validate(); + + $target = $this->selectedTarget(); + if (! $target) { + $this->addError('targetKey', 'Select a volume or directory owned by this application.'); + + return; + } + + if (! validate_cron_expression($this->frequency)) { + $this->addError('frequency', 'The frequency must be a valid cron or human expression.'); + + return; + } + + try { + $backup = $target->scheduledBackups()->updateOrCreate( + [], + [ + 'team_id' => currentTeam()->id, + 'frequency' => $this->frequency, + 'enabled' => true, + ], + ); + + $this->dispatch('success', $backup->wasRecentlyCreated ? 'Scheduled storage backup created.' : 'Scheduled storage backup updated.'); + redirectRoute($this, 'project.application.backup.show', [ + 'project_uuid' => $this->application->project()->uuid, + 'environment_uuid' => $this->application->environment->uuid, + 'application_uuid' => $this->application->uuid, + 'backup_uuid' => $backup->uuid, + ]); + } catch (\Throwable $e) { + handleError($e, $this); + } + } + + public function render() + { + return view('livewire.project.application.backup.create'); + } + + private function loadSelectedBackup(): void + { + $target = $this->selectedTarget(); + if (! $target) { + return; + } + + $backup = $target->scheduledBackups()->first(); + if ($backup) { + $this->frequency = $backup->frequency; + } + } + + private function selectedTarget(): LocalPersistentVolume|LocalFileVolume|null + { + [$type, $id] = array_pad(explode(':', (string) $this->targetKey, 2), 2, null); + if (! ctype_digit((string) $id)) { + return null; + } + + return match ($type) { + 'volume' => $this->application->persistentStorages()->whereKey((int) $id)->first(), + 'directory' => $this->application->fileStorages() + ->whereKey((int) $id) + ->where('is_directory', true) + ->where('is_host_file', false) + ->first(), + default => null, + }; + } +} diff --git a/app/Livewire/Project/Application/Backup/Index.php b/app/Livewire/Project/Application/Backup/Index.php new file mode 100644 index 000000000..130396a9b --- /dev/null +++ b/app/Livewire/Project/Application/Backup/Index.php @@ -0,0 +1,57 @@ + '$refresh']; + + public function mount(): void + { + $this->application = $this->findApplication(); + $this->authorize('view', $this->application); + $this->parameters = get_route_parameters(); + $this->search = request()->string('search')->toString(); + } + + public function render(): View + { + $backups = ScheduledVolumeBackup::query() + ->with(['backupable', 'latestExecution']) + ->withCount('executions') + ->forApplication($this->application) + ->latest() + ->get(); + + return view('livewire.project.application.backup.index', ['backups' => $backups]); + } + + private function findApplication(): Application + { + $project = currentTeam()->projects() + ->where('uuid', request()->route('project_uuid')) + ->firstOrFail(); + $environment = $project->environments() + ->where('uuid', request()->route('environment_uuid')) + ->firstOrFail(); + + return $environment->applications() + ->with('destination.server') + ->where('uuid', request()->route('application_uuid')) + ->firstOrFail(); + } +} diff --git a/app/Livewire/Project/Application/Backup/Show.php b/app/Livewire/Project/Application/Backup/Show.php new file mode 100644 index 000000000..5e10f0c8a --- /dev/null +++ b/app/Livewire/Project/Application/Backup/Show.php @@ -0,0 +1,55 @@ +projects() + ->where('uuid', request()->route('project_uuid')) + ->firstOrFail(); + $environment = $project->environments() + ->where('uuid', request()->route('environment_uuid')) + ->firstOrFail(); + $this->application = $environment->applications() + ->with('destination.server') + ->where('uuid', request()->route('application_uuid')) + ->firstOrFail(); + $this->authorize('view', $this->application); + + $this->backup = ScheduledVolumeBackup::query() + ->with('backupable') + ->where('uuid', request()->route('backup_uuid')) + ->forApplication($this->application) + ->firstOrFail(); + $this->parameters = get_route_parameters(); + $this->section = match (request()->route()?->getName()) { + 'project.application.backup.s3' => 's3', + 'project.application.backup.retention' => 'retention', + 'project.application.backup.executions' => 'executions', + 'project.application.backup.danger' => 'danger', + default => 'general', + }; + } + + public function render() + { + return view('livewire.project.application.backup.show'); + } +} diff --git a/app/Livewire/Project/Database/Backup/Execution.php b/app/Livewire/Project/Database/Backup/Execution.php index 4ac3b2e2c..00c177008 100644 --- a/app/Livewire/Project/Database/Backup/Execution.php +++ b/app/Livewire/Project/Database/Backup/Execution.php @@ -15,6 +15,10 @@ class Execution extends Component public $s3s; + public array $parameters = []; + + public string $section = 'general'; + public function mount() { $backup_uuid = request()->route('backup_uuid'); @@ -39,6 +43,14 @@ public function mount() $this->backup = $backup; $this->executions = $executions; $this->s3s = currentTeam()->s3s; + $this->parameters = get_route_parameters(); + $this->section = match (request()->route()?->getName()) { + 'project.database.backup.s3' => 's3', + 'project.database.backup.retention' => 'retention', + 'project.database.backup.executions' => 'executions', + 'project.database.backup.danger' => 'danger', + default => 'general', + }; } public function render() diff --git a/app/Livewire/Project/Database/BackupEdit.php b/app/Livewire/Project/Database/BackupEdit.php index 1c1bea2f6..1dfea9322 100644 --- a/app/Livewire/Project/Database/BackupEdit.php +++ b/app/Livewire/Project/Database/BackupEdit.php @@ -9,6 +9,7 @@ use Exception; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Auth; use Livewire\Attributes\Locked; use Livewire\Attributes\Validate; use Livewire\Component; @@ -19,6 +20,8 @@ class BackupEdit extends Component public ScheduledDatabaseBackup $backup; + public string $section = 'general'; + #[Locked] public $availableS3Storages; @@ -82,17 +85,49 @@ class BackupEdit extends Component #[Validate(['required', 'int', 'min:60', 'max:36000'])] public int|string $timeout = 3600; + public function getListeners(): array + { + // Keep "Backup Now" in sync when the database starts/stops without a full page refresh. + $listeners = ['databaseUpdated' => 'refreshStatus']; + + $user = Auth::user(); + if (! $user) { + return $listeners; + } + + $listeners["echo-private:user.{$user->id},DatabaseStatusChanged"] = 'refreshStatus'; + + $team = $user->currentTeam(); + if ($team) { + $listeners["echo-private:team.{$team->id},ServiceChecked"] = 'refreshStatus'; + } + + return $listeners; + } + public function mount() { try { $this->authorize('view', $this->backup->database); $this->parameters = get_route_parameters(); $this->syncData(); + $this->refreshStatus(); } catch (Exception $e) { return handleError($e, $this); } } + public function refreshStatus(): void + { + $database = $this->backup->database; + if (! $database) { + return; + } + + $database->refresh(); + $this->status = $database->status; + } + public function syncData(bool $toModel = false) { if ($toModel) { @@ -184,7 +219,11 @@ public function delete($password, $selectedActions = []) 'stack_service_uuid' => $serviceDatabase->uuid, ]); } else { - return redirect()->route('project.database.backup.index', $this->parameters); + return redirect()->route('project.database.backup.index', [ + 'project_uuid' => $this->parameters['project_uuid'], + 'environment_uuid' => $this->parameters['environment_uuid'], + 'database_uuid' => $this->parameters['database_uuid'], + ]); } } catch (Exception $e) { $this->dispatch('error', 'Failed to delete backup: '.$e->getMessage()); @@ -200,6 +239,25 @@ public function backupNow() DatabaseBackupJob::dispatch($this->backup); $this->dispatch('success', 'Backup queued. It will be available in a few minutes.'); + + $database = $this->backup->database; + + if ($database instanceof ServiceDatabase) { + return redirect()->route('project.service.database.backup.executions', [ + 'project_uuid' => $database->service->project()->uuid, + 'environment_uuid' => $database->service->environment->uuid, + 'service_uuid' => $database->service->uuid, + 'stack_service_uuid' => $database->uuid, + 'backup_uuid' => $this->backup->uuid, + ]); + } + + return redirect()->route('project.database.backup.executions', [ + 'project_uuid' => $database->project()->uuid, + 'environment_uuid' => $database->environment->uuid, + 'database_uuid' => $database->uuid, + 'backup_uuid' => $this->backup->uuid, + ]); } catch (\Throwable $e) { return handleError($e, $this); } @@ -217,11 +275,37 @@ public function instantSave() } } + public function toggleEnabled(): void + { + try { + $this->authorize('manageBackups', $this->backup->database); + + $this->backupEnabled = ! $this->backupEnabled; + $this->backup->update(['enabled' => $this->backupEnabled]); + $this->dispatch('success', $this->backupEnabled ? 'Backup enabled.' : 'Backup disabled.'); + } catch (\Throwable $e) { + $this->dispatch('error', $e->getMessage()); + } + } + public function updatedS3StorageId(): void { $this->instantSave(); } + public function toggleS3(): void + { + if (! $this->saveS3 && $this->availableS3StorageIds()->isEmpty()) { + $this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.'); + + return; + } + + $this->saveS3 = ! $this->saveS3; + $this->disableLocalBackup = $this->saveS3 && $this->disableLocalBackup; + $this->instantSave(); + } + private function customValidate() { if (! is_numeric($this->backup->s3_storage_id)) { diff --git a/app/Livewire/Project/Database/CreateScheduledBackup.php b/app/Livewire/Project/Database/CreateScheduledBackup.php index 49065711b..e72277e80 100644 --- a/app/Livewire/Project/Database/CreateScheduledBackup.php +++ b/app/Livewire/Project/Database/CreateScheduledBackup.php @@ -2,11 +2,9 @@ namespace App\Livewire\Project\Database; -use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; use App\Models\ServiceDatabase; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; -use Illuminate\Support\Collection; use Livewire\Attributes\Locked; use Livewire\Attributes\Validate; use Livewire\Component; @@ -18,31 +16,11 @@ class CreateScheduledBackup extends Component #[Validate(['required', 'string'])] public $frequency; - #[Validate(['required', 'boolean'])] - public bool $saveToS3 = false; - #[Locked] public $database; public bool $enabled = true; - #[Validate(['nullable', 'integer'])] - public ?int $s3StorageId = null; - - public Collection $definedS3s; - - public function mount() - { - try { - $this->definedS3s = currentTeam()->s3s; - if ($this->definedS3s->count() > 0) { - $this->s3StorageId = $this->definedS3s->first()->id; - } - } catch (\Throwable $e) { - return handleError($e, $this); - } - } - public function submit() { try { @@ -56,20 +34,6 @@ public function submit() $this->validate(); - if ($this->saveToS3) { - $s3StorageExists = ! is_null($this->s3StorageId) - && S3Storage::where('team_id', currentTeam()->id) - ->where('is_usable', true) - ->whereKey($this->s3StorageId) - ->exists(); - - if (! $s3StorageExists) { - $this->dispatch('error', 'Please select a valid S3 storage to enable S3 backups.'); - - return; - } - } - $isValid = validate_cron_expression($this->frequency); if (! $isValid) { $this->dispatch('error', 'Invalid Cron / Human expression.'); @@ -80,8 +44,8 @@ public function submit() $payload = [ 'enabled' => true, 'frequency' => $this->frequency, - 'save_s3' => $this->saveToS3, - 's3_storage_id' => $this->s3StorageId, + 'save_s3' => false, + 's3_storage_id' => null, 'database_id' => $this->database->id, 'database_type' => $this->database->getMorphClass(), 'team_id' => currentTeam()->id, @@ -99,11 +63,22 @@ public function submit() $databaseBackup = ScheduledDatabaseBackup::create($payload); if ($this->database->getMorphClass() === ServiceDatabase::class) { - $this->dispatch('refreshScheduledBackups', $databaseBackup->id); + $service = $this->database->service; + $this->redirectRoute('project.service.database.backup.show', [ + 'project_uuid' => $service->project()->uuid, + 'environment_uuid' => $service->environment->uuid, + 'service_uuid' => $service->uuid, + 'stack_service_uuid' => $this->database->uuid, + 'backup_uuid' => $databaseBackup->uuid, + ], navigate: true); } else { - $this->dispatch('refreshScheduledBackups'); + $this->redirectRoute('project.database.backup.execution', [ + 'project_uuid' => $this->database->project()->uuid, + 'environment_uuid' => $this->database->environment->uuid, + 'database_uuid' => $this->database->uuid, + 'backup_uuid' => $databaseBackup->uuid, + ], navigate: true); } - } catch (\Throwable $e) { return handleError($e, $this); } finally { diff --git a/app/Livewire/Project/Database/ScheduledBackups.php b/app/Livewire/Project/Database/ScheduledBackups.php index 9d1f212e4..aa7e9f43c 100644 --- a/app/Livewire/Project/Database/ScheduledBackups.php +++ b/app/Livewire/Project/Database/ScheduledBackups.php @@ -2,8 +2,8 @@ namespace App\Livewire\Project\Database; -use App\Models\ScheduledDatabaseBackup; use App\Models\ServiceDatabase; +use Illuminate\Contracts\View\View; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; @@ -17,42 +17,18 @@ class ScheduledBackups extends Component public $type; - public ?ScheduledDatabaseBackup $selectedBackup; - - public $selectedBackupId; - - public $s3s; - public string $custom_type = 'mysql'; protected $listeners = ['refreshScheduledBackups']; - protected $queryString = ['selectedBackupId']; - public function mount(): void { - if ($this->selectedBackupId) { - $this->setSelectedBackup($this->selectedBackupId, true); - } $this->parameters = get_route_parameters(); if ($this->database->getMorphClass() === ServiceDatabase::class) { $this->type = 'service-database'; } else { $this->type = 'database'; } - $this->s3s = currentTeam()->s3s; - } - - public function setSelectedBackup($backupId, $force = false) - { - if ($this->selectedBackupId === $backupId && ! $force) { - return; - } - $this->selectedBackupId = $backupId; - $this->selectedBackup = $this->database->scheduledBackups->find($backupId); - if (is_null($this->selectedBackup)) { - $this->selectedBackupId = null; - } } public function setCustomType() @@ -86,9 +62,13 @@ public function delete($scheduled_backup_id): void public function refreshScheduledBackups(?int $id = null): void { $this->database->refresh(); - if ($id) { - $this->setSelectedBackup($id); - } $this->dispatch('refreshScheduledBackups'); } + + public function render(): View + { + $this->database->loadMissing('scheduledBackups.s3'); + + return view('livewire.project.database.scheduled-backups'); + } } diff --git a/app/Livewire/Project/Service/DatabaseBackups.php b/app/Livewire/Project/Service/DatabaseBackups.php index 883441ecb..90907abc6 100644 --- a/app/Livewire/Project/Service/DatabaseBackups.php +++ b/app/Livewire/Project/Service/DatabaseBackups.php @@ -2,6 +2,7 @@ namespace App\Livewire\Project\Service; +use App\Models\ScheduledDatabaseBackup; use App\Models\Service; use App\Models\ServiceDatabase; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; @@ -17,16 +18,28 @@ class DatabaseBackups extends Component public array $parameters; + public array $backupParameters = []; + public array $query; public bool $isImportSupported = false; + public ?ScheduledDatabaseBackup $backup = null; + + public string $section = 'index'; + + public $s3s; + protected $listeners = ['refreshScheduledBackups' => '$refresh']; public function mount() { try { - $this->parameters = get_route_parameters(); + $this->parameters = array_filter( + get_route_parameters(), + fn (string $key): bool => $key !== 'backup_uuid', + ARRAY_FILTER_USE_KEY, + ); $this->query = request()->query(); $project = currentTeam() ->projects() @@ -58,6 +71,21 @@ public function mount() $dbType = $this->serviceDatabase->databaseType(); $supportedTypes = ['mysql', 'mariadb', 'postgres', 'mongo']; $this->isImportSupported = collect($supportedTypes)->contains(fn ($type) => str_contains($dbType, $type)); + + if (request()->route('backup_uuid')) { + $this->backup = $this->serviceDatabase->scheduledBackups() + ->where('uuid', request()->route('backup_uuid')) + ->firstOrFail(); + $this->s3s = currentTeam()->s3s; + $this->backupParameters = [...$this->parameters, 'backup_uuid' => $this->backup->uuid]; + $this->section = match (request()->route()?->getName()) { + 'project.service.database.backup.s3' => 's3', + 'project.service.database.backup.retention' => 'retention', + 'project.service.database.backup.executions' => 'executions', + 'project.service.database.backup.danger' => 'danger', + default => 'general', + }; + } } catch (\Throwable $e) { return handleError($e, $this); } diff --git a/app/Livewire/Project/Service/FileStorage.php b/app/Livewire/Project/Service/FileStorage.php index e869ca91b..84a0daec8 100644 --- a/app/Livewire/Project/Service/FileStorage.php +++ b/app/Livewire/Project/Service/FileStorage.php @@ -4,6 +4,7 @@ use App\Models\Application; use App\Models\LocalFileVolume; +use App\Models\ScheduledVolumeBackup; use App\Models\ServiceApplication; use App\Models\ServiceDatabase; use App\Models\StandaloneClickhouse; @@ -15,6 +16,7 @@ use App\Models\StandalonePostgresql; use App\Models\StandaloneRedis; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; +use Livewire\Attributes\On; use Livewire\Attributes\Validate; use Livewire\Component; @@ -34,6 +36,10 @@ class FileStorage extends Component public bool $isReadOnly = false; + public bool $hasEnabledBackup = false; + + public ?string $backupUrl = null; + #[Validate(['nullable'])] public ?string $content = null; @@ -65,6 +71,53 @@ public function mount() $this->isReadOnly = $this->fileStorage->shouldBeReadOnlyInUI() || $this->fileStorage->is_too_large; $this->syncData(); + $this->refreshBackupStatus(); + } + + #[On('refreshVolumeBackups')] + public function refreshBackupStatus(): void + { + $backup = $this->fileStorage->is_directory + ? $this->fileStorage->scheduledBackups()->first() + : null; + + $this->hasEnabledBackup = $backup?->enabled ?? false; + $this->backupUrl = null; + + if (! $this->hasEnabledBackup) { + return; + } + + if ($this->resource instanceof ServiceDatabase) { + $this->backupUrl = route('project.service.database.backups', [ + 'project_uuid' => $this->resource->service->project()->uuid, + 'environment_uuid' => $this->resource->service->environment->uuid, + 'service_uuid' => $this->resource->service->uuid, + 'stack_service_uuid' => $this->resource->uuid, + ]); + + return; + } + + if (! $this->resource instanceof Application) { + $this->hasEnabledBackup = false; + + return; + } + + $parameters = [ + 'project_uuid' => $this->resource->project()->uuid, + 'environment_uuid' => $this->resource->environment->uuid, + 'application_uuid' => $this->resource->uuid, + ]; + $hasOtherBackups = ScheduledVolumeBackup::query() + ->forApplication($this->resource) + ->where('id', '!=', $backup->id) + ->exists(); + + $this->backupUrl = $hasOtherBackups + ? route('project.application.backup.index', [...$parameters, 'search' => $this->fileStorage->fs_path]) + : route('project.application.backup.show', [...$parameters, 'backup_uuid' => $backup->uuid]); } public function syncData(bool $toModel = false): void @@ -135,6 +188,10 @@ public function convertToFile() try { $this->authorize('update', $this->resource); + if ($this->fileStorage->scheduledBackups()->exists()) { + throw new \RuntimeException('Delete this directory backup schedule and its archives before converting it to a file.'); + } + if ($this->fileStorage->is_host_file) { throw new \Exception('Host file mounts are bind-only and cannot be converted.'); } @@ -162,6 +219,12 @@ public function delete($password, $selectedActions = []) return 'The provided password is incorrect.'; } + if ($this->fileStorage->scheduledBackups()->exists()) { + $this->dispatch('error', 'Delete this directory backup schedule and its archives before deleting the directory.'); + + return false; + } + try { $message = 'File deleted.'; if ($this->fileStorage->is_directory) { @@ -174,6 +237,7 @@ public function delete($password, $selectedActions = []) $this->fileStorage->deleteStorageOnServer(); } $this->fileStorage->delete(); + $this->dispatch('configurationChanged'); $this->dispatch('success', $message); } catch (\Throwable $e) { return handleError($e, $this); diff --git a/app/Livewire/Project/Service/Storage.php b/app/Livewire/Project/Service/Storage.php index 9b097f2e1..05be7eb8a 100644 --- a/app/Livewire/Project/Service/Storage.php +++ b/app/Livewire/Project/Service/Storage.php @@ -131,10 +131,12 @@ public function submitPersistentVolume() 'resource_type' => $this->resource->getMorphClass(), ]); $this->resource->refresh(); + $this->dispatch('configurationChanged'); $this->dispatch('success', 'Volume added successfully'); $this->dispatch('closeStorageModal', 'volume'); $this->clearForm(); $this->refreshStorages(); + $this->dispatch('refreshStorages'); } catch (\Throwable $e) { return handleError($e, $this); } @@ -163,6 +165,7 @@ public function submitFileStorage() 'resource_type' => get_class($this->resource), ]); + $this->dispatch('configurationChanged'); $this->dispatch('success', 'File mount added successfully'); $this->dispatch('closeStorageModal', 'file'); $this->clearForm(); @@ -195,6 +198,7 @@ public function submitHostFileStorage() 'resource_type' => get_class($this->resource), ]); + $this->dispatch('configurationChanged'); $this->dispatch('success', 'Host file mount added successfully'); $this->dispatch('closeStorageModal', 'host-file'); $this->clearForm(); @@ -231,6 +235,7 @@ public function submitFileStorageDirectory() 'resource_type' => get_class($this->resource), ]); + $this->dispatch('configurationChanged'); $this->dispatch('success', 'Directory mount added successfully'); $this->dispatch('closeStorageModal', 'directory'); $this->clearForm(); diff --git a/app/Livewire/Project/Shared/Storages/Show.php b/app/Livewire/Project/Shared/Storages/Show.php index 2aaca5e6f..242660eec 100644 --- a/app/Livewire/Project/Shared/Storages/Show.php +++ b/app/Livewire/Project/Shared/Storages/Show.php @@ -2,9 +2,12 @@ namespace App\Livewire\Project\Shared\Storages; +use App\Models\Application; use App\Models\LocalPersistentVolume; +use App\Models\ScheduledVolumeBackup; use App\Support\ValidationPatterns; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; +use Livewire\Attributes\On; use Livewire\Component; class Show extends Component @@ -32,6 +35,10 @@ class Show extends Component public bool $isPreviewSuffixEnabled = true; + public bool $hasEnabledBackup = false; + + public ?string $backupUrl = null; + protected $validationAttributes = [ 'name' => 'name', 'mountPath' => 'mount', @@ -81,10 +88,38 @@ private function syncData(bool $toModel = false): void } } - public function mount() + public function mount(): void { $this->syncData(false); $this->isReadOnly = $this->storage->shouldBeReadOnlyInUI(); + $this->refreshBackupStatus(); + } + + #[On('refreshVolumeBackups')] + public function refreshBackupStatus(): void + { + $backup = $this->storage->scheduledBackups()->first(); + + $this->hasEnabledBackup = $backup?->enabled ?? false; + $this->backupUrl = null; + + if (! $this->hasEnabledBackup || ! $this->resource instanceof Application) { + return; + } + + $parameters = [ + 'project_uuid' => $this->resource->project()->uuid, + 'environment_uuid' => $this->resource->environment->uuid, + 'application_uuid' => $this->resource->uuid, + ]; + $hasOtherBackups = ScheduledVolumeBackup::query() + ->forApplication($this->resource) + ->where('id', '!=', $backup->id) + ->exists(); + + $this->backupUrl = $hasOtherBackups + ? route('project.application.backup.index', [...$parameters, 'search' => $this->storage->name]) + : route('project.application.backup.show', [...$parameters, 'backup_uuid' => $backup->uuid]); } public function instantSave(): void @@ -115,8 +150,15 @@ public function delete($password, $selectedActions = []) return 'The provided password is incorrect.'; } + if ($this->storage->scheduledBackups()->exists()) { + $this->dispatch('error', 'Delete this volume backup schedule and its archives before deleting the volume.'); + + return false; + } + $this->storage->delete(); $this->dispatch('refreshStorages'); + $this->dispatch('configurationChanged'); return true; } diff --git a/app/Livewire/Project/Shared/Storages/VolumeBackups.php b/app/Livewire/Project/Shared/Storages/VolumeBackups.php new file mode 100644 index 000000000..3e56c4086 --- /dev/null +++ b/app/Livewire/Project/Shared/Storages/VolumeBackups.php @@ -0,0 +1,388 @@ + ['required', 'string'], + 'enabled' => ['required', 'boolean'], + 'saveToS3' => ['required', 'boolean'], + 'disableLocalBackup' => ['required', 'boolean'], + 'stopDuringBackup' => ['required', 'boolean'], + 's3StorageId' => ['nullable', 'integer'], + 'retentionAmountLocally' => ['required', 'integer', 'min:0', 'max:10000'], + 'retentionDaysLocally' => ['required', 'integer', 'min:0'], + 'retentionMaxStorageLocally' => ['required', 'numeric', 'min:0'], + 'retentionAmountS3' => ['required', 'integer', 'min:0', 'max:10000'], + 'retentionDaysS3' => ['required', 'integer', 'min:0'], + 'retentionMaxStorageS3' => ['required', 'numeric', 'min:0'], + 'timeout' => ['required', 'integer', 'min:60', 'max:36000'], + ]; + } + + public function mount(): void + { + $this->authorize('view', $this->resource); + $this->availableS3Storages = S3Storage::ownedByCurrentTeam() + ->where('is_usable', true) + ->get(); + $this->backup = $this->storage->scheduledBackups()->first(); + $server = $this->backup?->server() ?? data_get($this->resource, 'destination.server'); + $this->timezone = data_get($server, 'settings.server_timezone', 'Instance timezone'); + + if ($this->backup) { + $this->frequency = $this->backup->frequency; + $this->enabled = $this->backup->enabled; + $this->saveToS3 = $this->backup->save_s3; + $this->disableLocalBackup = $this->backup->disable_local_backup; + $this->stopDuringBackup = $this->backup->stop_during_backup; + $this->s3StorageId = $this->backup->s3_storage_id ?? $this->availableS3Storages->first()?->id; + $this->retentionAmountLocally = $this->backup->retention_amount_locally; + $this->retentionDaysLocally = $this->backup->retention_days_locally; + $this->retentionMaxStorageLocally = $this->backup->retention_max_storage_locally; + $this->retentionAmountS3 = $this->backup->retention_amount_s3; + $this->retentionDaysS3 = $this->backup->retention_days_s3; + $this->retentionMaxStorageS3 = $this->backup->retention_max_storage_s3; + $this->timeout = $this->backup->timeout; + } else { + $this->s3StorageId = $this->availableS3Storages->first()?->id; + } + } + + public function save(): void + { + $this->authorize('update', $this->resource); + + if (! $this->validateSettings()) { + return; + } + + $this->backup = $this->persistBackup($this->enabled); + $this->dispatch('success', 'Storage backup schedule saved.'); + } + + public function instantSave(): void + { + $this->save(); + } + + public function updatedS3StorageId(): void + { + $this->authorize('update', $this->resource); + + if (! $this->hasValidS3Storage()) { + $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.'); + + return; + } + + $this->resetErrorBag('s3StorageId'); + $this->backup?->update(['s3_storage_id' => $this->s3StorageId]); + $this->dispatch('success', 'S3 storage updated.'); + } + + public function toggleS3(): void + { + $this->authorize('update', $this->resource); + + if (! $this->saveToS3 && ! $this->hasValidS3Storage()) { + $this->dispatch('error', 'Select a usable S3 storage before enabling S3 backups.'); + + return; + } + + $this->saveToS3 = ! $this->saveToS3; + $this->disableLocalBackup = $this->saveToS3 && $this->disableLocalBackup; + $this->backup?->update([ + 'save_s3' => $this->saveToS3, + 'disable_local_backup' => $this->disableLocalBackup, + 's3_storage_id' => $this->s3StorageId, + ]); + $this->dispatch('success', $this->saveToS3 ? 'S3 backups enabled.' : 'S3 backups disabled.'); + } + + public function toggleEnabled(): void + { + $this->authorize('update', $this->resource); + + if (! $this->backup) { + if (! $this->validateSettings()) { + return; + } + + $this->enabled = true; + $this->backup = $this->persistBackup(true); + } else { + $this->enabled = ! $this->enabled; + $this->backup->update(['enabled' => $this->enabled]); + } + + $this->dispatch('success', $this->enabled ? 'Storage backups enabled.' : 'Storage backups disabled.'); + } + + public function backupNow(): Redirector|RedirectResponse|null + { + $this->authorize('update', $this->resource); + + if (! $this->backup) { + if (! $this->validateSettings()) { + return null; + } + + $this->enabled = false; + $this->backup = $this->persistBackup(false); + } + + VolumeBackupJob::dispatch($this->backup); + $this->dispatch('success', 'Storage backup queued.'); + + return redirect()->route('project.application.backup.executions', [ + 'project_uuid' => $this->resource->project()->uuid, + 'environment_uuid' => $this->resource->environment->uuid, + 'application_uuid' => $this->resource->uuid, + 'backup_uuid' => $this->backup->uuid, + ]); + } + + public function delete(?string $password = null, array $selectedActions = []): bool|string + { + $this->authorize('update', $this->resource); + + if (! verifyPasswordConfirmation($password, $this)) { + return 'The provided password is incorrect.'; + } + + if (! $this->backup) { + return false; + } + + try { + DeleteScheduledVolumeBackup::run($this->backup); + $this->backup = null; + $this->dispatch('success', 'Storage backup schedule and archives deleted.'); + $this->redirectRoute('project.application.backup.index', [ + 'project_uuid' => $this->resource->project()->uuid, + 'environment_uuid' => $this->resource->environment->uuid, + 'application_uuid' => $this->resource->uuid, + ], navigate: true); + + return true; + } catch (Throwable $exception) { + $this->dispatch('error', 'Could not delete the backup archives: '.$exception->getMessage()); + + return false; + } + } + + public function cleanupFailed(): void + { + $this->authorize('update', $this->resource); + + $deletedCount = $this->backup?->executions() + ->where('status', 'failed') + ->where('stop_recovery_pending', false) + ->where('s3_cleanup_pending', false) + ->where(fn ($query) => $query + ->whereNull('filename') + ->orWhere('local_storage_deleted', true)) + ->delete() ?? 0; + + $this->dispatch( + $deletedCount > 0 ? 'success' : 'info', + $deletedCount > 0 ? 'Failed backup entries cleaned up.' : 'No safely removable failed backup entries found.', + ); + } + + public function cleanupDeleted(): void + { + $this->authorize('update', $this->resource); + + $deletedCount = $this->backup?->executions() + ->where('local_storage_deleted', true) + ->where(fn ($query) => $query + ->where('s3_storage_deleted', true) + ->orWhereNull('s3_uploaded') + ->orWhere('s3_uploaded', false)) + ->delete() ?? 0; + + $this->dispatch( + $deletedCount > 0 ? 'success' : 'info', + $deletedCount > 0 ? "Cleaned up {$deletedCount} deleted backup entries." : 'No deleted backup entries found.', + ); + } + + public function deleteBackup(int $executionId, string $password, array $selectedActions = []): bool|string + { + $this->authorize('update', $this->resource); + + if (! verifyPasswordConfirmation($password, $this)) { + return 'The provided password is incorrect.'; + } + + $execution = $this->backup?->executions()->whereKey($executionId)->first(); + if (! $execution) { + $this->dispatch('error', 'Backup execution not found.'); + + return false; + } + + if ($execution->status === 'running' || $execution->stop_recovery_pending || $execution->s3_cleanup_pending) { + $this->dispatch('error', 'Wait for the backup and recovery operations to finish before deleting it.'); + + return false; + } + + try { + $server = $this->backup->server(); + if (! $execution->local_storage_deleted && filled($execution->filename)) { + if (! $server) { + throw new \RuntimeException('The server is unavailable.'); + } + + deleteBackupsLocally($execution->filename, $server, throwError: true); + } + + if ($this->delete_backup_s3 && $execution->s3_uploaded && ! $execution->s3_storage_deleted) { + if (! $execution->s3) { + throw new \RuntimeException('The S3 storage is unavailable.'); + } + + deleteBackupsS3($execution->filename, $execution->s3); + } + + $execution->delete(); + $this->delete_backup_s3 = false; + $this->dispatch('success', 'Backup deleted.'); + + return true; + } catch (Throwable $exception) { + $this->dispatch('error', 'Failed to delete backup: '.$exception->getMessage()); + + return false; + } + } + + public function render() + { + $executions = $this->backup?->executions()->paginate(10); + + return view('livewire.project.shared.storages.volume-backups', [ + 'executions' => $executions ?? collect(), + 'latestExecution' => $this->backup?->executions()->first(), + ]); + } + + private function validateSettings(): bool + { + $this->validate(); + + if (! validate_cron_expression($this->frequency)) { + $this->addError('frequency', 'The frequency must be a valid cron or human expression.'); + + return false; + } + + if ($this->saveToS3 && ! $this->hasValidS3Storage()) { + $this->addError('s3StorageId', 'Select a usable S3 storage owned by your team.'); + + return false; + } + + $this->disableLocalBackup = $this->saveToS3 && $this->disableLocalBackup; + + return true; + } + + private function persistBackup(bool $enabled): ScheduledVolumeBackup + { + return $this->storage->scheduledBackups()->updateOrCreate( + [], + [ + 'team_id' => currentTeam()->id, + 'frequency' => $this->frequency, + 'enabled' => $enabled, + 'save_s3' => $this->saveToS3, + 'disable_local_backup' => $this->disableLocalBackup, + 'stop_during_backup' => $this->stopDuringBackup, + 's3_storage_id' => $this->hasValidS3Storage() ? $this->s3StorageId : $this->backup?->s3_storage_id, + 'retention_amount_locally' => $this->retentionAmountLocally, + 'retention_days_locally' => $this->retentionDaysLocally, + 'retention_max_storage_locally' => $this->retentionMaxStorageLocally, + 'retention_amount_s3' => $this->retentionAmountS3, + 'retention_days_s3' => $this->retentionDaysS3, + 'retention_max_storage_s3' => $this->retentionMaxStorageS3, + 'timeout' => $this->timeout, + ], + ); + } + + private function hasValidS3Storage(): bool + { + return $this->s3StorageId !== null + && S3Storage::query() + ->whereKey($this->s3StorageId) + ->where('team_id', currentTeam()->id) + ->where('is_usable', true) + ->exists(); + } +} diff --git a/app/Livewire/Storage/Show.php b/app/Livewire/Storage/Show.php index dd6640c23..914366d4f 100644 --- a/app/Livewire/Storage/Show.php +++ b/app/Livewire/Storage/Show.php @@ -4,6 +4,7 @@ use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; +use App\Models\ScheduledVolumeBackup; use Illuminate\Auth\Access\AuthorizationException; use Illuminate\Foundation\Auth\Access\AuthorizesRequests; use Livewire\Component; @@ -30,7 +31,8 @@ public function mount() return $this->redirectRoute('storage.index', navigate: true); } $this->currentRoute = request()->route()->getName(); - $this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count(); + $this->backupCount = ScheduledDatabaseBackup::where('s3_storage_id', $this->storage->id)->count() + + ScheduledVolumeBackup::where('s3_storage_id', $this->storage->id)->count(); } public function delete() diff --git a/app/Models/LocalFileVolume.php b/app/Models/LocalFileVolume.php index 968e6c3d0..86873d1a1 100644 --- a/app/Models/LocalFileVolume.php +++ b/app/Models/LocalFileVolume.php @@ -6,6 +6,8 @@ use App\Jobs\ServerStorageSaveJob; use Illuminate\Database\Eloquent\Casts\Attribute; use Illuminate\Database\Eloquent\Factories\HasFactory; +use Illuminate\Database\Eloquent\Relations\MorphMany; +use Illuminate\Database\Eloquent\Relations\MorphTo; use Symfony\Component\Yaml\Yaml; class LocalFileVolume extends BaseModel @@ -57,6 +59,12 @@ protected static function booted() $fileVolume->load(['service']); dispatch(new ServerStorageSaveJob($fileVolume)); }); + + static::deleting(function (LocalFileVolume $fileVolume): void { + if ($fileVolume->scheduledBackups()->exists()) { + throw new \RuntimeException('Delete this directory backup schedule and its archives before deleting the directory.'); + } + }); } protected function isBinary(): Attribute @@ -73,11 +81,28 @@ protected function isTooLarge(): Attribute ); } - public function service() + public function resource(): MorphTo + { + return $this->morphTo(); + } + + public function service(): MorphTo { return $this->morphTo('resource'); } + public function scheduledBackups(): MorphMany + { + return $this->morphMany(ScheduledVolumeBackup::class, 'backupable'); + } + + public function abortIfScheduledBackupsExist(): void + { + if ($this->scheduledBackups()->exists()) { + abort(422, 'Delete this directory backup schedule and its archives before deleting the directory.'); + } + } + public function loadStorageOnServer() { if ($this->is_host_file) { diff --git a/app/Models/LocalPersistentVolume.php b/app/Models/LocalPersistentVolume.php index d44c86c0c..6b4e0fe5b 100644 --- a/app/Models/LocalPersistentVolume.php +++ b/app/Models/LocalPersistentVolume.php @@ -3,10 +3,20 @@ namespace App\Models; use Illuminate\Database\Eloquent\Casts\Attribute; +use Illuminate\Database\Eloquent\Relations\MorphMany; use Symfony\Component\Yaml\Yaml; class LocalPersistentVolume extends BaseModel { + protected static function booted(): void + { + static::deleting(function (LocalPersistentVolume $volume): void { + if ($volume->scheduledBackups()->exists()) { + throw new \RuntimeException('Delete this volume backup schedule and its archives before deleting the volume.'); + } + }); + } + protected $fillable = [ 'name', 'mount_path', @@ -41,6 +51,18 @@ public function database() return $this->morphTo('resource'); } + public function scheduledBackups(): MorphMany + { + return $this->morphMany(ScheduledVolumeBackup::class, 'backupable'); + } + + public function abortIfScheduledBackupsExist(): void + { + if ($this->scheduledBackups()->exists()) { + abort(422, 'Delete this volume backup schedule and its archives before deleting the volume.'); + } + } + protected function customizeName($value) { return str($value)->trim()->value; diff --git a/app/Models/S3Storage.php b/app/Models/S3Storage.php index 70703cd52..82a358344 100644 --- a/app/Models/S3Storage.php +++ b/app/Models/S3Storage.php @@ -69,6 +69,15 @@ protected static function boot(): void 'save_s3' => false, 's3_storage_id' => null, ]); + ScheduledVolumeBackupExecution::where('s3_storage_id', $storage->id) + ->update([ + 's3_storage_deleted' => true, + 's3_cleanup_pending' => false, + ]); + ScheduledVolumeBackup::where('s3_storage_id', $storage->id)->update([ + 'save_s3' => false, + 's3_storage_id' => null, + ]); }); } @@ -101,6 +110,11 @@ public function scheduledBackups() return $this->hasMany(ScheduledDatabaseBackup::class, 's3_storage_id'); } + public function scheduledVolumeBackups() + { + return $this->hasMany(ScheduledVolumeBackup::class, 's3_storage_id'); + } + public function awsUrl() { return "{$this->endpoint}/{$this->bucket}"; diff --git a/app/Models/ScheduledVolumeBackup.php b/app/Models/ScheduledVolumeBackup.php new file mode 100644 index 000000000..6cdc651fb --- /dev/null +++ b/app/Models/ScheduledVolumeBackup.php @@ -0,0 +1,149 @@ + 'boolean', + 'save_s3' => 'boolean', + 'disable_local_backup' => 'boolean', + 'stop_during_backup' => 'boolean', + 'retention_amount_locally' => 'integer', + 'retention_days_locally' => 'integer', + 'retention_max_storage_locally' => 'float', + 'retention_amount_s3' => 'integer', + 'retention_days_s3' => 'integer', + 'retention_max_storage_s3' => 'float', + 'timeout' => 'integer', + ]; + } + + public function scopeForApplication(Builder $query, Application $application): Builder + { + $volumeIds = $application->persistentStorages()->pluck('id'); + $directoryIds = $application->fileStorages() + ->where('is_directory', true) + ->where('is_host_file', false) + ->pluck('id'); + + return $query->where(function (Builder $query) use ($volumeIds, $directoryIds): void { + $query->where(function (Builder $query) use ($volumeIds): void { + $query->where('backupable_type', (new LocalPersistentVolume)->getMorphClass()) + ->whereIn('backupable_id', $volumeIds); + })->orWhere(function (Builder $query) use ($directoryIds): void { + $query->where('backupable_type', (new LocalFileVolume)->getMorphClass()) + ->whereIn('backupable_id', $directoryIds); + }); + }); + } + + public function backupable(): MorphTo + { + return $this->morphTo(); + } + + public function team(): BelongsTo + { + return $this->belongsTo(Team::class); + } + + public function s3(): BelongsTo + { + return $this->belongsTo(S3Storage::class, 's3_storage_id'); + } + + public function executions(): HasMany + { + return $this->hasMany(ScheduledVolumeBackupExecution::class)->latest(); + } + + public function latestExecution(): HasOne + { + return $this->hasOne(ScheduledVolumeBackupExecution::class)->latestOfMany(); + } + + public function server(): ?Server + { + $resource = $this->targetResource(); + + if ($resource instanceof ServiceApplication || $resource instanceof ServiceDatabase) { + return $resource->service?->server?->fresh(); + } + + return $resource?->destination?->server?->fresh(); + } + + public function targetResource(): ?Model + { + return $this->backupable?->resource; + } + + public function targetType(): string + { + return $this->backupable instanceof LocalFileVolume ? 'Directory' : 'Volume'; + } + + public function targetName(): string + { + return match (true) { + $this->backupable instanceof LocalFileVolume => $this->backupable->fs_path, + $this->backupable instanceof LocalPersistentVolume => $this->backupable->name, + default => 'Unknown storage', + }; + } + + public function sourcePath(): string + { + $target = $this->backupable; + + if ($target instanceof LocalPersistentVolume) { + return filled($target->host_path) ? $target->host_path : $target->name; + } + + if (! $target instanceof LocalFileVolume || ! $target->is_directory) { + throw new \RuntimeException('The backup target is not a directory or persistent volume.'); + } + + $path = str($target->fs_path); + if ($path->startsWith('.')) { + $resource = $this->targetResource(); + if (! $resource || ! method_exists($resource, 'workdir')) { + throw new \RuntimeException('The directory backup workdir is unavailable.'); + } + + return $resource->workdir().$path->after('.')->toString(); + } + + return $path->toString(); + } +} diff --git a/app/Models/ScheduledVolumeBackupExecution.php b/app/Models/ScheduledVolumeBackupExecution.php new file mode 100644 index 000000000..528734446 --- /dev/null +++ b/app/Models/ScheduledVolumeBackupExecution.php @@ -0,0 +1,49 @@ + 'integer', + 'finished_at' => 'datetime', + 'stop_container_ids' => 'array', + 'stop_recovery_pending' => 'boolean', + 's3_cleanup_pending' => 'boolean', + 'local_storage_deleted' => 'boolean', + 's3_storage_deleted' => 'boolean', + 's3_uploaded' => 'boolean', + ]; + } + + public function scheduledVolumeBackup(): BelongsTo + { + return $this->belongsTo(ScheduledVolumeBackup::class); + } + + public function s3(): BelongsTo + { + return $this->belongsTo(S3Storage::class, 's3_storage_id'); + } +} diff --git a/app/Services/DeploymentConfiguration/ApplicationConfigurationSnapshot.php b/app/Services/DeploymentConfiguration/ApplicationConfigurationSnapshot.php index aeb40364a..d7ba8ebd1 100644 --- a/app/Services/DeploymentConfiguration/ApplicationConfigurationSnapshot.php +++ b/app/Services/DeploymentConfiguration/ApplicationConfigurationSnapshot.php @@ -4,6 +4,8 @@ use App\Models\Application; use App\Models\EnvironmentVariable; +use App\Models\LocalFileVolume; +use App\Models\LocalPersistentVolume; use App\Services\DeploymentConfiguration\Concerns\SummarizesDiffText; use Illuminate\Support\Arr; @@ -47,6 +49,10 @@ public function toArray(): array 'label' => 'Environment Variables', 'items' => $this->environmentItems(), ], + 'storage' => [ + 'label' => 'Storage', + 'items' => $this->storageItems(), + ], ], ]; } @@ -214,6 +220,40 @@ private function environmentItems(): array ->all(); } + /** + * @return array> + */ + private function storageItems(): array + { + $volumes = $this->application->persistentStorages() + ->orderBy('id') + ->get(['id', 'name', 'mount_path', 'host_path']) + ->map(function (LocalPersistentVolume $volume): array { + $source = $volume->host_path ?: $volume->name; + + return $this->item( + key: 'volume_'.$volume->id, + label: 'Volume mount', + value: ['source' => $source, 'destination' => $volume->mount_path], + impact: 'redeploy', + displayValue: "{$source} → {$volume->mount_path}", + ); + }); + + $fileMounts = $this->application->fileStorages() + ->orderBy('id') + ->get(['id', 'fs_path', 'mount_path', 'is_directory']) + ->map(fn (LocalFileVolume $file): array => $this->item( + key: 'file_'.$file->id, + label: $file->is_directory ? 'Directory mount' : 'File mount', + value: ['source' => $file->fs_path, 'destination' => $file->mount_path], + impact: 'redeploy', + displayValue: "{$file->fs_path} → {$file->mount_path}", + )); + + return collect($volumes->all())->merge($fileMounts->all())->values()->all(); + } + /** * @return array> */ diff --git a/app/View/Components/Forms/Checkbox.php b/app/View/Components/Forms/Checkbox.php index e33e4b919..56bccb06f 100644 --- a/app/View/Components/Forms/Checkbox.php +++ b/app/View/Components/Forms/Checkbox.php @@ -25,6 +25,7 @@ public function __construct( public ?string $helper = null, public string|bool|null $checked = false, public string|bool $instantSave = false, + public bool $live = false, public bool $disabled = false, public string $defaultClass = 'dark:border-neutral-700 text-coolgray-400 dark:bg-coolgray-100 rounded-sm cursor-pointer dark:disabled:bg-base dark:disabled:cursor-not-allowed focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-coollabs dark:focus-visible:ring-warning focus-visible:ring-offset-2 dark:focus-visible:ring-offset-base', public ?string $canGate = null, diff --git a/bootstrap/helpers/databases.php b/bootstrap/helpers/databases.php index 5f0b2e690..8f4f373b7 100644 --- a/bootstrap/helpers/databases.php +++ b/bootstrap/helpers/databases.php @@ -17,6 +17,10 @@ use Illuminate\Support\Collection; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; +use RuntimeException; +use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException; +use Symfony\Component\HttpFoundation\StreamedResponse; +use Throwable; function create_standalone_postgresql($environmentId, StandaloneDocker|SwarmDocker $destination, ?array $otherData = null, string $databaseImage = 'postgres:16-alpine'): StandalonePostgresql { @@ -179,7 +183,7 @@ function create_standalone_clickhouse($environment_id, StandaloneDocker|SwarmDoc return $database; } -function deleteBackupsLocally(string|array|null $filenames, Server $server): void +function deleteBackupsLocally(string|array|null $filenames, Server $server, bool $throwError = false): void { if (empty($filenames)) { return; @@ -187,13 +191,48 @@ function deleteBackupsLocally(string|array|null $filenames, Server $server): voi if (is_string($filenames)) { $filenames = [$filenames]; } - $quotedFiles = array_map(fn ($file) => "\"$file\"", $filenames); - instant_remote_process(['rm -f '.implode(' ', $quotedFiles)], $server, throwError: false); + $quotedFiles = array_map(fn ($file) => escapeshellarg($file), $filenames); + instant_remote_process(['rm -f '.implode(' ', $quotedFiles)], $server, throwError: $throwError); $foldersToCheck = collect($filenames)->map(fn ($file) => dirname($file))->unique(); $foldersToCheck->each(fn ($folder) => deleteEmptyBackupFolder($folder, $server)); } +function streamBackupFromServer(Server $server, string $filename, string $contentType): StreamedResponse +{ + $disk = Storage::build([ + 'driver' => 'sftp', + 'host' => $server->ip, + 'port' => (int) $server->port, + 'username' => $server->user, + 'privateKey' => $server->privateKey->getKeyLocation(), + 'root' => '/', + ]); + + if (! $disk->exists($filename)) { + throw new FileNotFoundException($filename); + } + + return new StreamedResponse(function () use ($disk, $filename) { + if (ob_get_level()) { + ob_end_clean(); + } + $stream = $disk->readStream($filename); + if ($stream === false || is_null($stream)) { + abort(500, 'Failed to open stream for the requested file.'); + } + while (! feof($stream)) { + echo fread($stream, 2048); + flush(); + } + + fclose($stream); + }, 200, [ + 'Content-Type' => $contentType, + 'Content-Disposition' => 'attachment; filename="'.basename($filename).'"', + ]); +} + function deleteBackupsS3(string|array|null $filenames, S3Storage $s3): void { if (empty($filenames) || ! $s3) { @@ -214,7 +253,9 @@ function deleteBackupsS3(string|array|null $filenames, S3Storage $s3): void 'aws_url' => $s3->awsUrl(), ]); - $disk->delete($filenames); + if (! $disk->delete($filenames)) { + throw new RuntimeException('One or more S3 backup files could not be deleted.'); + } } function deleteEmptyBackupFolder($folderPath, Server $server): void @@ -428,8 +469,12 @@ function deleteOldBackupsFromS3($backup): Collection ->all(); if (! empty($filesToDelete)) { - deleteBackupsS3($filesToDelete, $backup->s3); - $processedBackups = $backupsToDelete; + try { + deleteBackupsS3($filesToDelete, $backup->s3); + $processedBackups = $backupsToDelete; + } catch (Throwable $e) { + report($e); + } } return $processedBackups; diff --git a/database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php b/database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php new file mode 100644 index 000000000..4ef178cb7 --- /dev/null +++ b/database/migrations/2026_07_15_102537_create_scheduled_volume_backups_table.php @@ -0,0 +1,46 @@ +id(); + $table->string('uuid')->unique(); + $table->string('backupable_type'); + $table->unsignedBigInteger('backupable_id'); + $table->foreignId('team_id')->constrained()->cascadeOnDelete(); + $table->foreignId('s3_storage_id')->nullable()->constrained()->nullOnDelete(); + $table->string('frequency'); + $table->boolean('enabled')->default(true); + $table->boolean('save_s3')->default(false); + $table->boolean('disable_local_backup')->default(false); + $table->boolean('stop_during_backup')->default(false); + $table->unsignedInteger('retention_amount_locally')->default(7); + $table->unsignedInteger('retention_days_locally')->default(0); + $table->decimal('retention_max_storage_locally', 17, 7)->default(0); + $table->unsignedInteger('retention_amount_s3')->default(7); + $table->unsignedInteger('retention_days_s3')->default(0); + $table->decimal('retention_max_storage_s3', 17, 7)->default(0); + $table->unsignedInteger('timeout')->default(3600); + $table->timestamps(); + + $table->unique(['backupable_type', 'backupable_id']); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('scheduled_volume_backups'); + } +}; diff --git a/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php new file mode 100644 index 000000000..236c937dd --- /dev/null +++ b/database/migrations/2026_07_15_102538_create_scheduled_volume_backup_executions_table.php @@ -0,0 +1,43 @@ +id(); + $table->string('uuid')->unique(); + $table->foreignId('scheduled_volume_backup_id')->constrained()->cascadeOnDelete(); + $table->foreignId('s3_storage_id')->nullable()->constrained()->nullOnDelete(); + $table->enum('status', ['success', 'failed', 'running'])->default('running'); + $table->longText('message')->nullable(); + $table->unsignedBigInteger('size')->default(0); + $table->text('filename')->nullable(); + $table->json('stop_container_ids')->nullable(); + $table->boolean('stop_recovery_pending')->default(false); + $table->boolean('s3_cleanup_pending')->default(false); + $table->timestamp('finished_at')->nullable(); + $table->boolean('local_storage_deleted')->default(false); + $table->boolean('s3_storage_deleted')->default(false); + $table->boolean('s3_uploaded')->nullable(); + $table->timestamps(); + + $table->index(['scheduled_volume_backup_id', 'created_at'], 'scheduled_volume_executions_backup_created_index'); + }); + } + + /** + * Reverse the migrations. + */ + public function down(): void + { + Schema::dropIfExists('scheduled_volume_backup_executions'); + } +}; diff --git a/openapi.json b/openapi.json index 1d78ff8ba..47060a4b0 100644 --- a/openapi.json +++ b/openapi.json @@ -15536,6 +15536,396 @@ ] } }, + "\/applications\/{uuid}\/storages\/{storage_uuid}\/backups": { + "put": { + "tags": [ + "Applications" + ], + "summary": "Set application storage backup schedule", + "description": "Create or replace the backup schedule for an application persistent volume or directory storage.", + "operationId": "set-application-storage-backup-schedule", + "parameters": [ + { + "name": "uuid", + "in": "path", + "description": "UUID of the application.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "storage_uuid", + "in": "path", + "description": "UUID of the persistent volume or directory storage.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Backup schedule replaced.", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleResponse" + } + } + } + }, + "201": { + "description": "Backup schedule created.", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleResponse" + } + } + } + }, + "400": { + "$ref": "#\/components\/responses\/400" + }, + "401": { + "$ref": "#\/components\/responses\/401" + }, + "403": { + "description": "Forbidden." + }, + "404": { + "$ref": "#\/components\/responses\/404" + }, + "422": { + "$ref": "#\/components\/responses\/422" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "tags": [ + "Applications" + ], + "summary": "Delete application storage backup schedule", + "description": "Delete the backup schedule and its local and S3 archives for an application storage.", + "operationId": "delete-application-storage-backup-schedule", + "parameters": [ + { + "name": "uuid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "storage_uuid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Backup schedule and archives deleted." + }, + "401": { + "$ref": "#\/components\/responses\/401" + }, + "403": { + "description": "Forbidden." + }, + "404": { + "$ref": "#\/components\/responses\/404" + }, + "409": { + "description": "Backup or recovery operation is still running." + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "\/databases\/{uuid}\/storages\/{storage_uuid}\/backups": { + "put": { + "tags": [ + "Databases" + ], + "summary": "Set database storage backup schedule", + "description": "Create or replace the backup schedule for a database persistent volume or directory storage.", + "operationId": "set-database-storage-backup-schedule", + "parameters": [ + { + "name": "uuid", + "in": "path", + "description": "UUID of the database.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "storage_uuid", + "in": "path", + "description": "UUID of the persistent volume or directory storage.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Backup schedule replaced.", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleResponse" + } + } + } + }, + "201": { + "description": "Backup schedule created.", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleResponse" + } + } + } + }, + "400": { + "$ref": "#\/components\/responses\/400" + }, + "401": { + "$ref": "#\/components\/responses\/401" + }, + "403": { + "description": "Forbidden." + }, + "404": { + "$ref": "#\/components\/responses\/404" + }, + "422": { + "$ref": "#\/components\/responses\/422" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "tags": [ + "Databases" + ], + "summary": "Delete database storage backup schedule", + "description": "Delete the backup schedule and its local and S3 archives for a database storage.", + "operationId": "delete-database-storage-backup-schedule", + "parameters": [ + { + "name": "uuid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "storage_uuid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Backup schedule and archives deleted." + }, + "401": { + "$ref": "#\/components\/responses\/401" + }, + "403": { + "description": "Forbidden." + }, + "404": { + "$ref": "#\/components\/responses\/404" + }, + "409": { + "description": "Backup or recovery operation is still running." + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, + "\/services\/{uuid}\/storages\/{storage_uuid}\/backups": { + "put": { + "tags": [ + "Services" + ], + "summary": "Set service storage backup schedule", + "description": "Create or replace the backup schedule for a service persistent volume or directory storage.", + "operationId": "set-service-storage-backup-schedule", + "parameters": [ + { + "name": "uuid", + "in": "path", + "description": "UUID of the service.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "storage_uuid", + "in": "path", + "description": "UUID of the persistent volume or directory storage.", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleRequest" + } + } + } + }, + "responses": { + "200": { + "description": "Backup schedule replaced.", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleResponse" + } + } + } + }, + "201": { + "description": "Backup schedule created.", + "content": { + "application\/json": { + "schema": { + "$ref": "#\/components\/schemas\/VolumeBackupScheduleResponse" + } + } + } + }, + "400": { + "$ref": "#\/components\/responses\/400" + }, + "401": { + "$ref": "#\/components\/responses\/401" + }, + "403": { + "description": "Forbidden." + }, + "404": { + "$ref": "#\/components\/responses\/404" + }, + "422": { + "$ref": "#\/components\/responses\/422" + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + }, + "delete": { + "tags": [ + "Services" + ], + "summary": "Delete service storage backup schedule", + "description": "Delete the backup schedule and its local and S3 archives for a service storage.", + "operationId": "delete-service-storage-backup-schedule", + "parameters": [ + { + "name": "uuid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "storage_uuid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Backup schedule and archives deleted." + }, + "401": { + "$ref": "#\/components\/responses\/401" + }, + "403": { + "description": "Forbidden." + }, + "404": { + "$ref": "#\/components\/responses\/404" + }, + "409": { + "description": "Backup or recovery operation is still running." + } + }, + "security": [ + { + "bearerAuth": [] + } + ] + } + }, "\/vultr\/regions": { "get": { "tags": [ @@ -15672,6 +16062,169 @@ }, "components": { "schemas": { + "VolumeBackupScheduleRequest": { + "required": [ + "frequency" + ], + "properties": { + "frequency": { + "type": "string", + "maxLength": 255, + "example": "0 2 * * *" + }, + "enabled": { + "type": "boolean", + "default": true + }, + "save_s3": { + "type": "boolean", + "default": false + }, + "disable_local_backup": { + "type": "boolean", + "default": false + }, + "stop_during_backup": { + "type": "boolean", + "default": false + }, + "s3_storage_uuid": { + "type": [ + "string", + "null" + ] + }, + "retention_amount_locally": { + "type": "integer", + "default": 7, + "maximum": 10000, + "minimum": 0 + }, + "retention_days_locally": { + "type": "integer", + "default": 0, + "maximum": 2147483647, + "minimum": 0 + }, + "retention_max_storage_locally": { + "type": "number", + "format": "float", + "default": 0, + "maximum": 9999999999, + "minimum": 0 + }, + "retention_amount_s3": { + "type": "integer", + "default": 7, + "maximum": 10000, + "minimum": 0 + }, + "retention_days_s3": { + "type": "integer", + "default": 0, + "maximum": 2147483647, + "minimum": 0 + }, + "retention_max_storage_s3": { + "type": "number", + "format": "float", + "default": 0, + "maximum": 9999999999, + "minimum": 0 + }, + "timeout": { + "type": "integer", + "default": 3600, + "maximum": 36000, + "minimum": 60 + } + }, + "type": "object", + "additionalProperties": false + }, + "VolumeBackupScheduleResponse": { + "required": [ + "uuid", + "message", + "storage_uuid", + "storage_type", + "frequency", + "enabled", + "save_s3", + "disable_local_backup", + "stop_during_backup", + "retention_amount_locally", + "retention_days_locally", + "retention_max_storage_locally", + "retention_amount_s3", + "retention_days_s3", + "retention_max_storage_s3", + "timeout" + ], + "properties": { + "uuid": { + "type": "string" + }, + "message": { + "type": "string" + }, + "storage_uuid": { + "type": "string" + }, + "storage_type": { + "type": "string", + "enum": [ + "persistent", + "directory" + ] + }, + "frequency": { + "type": "string" + }, + "enabled": { + "type": "boolean" + }, + "save_s3": { + "type": "boolean" + }, + "disable_local_backup": { + "type": "boolean" + }, + "stop_during_backup": { + "type": "boolean" + }, + "s3_storage_uuid": { + "type": [ + "string", + "null" + ] + }, + "retention_amount_locally": { + "type": "integer" + }, + "retention_days_locally": { + "type": "integer" + }, + "retention_max_storage_locally": { + "type": "number", + "format": "float" + }, + "retention_amount_s3": { + "type": "integer" + }, + "retention_days_s3": { + "type": "integer" + }, + "retention_max_storage_s3": { + "type": "number", + "format": "float" + }, + "timeout": { + "type": "integer" + } + }, + "type": "object" + }, "Application": { "description": "Application model", "properties": { diff --git a/openapi.yaml b/openapi.yaml index 2c0f4e9ea..fe3b85ad4 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -9925,6 +9925,267 @@ paths: security: - bearerAuth: [] + '/applications/{uuid}/storages/{storage_uuid}/backups': + put: + tags: + - Applications + summary: 'Set application storage backup schedule' + description: 'Create or replace the backup schedule for an application persistent volume or directory storage.' + operationId: set-application-storage-backup-schedule + parameters: + - + name: uuid + in: path + description: 'UUID of the application.' + required: true + schema: + type: string + - + name: storage_uuid + in: path + description: 'UUID of the persistent volume or directory storage.' + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleRequest' + responses: + '200': + description: 'Backup schedule replaced.' + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleResponse' + '201': + description: 'Backup schedule created.' + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleResponse' + '400': + $ref: '#/components/responses/400' + '401': + $ref: '#/components/responses/401' + '403': + description: Forbidden. + '404': + $ref: '#/components/responses/404' + '422': + $ref: '#/components/responses/422' + security: + - + bearerAuth: [] + delete: + tags: + - Applications + summary: 'Delete application storage backup schedule' + description: 'Delete the backup schedule and its local and S3 archives for an application storage.' + operationId: delete-application-storage-backup-schedule + parameters: + - + name: uuid + in: path + required: true + schema: + type: string + - + name: storage_uuid + in: path + required: true + schema: + type: string + responses: + '200': + description: 'Backup schedule and archives deleted.' + '401': + $ref: '#/components/responses/401' + '403': + description: Forbidden. + '404': + $ref: '#/components/responses/404' + '409': + description: 'Backup or recovery operation is still running.' + security: + - + bearerAuth: [] + '/databases/{uuid}/storages/{storage_uuid}/backups': + put: + tags: + - Databases + summary: 'Set database storage backup schedule' + description: 'Create or replace the backup schedule for a database persistent volume or directory storage.' + operationId: set-database-storage-backup-schedule + parameters: + - + name: uuid + in: path + description: 'UUID of the database.' + required: true + schema: + type: string + - + name: storage_uuid + in: path + description: 'UUID of the persistent volume or directory storage.' + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleRequest' + responses: + '200': + description: 'Backup schedule replaced.' + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleResponse' + '201': + description: 'Backup schedule created.' + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleResponse' + '400': + $ref: '#/components/responses/400' + '401': + $ref: '#/components/responses/401' + '403': + description: Forbidden. + '404': + $ref: '#/components/responses/404' + '422': + $ref: '#/components/responses/422' + security: + - + bearerAuth: [] + delete: + tags: + - Databases + summary: 'Delete database storage backup schedule' + description: 'Delete the backup schedule and its local and S3 archives for a database storage.' + operationId: delete-database-storage-backup-schedule + parameters: + - + name: uuid + in: path + required: true + schema: + type: string + - + name: storage_uuid + in: path + required: true + schema: + type: string + responses: + '200': + description: 'Backup schedule and archives deleted.' + '401': + $ref: '#/components/responses/401' + '403': + description: Forbidden. + '404': + $ref: '#/components/responses/404' + '409': + description: 'Backup or recovery operation is still running.' + security: + - + bearerAuth: [] + '/services/{uuid}/storages/{storage_uuid}/backups': + put: + tags: + - Services + summary: 'Set service storage backup schedule' + description: 'Create or replace the backup schedule for a service persistent volume or directory storage.' + operationId: set-service-storage-backup-schedule + parameters: + - + name: uuid + in: path + description: 'UUID of the service.' + required: true + schema: + type: string + - + name: storage_uuid + in: path + description: 'UUID of the persistent volume or directory storage.' + required: true + schema: + type: string + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleRequest' + responses: + '200': + description: 'Backup schedule replaced.' + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleResponse' + '201': + description: 'Backup schedule created.' + content: + application/json: + schema: + $ref: '#/components/schemas/VolumeBackupScheduleResponse' + '400': + $ref: '#/components/responses/400' + '401': + $ref: '#/components/responses/401' + '403': + description: Forbidden. + '404': + $ref: '#/components/responses/404' + '422': + $ref: '#/components/responses/422' + security: + - + bearerAuth: [] + delete: + tags: + - Services + summary: 'Delete service storage backup schedule' + description: 'Delete the backup schedule and its local and S3 archives for a service storage.' + operationId: delete-service-storage-backup-schedule + parameters: + - + name: uuid + in: path + required: true + schema: + type: string + - + name: storage_uuid + in: path + required: true + schema: + type: string + responses: + '200': + description: 'Backup schedule and archives deleted.' + '401': + $ref: '#/components/responses/401' + '403': + description: Forbidden. + '404': + $ref: '#/components/responses/404' + '409': + description: 'Backup or recovery operation is still running.' + security: + - + bearerAuth: [] /vultr/regions: get: tags: @@ -10014,6 +10275,130 @@ paths: bearerAuth: [] components: schemas: + VolumeBackupScheduleRequest: + required: + - frequency + properties: + frequency: + type: string + maxLength: 255 + example: '0 2 * * *' + enabled: + type: boolean + default: true + save_s3: + type: boolean + default: false + disable_local_backup: + type: boolean + default: false + stop_during_backup: + type: boolean + default: false + s3_storage_uuid: + type: + - string + - 'null' + retention_amount_locally: + type: integer + default: 7 + maximum: 10000 + minimum: 0 + retention_days_locally: + type: integer + default: 0 + maximum: 2147483647 + minimum: 0 + retention_max_storage_locally: + type: number + format: float + default: 0 + maximum: 9999999999 + minimum: 0 + retention_amount_s3: + type: integer + default: 7 + maximum: 10000 + minimum: 0 + retention_days_s3: + type: integer + default: 0 + maximum: 2147483647 + minimum: 0 + retention_max_storage_s3: + type: number + format: float + default: 0 + maximum: 9999999999 + minimum: 0 + timeout: + type: integer + default: 3600 + maximum: 36000 + minimum: 60 + type: object + additionalProperties: false + VolumeBackupScheduleResponse: + required: + - uuid + - message + - storage_uuid + - storage_type + - frequency + - enabled + - save_s3 + - disable_local_backup + - stop_during_backup + - retention_amount_locally + - retention_days_locally + - retention_max_storage_locally + - retention_amount_s3 + - retention_days_s3 + - retention_max_storage_s3 + - timeout + properties: + uuid: + type: string + message: + type: string + storage_uuid: + type: string + storage_type: + type: string + enum: + - persistent + - directory + frequency: + type: string + enabled: + type: boolean + save_s3: + type: boolean + disable_local_backup: + type: boolean + stop_during_backup: + type: boolean + s3_storage_uuid: + type: + - string + - 'null' + retention_amount_locally: + type: integer + retention_days_locally: + type: integer + retention_max_storage_locally: + type: number + format: float + retention_amount_s3: + type: integer + retention_days_s3: + type: integer + retention_max_storage_s3: + type: number + format: float + timeout: + type: integer + type: object Application: description: 'Application model' properties: diff --git a/resources/views/components/forms/checkbox.blade.php b/resources/views/components/forms/checkbox.blade.php index 29717b9b8..efd2d9e3b 100644 --- a/resources/views/components/forms/checkbox.blade.php +++ b/resources/views/components/forms/checkbox.blade.php @@ -39,7 +39,8 @@ value={{ $domValue }} id="{{ $htmlId }}" @if ($checked) checked @endif /> @else class([$defaultClass, 'shrink-0']) }} - wire:model={{ $value ?? $modelBinding }} id="{{ $htmlId }}" @if ($checked) checked @endif /> + @if ($live) wire:model.live={{ $value ?? $modelBinding }} @else wire:model={{ $value ?? $modelBinding }} @endif + id="{{ $htmlId }}" @if ($checked) checked @endif /> @endif @endif diff --git a/resources/views/components/forms/input.blade.php b/resources/views/components/forms/input.blade.php index c81f53ce9..e905aed24 100644 --- a/resources/views/components/forms/input.blade.php +++ b/resources/views/components/forms/input.blade.php @@ -10,6 +10,9 @@ @if ($helper) @endif + @isset($labelSuffix) + {{ $labelSuffix }} + @endisset @endif @if ($type === 'password') diff --git a/resources/views/components/status/running.blade.php b/resources/views/components/status/running.blade.php index 7686d30bb..752c1d576 100644 --- a/resources/views/components/status/running.blade.php +++ b/resources/views/components/status/running.blade.php @@ -21,7 +21,7 @@ $showUnknownHelper = ! str($status)->startsWith('Proxy') && (str($status)->contains('unknown') || str($healthStatus)->contains('unknown')); $showUnhealthyHelper = ! str($status)->startsWith('Proxy') && (str($status)->contains('unhealthy') || str($healthStatus)->contains('unhealthy')); @endphp -
+
@if ($lastDeploymentLink) diff --git a/resources/views/livewire/project/application/backup/create.blade.php b/resources/views/livewire/project/application/backup/create.blade.php new file mode 100644 index 000000000..8beb6e539 --- /dev/null +++ b/resources/views/livewire/project/application/backup/create.blade.php @@ -0,0 +1,18 @@ +
+ @if ($targets->isEmpty()) +
Add a persistent volume or directory mount before configuring a backup.
+ @else +
+ + @foreach ($targets as $target) + + @endforeach + + +
+ + Save + @endif +
diff --git a/resources/views/livewire/project/application/backup/index.blade.php b/resources/views/livewire/project/application/backup/index.blade.php new file mode 100644 index 000000000..2ff16d355 --- /dev/null +++ b/resources/views/livewire/project/application/backup/index.blade.php @@ -0,0 +1,89 @@ +
+ + {{ data_get_str($application, 'name')->limit(10) }} > Backups | Coolify + +

Backups

+ + + +
+

Scheduled Backups

+ @can('update', $application) + + + + @endcan +
+ +
+ +
+ +
+
+ No scheduled backups match your search. +
+ @forelse ($backups as $backup) + @php($latestExecution = $backup->latestExecution) + $latestExecution?->status === 'running', + 'border-error' => $latestExecution?->status === 'failed', + 'border-success' => $latestExecution?->status === 'success', + 'border-gray-200 dark:border-coolgray-300' => !$latestExecution, + ]) {{ wireNavigate() }} + href="{{ route('project.application.backup.show', [...$parameters, 'backup_uuid' => $backup->uuid]) }}"> +
+ @if ($latestExecution) + $latestExecution->status === 'running', + 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200' => $latestExecution->status === 'failed', + 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => $latestExecution->status === 'success', + ])> + {{ $latestExecution->status === 'running' ? 'In Progress' : ucfirst($latestExecution->status) }} + + @else + + No executions yet + + @endif +

{{ $backup->frequency }}

+ @if (!$backup->enabled) + Disabled + @endif +
+
+ {{ $backup->targetType() }}: {{ $backup->targetName() }} + @if ($latestExecution?->finished_at) + • Last run {{ $latestExecution->finished_at->diffForHumans() }} + @else + • Last run: Never + @endif + • Total executions: {{ $backup->executions_count }} + @if ($backup->save_s3) + • S3: Enabled + @endif + @if (($latestExecution?->size ?? 0) > 0) + • Size: {{ formatBytes($latestExecution->size) }} + @endif +
+ + @empty +
No scheduled backups configured.
+ @endforelse +
+
diff --git a/resources/views/livewire/project/application/backup/show.blade.php b/resources/views/livewire/project/application/backup/show.blade.php new file mode 100644 index 000000000..3627dd862 --- /dev/null +++ b/resources/views/livewire/project/application/backup/show.blade.php @@ -0,0 +1,37 @@ + diff --git a/resources/views/livewire/project/application/heading.blade.php b/resources/views/livewire/project/application/heading.blade.php index 443234869..3d9f23cbb 100644 --- a/resources/views/livewire/project/application/heading.blade.php +++ b/resources/views/livewire/project/application/heading.blade.php @@ -11,6 +11,11 @@ 'route' => 'project.application.deployment.index', 'active' => request()->routeIs('project.application.deployment.index', 'project.application.deployment.show'), ], + [ + 'label' => 'Backups', + 'route' => 'project.application.backup.index', + 'active' => request()->routeIs('project.application.backup.*'), + ], [ 'label' => 'Logs', 'route' => 'project.application.logs', @@ -447,6 +452,10 @@ class="scrollbar hidden min-h-10 w-full flex-nowrap items-center gap-6 overflow- href="{{ route('project.application.deployment.index', $parameters) }}"> Deployments +
+
+@else +
+
+

S3

+ Save + @if (!$saveS3) + Enable S3 + @else + Disable S3 + @endif +
+ +
+
+ S3 Storage + @if (!$saveS3) + (currently disabled) + @else + + @endif +
+ + @foreach ($availableS3Storages as $s3) + + @endforeach + +
+ +
+ @if ($saveS3) + + @else + + @endif +
+ +
+@endif diff --git a/resources/views/livewire/project/database/backup-executions.blade.php b/resources/views/livewire/project/database/backup-executions.blade.php index 0b7a9724a..3870a70b2 100644 --- a/resources/views/livewire/project/database/backup-executions.blade.php +++ b/resources/views/livewire/project/database/backup-executions.blade.php @@ -1,7 +1,7 @@
@isset($backup) -
-

Executions ({{ $executions_count }})

+
+

Executions

@if ($executions_count > 0)
@@ -36,7 +36,7 @@
+ class="flex flex-col gap-4 pt-2"> @forelse($executions as $execution)
Backups - diff --git a/resources/views/livewire/project/database/create-scheduled-backup.blade.php b/resources/views/livewire/project/database/create-scheduled-backup.blade.php index 8fa9924b9..0ec042718 100644 --- a/resources/views/livewire/project/database/create-scheduled-backup.blade.php +++ b/resources/views/livewire/project/database/create-scheduled-backup.blade.php @@ -2,19 +2,6 @@ -

S3

- @if ($definedS3s->count() === 0) -
No validated S3 Storages found.
- @else - - @if ($saveToS3) - - @foreach ($definedS3s as $s3) - - @endforeach - - @endif - @endif Save diff --git a/resources/views/livewire/project/database/heading.blade.php b/resources/views/livewire/project/database/heading.blade.php index 66af4895f..78b533ec0 100644 --- a/resources/views/livewire/project/database/heading.blade.php +++ b/resources/views/livewire/project/database/heading.blade.php @@ -2,14 +2,14 @@ @php $databasePageItems = [ ['label' => 'Configuration', 'route' => 'project.database.configuration', 'active' => request()->routeIs('project.database.configuration')], - ['label' => 'Logs', 'route' => 'project.database.logs', 'active' => request()->routeIs('project.database.logs')], - ['label' => 'Terminal', 'route' => 'project.database.command', 'active' => request()->routeIs('project.database.command'), 'navigate' => false, 'visible' => auth()->user()?->can('canAccessTerminal')], [ 'label' => 'Backups', 'route' => 'project.database.backup.index', - 'active' => request()->routeIs('project.database.backup.index', 'project.database.backup.execution'), + 'active' => request()->routeIs('project.database.backup.*'), 'visible' => $database->isBackupSolutionAvailable(), ], + ['label' => 'Logs', 'route' => 'project.database.logs', 'active' => request()->routeIs('project.database.logs')], + ['label' => 'Terminal', 'route' => 'project.database.command', 'active' => request()->routeIs('project.database.command'), 'navigate' => false, 'visible' => auth()->user()?->can('canAccessTerminal')], ]; $databaseConfigurationItems = [ @@ -185,6 +185,12 @@ class="scrollbar hidden min-h-10 w-full flex-nowrap items-center gap-6 overflow- Configuration + @if ($database->isBackupSolutionAvailable()) + + Backups + + @endif Logs @@ -195,12 +201,7 @@ class="scrollbar hidden min-h-10 w-full flex-nowrap items-center gap-6 overflow- Terminal @endcan - @if ($database->isBackupSolutionAvailable()) - - Backups - - @endif + @if ($database->destination->server->isFunctional()) diff --git a/resources/views/livewire/project/database/scheduled-backups.blade.php b/resources/views/livewire/project/database/scheduled-backups.blade.php index b8241569c..f93fc050f 100644 --- a/resources/views/livewire/project/database/scheduled-backups.blade.php +++ b/resources/views/livewire/project/database/scheduled-backups.blade.php @@ -1,4 +1,16 @@ -
+
@if ($database->is_migrated && blank($database->custom_type))
@@ -18,9 +30,16 @@
@else +
+ +
+
+ No scheduled backups match your search. +
@forelse($database->scheduledBackups as $backup) @if ($type == 'database') - $backup->latest_log && @@ -94,21 +113,19 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @endif @endif @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @else Last Run: Never • Total Executions: 0 @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @endif
@else -
- data_get($backup, 'id') === data_get($selectedBackup, 'id'), 'border-blue-500/50 border-dashed' => $backup->latest_log && data_get($backup->latest_log, 'status') === 'running', @@ -119,9 +136,8 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- $backup->latest_log && data_get($backup->latest_log, 'status') === 'success', 'border-gray-200 dark:border-coolgray-300' => !$backup->latest_log, - 'border-coollabs' => - data_get($backup, 'id') === data_get($selectedBackup, 'id'), - ]) wire:click="setSelectedBackup('{{ data_get($backup, 'id') }}')"> + ]) {{ wireNavigate() }} + href="{{ route('project.service.database.backup.show', [...$parameters, 'backup_uuid' => $backup->uuid]) }}"> @if ($backup->latest_log && data_get($backup->latest_log, 'status') === 'running')
@@ -182,7 +198,7 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @endif @endif @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif
Total Executions: {{ $backup->executions()->count() }} @php @@ -202,23 +218,15 @@ class="px-3 py-1 rounded-md text-xs font-medium tracking-wide shadow-xs bg-gray- @else Last Run: Never • Total Executions: 0 @if ($backup->save_s3) - • S3: Enabled + • S3: {{ $backup->s3?->name ?? 'Storage unavailable' }} @endif @endif
-
+ @endif @empty
No scheduled backups configured.
@endforelse @endif
- @if ($type === 'service-database' && $selectedBackup) -
- - -
- @endif
diff --git a/resources/views/livewire/project/service/database-backups.blade.php b/resources/views/livewire/project/service/database-backups.blade.php index 296289178..9eba9af2e 100644 --- a/resources/views/livewire/project/service/database-backups.blade.php +++ b/resources/views/livewire/project/service/database-backups.blade.php @@ -1,23 +1,65 @@
- + @if ($backup) +
+ + Back + + $section === 'general']) {{ wireNavigate() }} + href="{{ route('project.service.database.backup.show', $backupParameters) }}"> + General + + $section === 's3']) {{ wireNavigate() }} + href="{{ route('project.service.database.backup.s3', $backupParameters) }}"> + S3 + + $section === 'retention']) {{ wireNavigate() }} + href="{{ route('project.service.database.backup.retention', $backupParameters) }}"> + Retention + + $section === 'executions']) {{ wireNavigate() }} + href="{{ route('project.service.database.backup.executions', $backupParameters) }}"> + Executions + + $section === 'danger']) {{ wireNavigate() }} + href="{{ route('project.service.database.backup.danger', $backupParameters) }}"> + Danger Zone + +
+ @else + + @endif +
{{ data_get_str($service, 'name')->limit(10) }} > {{ data_get_str($serviceDatabase, 'name')->limit(10) }} > Backups | Coolify -
-

Scheduled Backups

- @if (filled($serviceDatabase->custom_type) || !$serviceDatabase->is_migrated) - @can('update', $serviceDatabase) - - - - @endcan + + @if ($backup) + @if ($section === 'executions') + + @else + @endif -
- + @else +
+

Scheduled Backups

+ @if (filled($serviceDatabase->custom_type) || !$serviceDatabase->is_migrated) + @can('update', $serviceDatabase) + + + + @endcan + @endif +
+ + @endif
diff --git a/resources/views/livewire/project/service/file-storage.blade.php b/resources/views/livewire/project/service/file-storage.blade.php index e47d290b8..5479eea35 100644 --- a/resources/views/livewire/project/service/file-storage.blade.php +++ b/resources/views/livewire/project/service/file-storage.blade.php @@ -19,7 +19,15 @@ @endif
- + + + @if ($hasEnabledBackup) + + @endif + +
@@ -51,6 +59,14 @@ confirmationText="{{ $fs_path }}" confirmationLabel="Please confirm the execution of the actions by entering the Filepath below" shortConfirmationLabel="Filepath" :confirmWithPassword="false" step2ButtonText="Convert to file" /> + @if ($resource instanceof \App\Models\Application) + + + + @endif + + +
+ @endcan + @endif
diff --git a/resources/views/livewire/project/service/heading.blade.php b/resources/views/livewire/project/service/heading.blade.php index 820d99080..c8933781d 100644 --- a/resources/views/livewire/project/service/heading.blade.php +++ b/resources/views/livewire/project/service/heading.blade.php @@ -23,7 +23,7 @@ ['label' => 'Back', 'route' => 'project.service.configuration', 'active' => false, 'parameters' => [...$parameters, 'stack_service_uuid' => null]], ['label' => 'General', 'route' => 'project.service.index', 'active' => request()->routeIs('project.service.index')], ['label' => 'Advanced', 'route' => 'project.service.index.advanced', 'active' => request()->routeIs('project.service.index.advanced')], - ['label' => 'Backups', 'route' => 'project.service.database.backups', 'active' => request()->routeIs('project.service.database.backups')], + ['label' => 'Backups', 'route' => 'project.service.database.backups', 'active' => request()->routeIs('project.service.database.backup*')], ['label' => 'Import Backup', 'route' => 'project.service.database.import', 'active' => request()->routeIs('project.service.database.import')], ]; } diff --git a/resources/views/livewire/project/shared/configuration-checker.blade.php b/resources/views/livewire/project/shared/configuration-checker.blade.php index 19974c587..8b0e8298d 100644 --- a/resources/views/livewire/project/shared/configuration-checker.blade.php +++ b/resources/views/livewire/project/shared/configuration-checker.blade.php @@ -55,6 +55,7 @@ class="relative flex max-h-[85vh] w-full flex-col rounded-sm border border-neutr

Configuration changes

These changes are not applied to the latest deployment yet. + (If you think it is incorrect, just ignore)

@endcan @endif + @if ($resource instanceof \App\Models\Application) + @can('update', $resource) + + + + @endcan + @endif @else @can('update', $resource) @if ($isFirst)
- + + + @if ($hasEnabledBackup) + + @endif + +
@else
- + + + @if ($hasEnabledBackup) + + @endif + +
@@ -74,6 +123,13 @@ Update + @if ($resource instanceof \App\Models\Application) + + + + @endif - + + + @if ($hasEnabledBackup) + + @endif + + @else
- + + + @if ($hasEnabledBackup) + + @endif + +
diff --git a/resources/views/livewire/project/shared/storages/volume-backups.blade.php b/resources/views/livewire/project/shared/storages/volume-backups.blade.php new file mode 100644 index 000000000..cc354e88d --- /dev/null +++ b/resources/views/livewire/project/shared/storages/volume-backups.blade.php @@ -0,0 +1,21 @@ +
+ @if ($section === 'retention') + @include('livewire.project.shared.storages.volume-backups.retention') + @elseif ($section === 's3') + @include('livewire.project.shared.storages.volume-backups.s3') + @elseif ($section === 'executions') + @include('livewire.project.shared.storages.volume-backups.executions') + @elseif ($section === 'danger') + @include('livewire.project.shared.storages.volume-backups.danger') + @else + @include('livewire.project.shared.storages.volume-backups.general') + @endif + + @script + + @endscript +
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/danger.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/danger.blade.php new file mode 100644 index 000000000..f266abb71 --- /dev/null +++ b/resources/views/livewire/project/shared/storages/volume-backups/danger.blade.php @@ -0,0 +1,24 @@ +
+
+

Danger Zone

+
Woah. I hope you know what you are doing.
+
+ +
+

Delete Scheduled Backup

+
+ This permanently deletes the schedule and all local and S3 archives created by it. There is no coming back. +
+ @if ($backup) + + + Delete Backups and Schedule + + + @endif +
+
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php new file mode 100644 index 000000000..271956ea4 --- /dev/null +++ b/resources/views/livewire/project/shared/storages/volume-backups/executions.blade.php @@ -0,0 +1,183 @@ + @if ($backup) + @php + $executionCount = method_exists($executions, 'total') ? $executions->total() : $executions->count(); + $currentPage = method_exists($executions, 'currentPage') ? $executions->currentPage() : 1; + $lastPage = method_exists($executions, 'lastPage') ? $executions->lastPage() : 1; + @endphp +
+
+

Executions

+ @if ($executionCount > 0) +
+ + + + + + + Page {{ $currentPage }} of {{ $lastPage }} + + + + + + +
+ @endif +
+ + Cleanup Failed Backups + + + + Cleanup Deleted + + +
+
+ +
+ @forelse ($executions as $execution) +
$execution->status === 'running', + 'border-error' => $execution->status === 'failed', + 'border-success' => $execution->status === 'success', + ])> + @if ($execution->status === 'running') +
+ @endif +
+ $execution->status === 'running', + 'bg-red-100 text-red-800 dark:bg-red-900/30 dark:text-red-200' => $execution->status === 'failed', + 'bg-amber-100 text-amber-800 dark:bg-amber-900/30 dark:text-amber-200' => $execution->status === 'success' && $execution->s3_uploaded === false, + 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => $execution->status === 'success' && $execution->s3_uploaded !== false, + ])> + @if ($execution->status === 'running') + In Progress + @elseif ($execution->status === 'success' && $execution->s3_uploaded === false) + Success (S3 Warning) + @else + {{ ucfirst($execution->status) }} + @endif + +
+
+ @if ($execution->status === 'running') + + Running for {{ calculateDuration($execution->created_at, now()) }} + + @else + @php + $finishedAt = $execution->finished_at ?? $execution->updated_at; + @endphp + + {{ $finishedAt->diffForHumans() }} + ({{ calculateDuration($execution->created_at, $finishedAt) }}) + • {{ $finishedAt->format('M j, H:i') }} + + @endif + • {{ $backup->targetType() }}: {{ $backup->targetName() }} + @if ($execution->size > 0) + • Size: {{ formatBytes($execution->size) }} + @endif +
+
+ Location: {{ $execution->filename ?? 'N/A' }} +
+
+
Backup Availability:
+ !$execution->local_storage_deleted, + 'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => $execution->local_storage_deleted, + ])> + + @if (!$execution->local_storage_deleted) + + + + @else + + + + @endif + Local Storage + + + @if ($execution->s3_uploaded !== null) + $execution->s3_uploaded === false && !$execution->s3_storage_deleted, + 'bg-green-100 text-green-800 dark:bg-green-900/30 dark:text-green-200' => $execution->s3_uploaded === true && !$execution->s3_storage_deleted, + 'bg-gray-100 text-gray-600 dark:bg-gray-800/50 dark:text-gray-400' => $execution->s3_storage_deleted, + ])> + + @if ($execution->s3_uploaded === true && !$execution->s3_storage_deleted) + + + + @else + + + + @endif + S3 Storage + + + @endif +
+ @if ($execution->message) +
+ 
{{ $execution->message }}
+
+ @endif + @if ($execution->status !== 'running') +
+ @if ($execution->status === 'success' && !$execution->local_storage_deleted) + + Download + + @endif + @php + $executionCheckboxes = []; + $deleteActions = []; + if (!$execution->local_storage_deleted) { + $deleteActions[] = 'This backup will be permanently deleted from local storage.'; + } + if ($execution->s3_uploaded === true && !$execution->s3_storage_deleted) { + $executionCheckboxes[] = ['id' => 'delete_backup_s3', 'label' => 'Delete the selected backup permanently from S3 Storage']; + } + if (empty($deleteActions)) { + $deleteActions[] = 'This backup execution record will be deleted.'; + } + @endphp + + + Delete + + +
+ @endif +
+ @empty +
No executions found.
+ @endforelse +
+
+ @endif diff --git a/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php new file mode 100644 index 000000000..3b66f900c --- /dev/null +++ b/resources/views/livewire/project/shared/storages/volume-backups/general.blade.php @@ -0,0 +1,47 @@ +
+
+
+

General

+
+ Save + @if (!$enabled) + Enable Backup + @else + Disable Backup + @endif + + Backup Now +
+
+

+ {{ $backup?->targetType() ?? ($storage instanceof \App\Models\LocalFileVolume ? 'Directory' : 'Volume') }}: + + {{ $backup?->targetName() ?? ($storage instanceof \App\Models\LocalFileVolume ? $storage->fs_path : $storage->name) }} + +

+
+ +
+ Backups made while the application is writing to this storage may be inconsistent or corrupted. You can + gracefully stop containers during the archive step for a safer file-level backup, but this briefly + interrupts the application. +
+ +
+ +
+ +
+
+ + + +
+
+
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/retention.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/retention.blade.php new file mode 100644 index 000000000..6ee971b1d --- /dev/null +++ b/resources/views/livewire/project/shared/storages/volume-backups/retention.blade.php @@ -0,0 +1,52 @@ +
+
+

Retention

+ Save +
+ +
+
    +
  • Setting a value to 0 means unlimited retention.
    • +
  • The retention rules work independently - whichever limit is reached first will trigger cleanup.
    • +
+
+ +
+
+

Local Backup Retention

+
+ + + +
+
+ +
+

S3 Storage Retention

+
+ + + +
+
+
+
diff --git a/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php b/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php new file mode 100644 index 000000000..2d5e328e4 --- /dev/null +++ b/resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php @@ -0,0 +1,54 @@ +@if ($availableS3Storages->isEmpty()) +
+

S3

+
+ No validated S3 available. Configure one here. +
+
+@else +
+
+

S3

+ Save + @if (!$saveToS3) + Enable S3 + @else + Disable S3 + @endif +
+ +
+
+ S3 Storage + @if (!$saveToS3) + (currently disabled) + @else + + @endif +
+ + @foreach ($availableS3Storages as $s3Storage) + + @endforeach + +
+ +
+ @if ($saveToS3) + + @else + + @endif +
+ +
+@endif diff --git a/resources/views/livewire/storage/show.blade.php b/resources/views/livewire/storage/show.blade.php index 0d580486e..f510da1ba 100644 --- a/resources/views/livewire/storage/show.blade.php +++ b/resources/views/livewire/storage/show.blade.php @@ -20,7 +20,7 @@ submitAction="delete({{ $storage->id }})" :actions="array_filter([ 'The selected storage location will be permanently deleted from Coolify.', $backupCount > 0 - ? $backupCount . ' backup schedule(s) will be updated to no longer save to S3 and will only store backups locally on the server.' + ? $backupCount . ' backup schedule(s) will stop saving to S3. Existing objects in this storage will not be deleted.' : null, ])" confirmationText="{{ $storage->name }}" confirmationLabel="Please confirm the execution of the actions by entering the Storage Name below" diff --git a/routes/api.php b/routes/api.php index 7407f2924..794ca9269 100644 --- a/routes/api.php +++ b/routes/api.php @@ -20,6 +20,7 @@ use App\Http\Controllers\Api\ServicesController; use App\Http\Controllers\Api\TagsController; use App\Http\Controllers\Api\TeamController; +use App\Http\Controllers\Api\VolumeBackupsController; use App\Http\Controllers\Api\VultrController; use App\Http\Middleware\ApiAllowed; use Illuminate\Support\Facades\Route; @@ -154,6 +155,12 @@ Route::post('/applications/{uuid}/storages', [ApplicationsController::class, 'create_storage'])->middleware(['api.ability:write']); Route::patch('/applications/{uuid}/storages', [ApplicationsController::class, 'update_storage'])->middleware(['api.ability:write']); Route::delete('/applications/{uuid}/storages/{storage_uuid}', [ApplicationsController::class, 'delete_storage'])->middleware(['api.ability:write']); + Route::put('/applications/{uuid}/storages/{storage_uuid}/backups', [VolumeBackupsController::class, 'upsert']) + ->defaults('resource_type', 'application') + ->middleware(['api.ability:write']); + Route::delete('/applications/{uuid}/storages/{storage_uuid}/backups', [VolumeBackupsController::class, 'destroy']) + ->defaults('resource_type', 'application') + ->middleware(['api.ability:write']); Route::get('/applications/{uuid}/tags', [ApplicationsController::class, 'tags'])->middleware(['api.ability:read']); Route::post('/applications/{uuid}/tags', [ApplicationsController::class, 'create_tag'])->middleware(['api.ability:write']); @@ -201,6 +208,12 @@ Route::post('/databases/{uuid}/storages', [DatabasesController::class, 'create_storage'])->middleware(['api.ability:write']); Route::patch('/databases/{uuid}/storages', [DatabasesController::class, 'update_storage'])->middleware(['api.ability:write']); Route::delete('/databases/{uuid}/storages/{storage_uuid}', [DatabasesController::class, 'delete_storage'])->middleware(['api.ability:write']); + Route::put('/databases/{uuid}/storages/{storage_uuid}/backups', [VolumeBackupsController::class, 'upsert']) + ->defaults('resource_type', 'database') + ->middleware(['api.ability:write']); + Route::delete('/databases/{uuid}/storages/{storage_uuid}/backups', [VolumeBackupsController::class, 'destroy']) + ->defaults('resource_type', 'database') + ->middleware(['api.ability:write']); Route::get('/databases/{uuid}/envs', [DatabasesController::class, 'envs'])->middleware(['api.ability:read']); Route::post('/databases/{uuid}/envs', [DatabasesController::class, 'create_env'])->middleware(['api.ability:write']); @@ -231,6 +244,12 @@ Route::post('/services/{uuid}/storages', [ServicesController::class, 'create_storage'])->middleware(['api.ability:write']); Route::patch('/services/{uuid}/storages', [ServicesController::class, 'update_storage'])->middleware(['api.ability:write']); Route::delete('/services/{uuid}/storages/{storage_uuid}', [ServicesController::class, 'delete_storage'])->middleware(['api.ability:write']); + Route::put('/services/{uuid}/storages/{storage_uuid}/backups', [VolumeBackupsController::class, 'upsert']) + ->defaults('resource_type', 'service') + ->middleware(['api.ability:write']); + Route::delete('/services/{uuid}/storages/{storage_uuid}/backups', [VolumeBackupsController::class, 'destroy']) + ->defaults('resource_type', 'service') + ->middleware(['api.ability:write']); Route::get('/services/{uuid}/envs', [ServicesController::class, 'envs'])->middleware(['api.ability:read']); Route::post('/services/{uuid}/envs', [ServicesController::class, 'create_env'])->middleware(['api.ability:write']); diff --git a/routes/web.php b/routes/web.php index f729af28e..c26b3eb15 100644 --- a/routes/web.php +++ b/routes/web.php @@ -18,6 +18,8 @@ use App\Livewire\Notifications\Webhook as NotificationWebhook; use App\Livewire\Profile\Appearance as ProfileAppearance; use App\Livewire\Profile\Index as ProfileIndex; +use App\Livewire\Project\Application\Backup\Index as ApplicationBackupIndex; +use App\Livewire\Project\Application\Backup\Show as ApplicationBackupShow; use App\Livewire\Project\Application\Configuration as ApplicationConfiguration; use App\Livewire\Project\Application\Deployment\Index as DeploymentIndex; use App\Livewire\Project\Application\Deployment\Show as DeploymentShow; @@ -91,12 +93,12 @@ use App\Livewire\Team\Member\Index as TeamMemberIndex; use App\Livewire\Terminal\Index as TerminalIndex; use App\Models\ScheduledDatabaseBackupExecution; +use App\Models\ScheduledVolumeBackupExecution; use App\Models\Server; use App\Models\ServiceDatabase; use App\Providers\RouteServiceProvider; use Illuminate\Support\Facades\Route; -use Illuminate\Support\Facades\Storage; -use Symfony\Component\HttpFoundation\StreamedResponse; +use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException; Route::post('/forgot-password', [Controller::class, 'forgot_password'])->name('password.forgot')->middleware('throttle:forgot-password'); Route::get('/realtime', [Controller::class, 'realtime_test'])->middleware('auth'); @@ -224,6 +226,12 @@ Route::get('/advanced', ApplicationConfiguration::class)->name('project.application.advanced'); Route::get('/environment-variables', ApplicationConfiguration::class)->name('project.application.environment-variables'); Route::get('/persistent-storage', ApplicationConfiguration::class)->name('project.application.persistent-storage'); + Route::get('/backups', ApplicationBackupIndex::class)->name('project.application.backup.index'); + Route::get('/backups/{backup_uuid}', ApplicationBackupShow::class)->name('project.application.backup.show'); + Route::get('/backups/{backup_uuid}/s3', ApplicationBackupShow::class)->name('project.application.backup.s3'); + Route::get('/backups/{backup_uuid}/retention', ApplicationBackupShow::class)->name('project.application.backup.retention'); + Route::get('/backups/{backup_uuid}/executions', ApplicationBackupShow::class)->name('project.application.backup.executions'); + Route::get('/backups/{backup_uuid}/danger', ApplicationBackupShow::class)->name('project.application.backup.danger'); Route::get('/source', ApplicationConfiguration::class)->name('project.application.source'); Route::get('/servers', ApplicationConfiguration::class)->name('project.application.servers'); Route::get('/scheduled-tasks', ApplicationConfiguration::class)->name('project.application.scheduled-tasks.show'); @@ -261,6 +269,10 @@ Route::get('/terminal', ExecuteContainerCommand::class)->name('project.database.command')->middleware('can.access.terminal'); Route::get('/backups', DatabaseBackupIndex::class)->name('project.database.backup.index'); Route::get('/backups/{backup_uuid}', DatabaseBackupExecution::class)->name('project.database.backup.execution'); + Route::get('/backups/{backup_uuid}/s3', DatabaseBackupExecution::class)->name('project.database.backup.s3'); + Route::get('/backups/{backup_uuid}/retention', DatabaseBackupExecution::class)->name('project.database.backup.retention'); + Route::get('/backups/{backup_uuid}/executions', DatabaseBackupExecution::class)->name('project.database.backup.executions'); + Route::get('/backups/{backup_uuid}/danger', DatabaseBackupExecution::class)->name('project.database.backup.danger'); }); Route::prefix('project/{project_uuid}/environment/{environment_uuid}/service/{service_uuid}')->group(function () { Route::get('/', ServiceConfiguration::class)->name('project.service.configuration'); @@ -274,6 +286,11 @@ Route::get('/danger', ServiceConfiguration::class)->name('project.service.danger'); Route::get('/terminal', ExecuteContainerCommand::class)->name('project.service.command')->middleware('can.access.terminal'); Route::get('/{stack_service_uuid}/backups', ServiceDatabaseBackups::class)->name('project.service.database.backups'); + Route::get('/{stack_service_uuid}/backups/{backup_uuid}', ServiceDatabaseBackups::class)->name('project.service.database.backup.show'); + Route::get('/{stack_service_uuid}/backups/{backup_uuid}/s3', ServiceDatabaseBackups::class)->name('project.service.database.backup.s3'); + Route::get('/{stack_service_uuid}/backups/{backup_uuid}/retention', ServiceDatabaseBackups::class)->name('project.service.database.backup.retention'); + Route::get('/{stack_service_uuid}/backups/{backup_uuid}/executions', ServiceDatabaseBackups::class)->name('project.service.database.backup.executions'); + Route::get('/{stack_service_uuid}/backups/{backup_uuid}/danger', ServiceDatabaseBackups::class)->name('project.service.database.backup.danger'); Route::get('/{stack_service_uuid}/import', ServiceIndex::class)->name('project.service.database.import')->middleware('can.update.resource'); Route::get('/{stack_service_uuid}/advanced', ServiceIndex::class)->name('project.service.index.advanced'); Route::get('/{stack_service_uuid}', ServiceIndex::class)->name('project.service.index'); @@ -371,46 +388,52 @@ $server = $execution->scheduledDatabaseBackup->database->destination->server; } - $privateKeyLocation = $server->privateKey->getKeyLocation(); - $disk = Storage::build([ - 'driver' => 'sftp', - 'host' => $server->ip, - 'port' => (int) $server->port, - 'username' => $server->user, - 'privateKey' => $privateKeyLocation, - 'root' => '/', - ]); - if (! $disk->exists($filename)) { - if ($execution->scheduledDatabaseBackup->disable_local_backup === true && $execution->scheduledDatabaseBackup->save_s3 === true) { - return response()->json(['message' => 'Backup not available locally, but available on S3.'], 404); - } - - return response()->json(['message' => 'Backup not found locally on the server.'], 404); + return streamBackupFromServer($server, $filename, 'application/octet-stream'); + } catch (FileNotFoundException) { + if (isset($execution) && $execution->scheduledDatabaseBackup->disable_local_backup === true && $execution->scheduledDatabaseBackup->save_s3 === true) { + return response()->json(['message' => 'Backup not available locally, but available on S3.'], 404); } - return new StreamedResponse(function () use ($disk, $filename) { - if (ob_get_level()) { - ob_end_clean(); - } - $stream = $disk->readStream($filename); - if ($stream === false || is_null($stream)) { - abort(500, 'Failed to open stream for the requested file.'); - } - while (! feof($stream)) { - echo fread($stream, 2048); - flush(); - } - - fclose($stream); - }, 200, [ - 'Content-Type' => 'application/octet-stream', - 'Content-Disposition' => 'attachment; filename="'.basename($filename).'"', - ]); + return response()->json(['message' => 'Backup not found locally on the server.'], 404); } catch (Throwable $e) { return response()->json(['message' => 'Failed to download backup.'], 500); } })->name('download.backup'); + Route::get('/download/volume-backup/{executionId}', function () { + try { + $user = auth()->user(); + $team = $user->currentTeam(); + if (is_null($team)) { + return response()->json(['message' => 'Team not found.'], 404); + } + if ($user->isAdminFromSession() === false) { + return response()->json(['message' => 'Only team admins/owners can download backups.'], 403); + } + + $execution = ScheduledVolumeBackupExecution::query() + ->with('scheduledVolumeBackup.backupable.resource') + ->findOrFail(request()->route('executionId')); + if ($team->id !== 0 && $team->id !== $execution->scheduledVolumeBackup->team_id) { + return response()->json(['message' => 'Permission denied.'], 403); + } + if ($execution->local_storage_deleted || blank($execution->filename)) { + return response()->json(['message' => 'Backup not found locally on the server.'], 404); + } + + $server = $execution->scheduledVolumeBackup->server(); + if (! $server) { + return response()->json(['message' => 'Server not found.'], 404); + } + + return streamBackupFromServer($server, $execution->filename, 'application/gzip'); + } catch (FileNotFoundException) { + return response()->json(['message' => 'Backup not found locally on the server.'], 404); + } catch (Throwable) { + return response()->json(['message' => 'Failed to download backup.'], 500); + } + })->name('download.volume-backup'); + }); Route::any('/{any}', function () { diff --git a/tests/Feature/Api/VolumeBackupScheduleApiTest.php b/tests/Feature/Api/VolumeBackupScheduleApiTest.php new file mode 100644 index 000000000..4f4bc395b --- /dev/null +++ b/tests/Feature/Api/VolumeBackupScheduleApiTest.php @@ -0,0 +1,406 @@ + InstanceSettings::updateOrCreate(['id' => 0], ['id' => 0])); + + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->team->members()->attach($this->user, ['role' => 'owner']); + + $plainTextToken = Str::random(40); + $token = $this->user->tokens()->create([ + 'name' => 'volume-backup-api-test', + 'token' => hash('sha256', $plainTextToken), + 'abilities' => ['*'], + 'team_id' => $this->team->id, + ]); + $this->headers = [ + 'Authorization' => 'Bearer '.$token->getKey().'|'.$plainTextToken, + 'Content-Type' => 'application/json', + ]; + + $this->server = Server::factory()->create(['team_id' => $this->team->id]); + $this->destination = StandaloneDocker::query()->where('server_id', $this->server->id)->firstOrFail(); + $this->project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); + + $this->application = Application::factory()->create([ + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + $this->volume = LocalPersistentVolume::create([ + 'name' => 'api-volume', + 'mount_path' => '/data', + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]); + + $this->s3Storage = S3Storage::create([ + 'name' => 'volume-backup-s3', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $this->team->id, + 'is_usable' => true, + ]); +}); + +function createVolumeBackupApiToken($context, User $user, array $abilities): string +{ + $plainTextToken = Str::random(40); + $token = $user->tokens()->create([ + 'name' => 'volume-backup-api-permission-test', + 'token' => hash('sha256', $plainTextToken), + 'abilities' => $abilities, + 'team_id' => $context->team->id, + ]); + + return $token->getKey().'|'.$plainTextToken; +} + +it('sets an application volume backup schedule through the API', function () { + $response = $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => '0 2 * * *', + 'enabled' => true, + 'save_s3' => true, + 'disable_local_backup' => true, + 'stop_during_backup' => true, + 's3_storage_uuid' => $this->s3Storage->uuid, + 'retention_amount_locally' => 3, + 'retention_days_locally' => 4, + 'retention_max_storage_locally' => 5.5, + 'retention_amount_s3' => 6, + 'retention_days_s3' => 7, + 'retention_max_storage_s3' => 8.5, + 'timeout' => 600, + ]); + + $response->assertCreated()->assertJsonStructure(['uuid', 'message']); + + $backup = ScheduledVolumeBackup::query()->sole(); + expect($backup->backupable->is($this->volume))->toBeTrue() + ->and($backup->team_id)->toBe($this->team->id) + ->and($backup->frequency)->toBe('0 2 * * *') + ->and($backup->enabled)->toBeTrue() + ->and($backup->save_s3)->toBeTrue() + ->and($backup->disable_local_backup)->toBeTrue() + ->and($backup->stop_during_backup)->toBeTrue() + ->and($backup->s3_storage_id)->toBe($this->s3Storage->id) + ->and($backup->retention_amount_locally)->toBe(3) + ->and($backup->retention_days_locally)->toBe(4) + ->and($backup->retention_max_storage_locally)->toBe(5.5) + ->and($backup->retention_amount_s3)->toBe(6) + ->and($backup->retention_days_s3)->toBe(7) + ->and($backup->retention_max_storage_s3)->toBe(8.5) + ->and($backup->timeout)->toBe(600); +}); + +it('updates the existing backup schedule instead of creating another one', function () { + $this->volume->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $this->s3Storage->id, + ]); + + $response = $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => 'hourly', + 'enabled' => false, + 'save_s3' => false, + ]); + + $response->assertOk(); + + $backup = ScheduledVolumeBackup::query()->sole(); + expect($backup->frequency)->toBe('hourly') + ->and($backup->enabled)->toBeFalse() + ->and($backup->save_s3)->toBeFalse() + ->and($backup->disable_local_backup)->toBeFalse() + ->and($backup->s3_storage_id)->toBeNull(); +}); + +it('sets a directory backup schedule through the API', function () { + $directory = LocalFileVolume::unguarded(fn () => LocalFileVolume::withoutEvents(fn () => LocalFileVolume::create([ + 'uuid' => new_public_id(), + 'fs_path' => './uploads', + 'mount_path' => '/app/uploads', + 'is_directory' => true, + 'is_host_file' => false, + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]))); + + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$directory->uuid}/backups", [ + 'frequency' => 'daily', + ]) + ->assertCreated(); + + expect(ScheduledVolumeBackup::query()->sole()->backupable->is($directory))->toBeTrue(); +}); + +it('refuses to delete an application directory before its backup schedule is removed', function () { + Process::fake(); + $directory = LocalFileVolume::unguarded(fn () => LocalFileVolume::withoutEvents(fn () => LocalFileVolume::create([ + 'uuid' => new_public_id(), + 'fs_path' => './uploads', + 'mount_path' => '/app/uploads', + 'is_directory' => true, + 'is_host_file' => false, + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]))); + $directory->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + ]); + + $this->withHeaders($this->headers) + ->deleteJson("/api/v1/applications/{$this->application->uuid}/storages/{$directory->uuid}") + ->assertUnprocessable() + ->assertJsonPath('message', 'Delete this directory backup schedule and its archives before deleting the directory.'); + + expect($directory->fresh())->not->toBeNull(); + Process::assertNothingRan(); +}); + +it('deletes an application volume backup schedule through the API', function () { + $backup = $this->volume->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + ]); + + $this->withHeaders($this->headers) + ->deleteJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups") + ->assertOk() + ->assertJsonPath('message', 'Storage backup schedule and archives deleted.'); + + expect($backup->fresh())->toBeNull(); +}); + +it('sets and deletes database and service volume backup schedules through their storage APIs', function () { + $database = StandalonePostgresql::create([ + 'name' => 'api-postgres', + 'image' => 'postgres:17-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + $databaseVolume = $database->persistentStorages()->firstOrFail(); + + $service = Service::factory()->create([ + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + $serviceApplication = ServiceApplication::create([ + 'uuid' => new_public_id(), + 'name' => 'api-service-application', + 'image' => 'nginx:alpine', + 'service_id' => $service->id, + ]); + $serviceVolume = LocalPersistentVolume::create([ + 'name' => 'service-api-volume', + 'mount_path' => '/data', + 'resource_id' => $serviceApplication->id, + 'resource_type' => $serviceApplication->getMorphClass(), + ]); + + $this->withHeaders($this->headers) + ->putJson("/api/v1/databases/{$database->uuid}/storages/{$databaseVolume->uuid}/backups", ['frequency' => 'daily']) + ->assertCreated(); + $this->withHeaders($this->headers) + ->putJson("/api/v1/services/{$service->uuid}/storages/{$serviceVolume->uuid}/backups", ['frequency' => 'weekly']) + ->assertCreated(); + + expect($databaseVolume->scheduledBackups()->sole()->frequency)->toBe('daily') + ->and($serviceVolume->scheduledBackups()->sole()->frequency)->toBe('weekly'); + + $this->withHeaders($this->headers) + ->deleteJson("/api/v1/databases/{$database->uuid}/storages/{$databaseVolume->uuid}/backups") + ->assertOk(); + $this->withHeaders($this->headers) + ->deleteJson("/api/v1/services/{$service->uuid}/storages/{$serviceVolume->uuid}/backups") + ->assertOk(); + + expect($databaseVolume->scheduledBackups()->count())->toBe(0) + ->and($serviceVolume->scheduledBackups()->count())->toBe(0); +}); + +it('rejects ineligible file storages and invalid schedule settings', function () { + $file = LocalFileVolume::unguarded(fn () => LocalFileVolume::withoutEvents(fn () => LocalFileVolume::create([ + 'uuid' => new_public_id(), + 'fs_path' => '/tmp/config.json', + 'mount_path' => '/app/config.json', + 'is_directory' => false, + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]))); + + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$file->uuid}/backups", ['frequency' => 'daily']) + ->assertUnprocessable() + ->assertJsonValidationErrors(['storage_uuid']); + + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => 'not-a-frequency', + 'save_s3' => false, + 'disable_local_backup' => true, + 'unexpected' => true, + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors(['frequency', 'disable_local_backup', 'unexpected']); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('rejects an s3 storage owned by another team', function () { + $otherTeam = Team::factory()->create(); + $otherS3Storage = S3Storage::create([ + 'name' => 'other-team-s3', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $otherTeam->id, + 'is_usable' => true, + ]); + + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_uuid' => $otherS3Storage->uuid, + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors(['s3_storage_uuid']); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('rejects schedule values larger than the database columns support', function () { + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups", [ + 'frequency' => str_repeat('a', 256), + 'retention_days_locally' => 2147483648, + 'retention_days_s3' => 2147483648, + 'retention_max_storage_locally' => 10000000000, + 'retention_max_storage_s3' => 10000000000, + ]) + ->assertUnprocessable() + ->assertJsonValidationErrors([ + 'frequency', + 'retention_days_locally', + 'retention_days_s3', + 'retention_max_storage_locally', + 'retention_max_storage_s3', + ]); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('requires authentication, write ability, and an admin team role', function () { + $endpoint = "/api/v1/applications/{$this->application->uuid}/storages/{$this->volume->uuid}/backups"; + + $this->putJson($endpoint, ['frequency' => 'daily'])->assertUnauthorized(); + + $readToken = createVolumeBackupApiToken($this, $this->user, ['read']); + $this->withToken($readToken)->putJson($endpoint, ['frequency' => 'daily'])->assertForbidden(); + + auth()->forgetGuards(); + $member = User::factory()->create(); + $this->team->members()->attach($member, ['role' => 'member']); + $memberWriteToken = createVolumeBackupApiToken($this, $member, ['write']); + $this->withToken($memberWriteToken)->putJson($endpoint, ['frequency' => 'daily'])->assertForbidden(); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('does not set schedules through resources owned by another team', function () { + $otherTeam = Team::factory()->create(); + $otherProject = Project::factory()->create(['team_id' => $otherTeam->id]); + $otherEnvironment = Environment::factory()->create(['project_id' => $otherProject->id]); + $otherApplication = Application::factory()->create([ + 'environment_id' => $otherEnvironment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + $otherDatabase = StandalonePostgresql::create([ + 'name' => 'other-team-postgres', + 'image' => 'postgres:17-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $otherEnvironment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + $otherService = Service::factory()->create([ + 'environment_id' => $otherEnvironment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + + foreach (['applications' => $otherApplication, 'databases' => $otherDatabase, 'services' => $otherService] as $type => $resource) { + $this->withHeaders($this->headers) + ->putJson("/api/v1/{$type}/{$resource->uuid}/storages/{$this->volume->uuid}/backups", ['frequency' => 'daily']) + ->assertNotFound(); + } + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('does not set a schedule for storage outside the scoped parent', function () { + $otherApplication = Application::factory()->create([ + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + $otherVolume = LocalPersistentVolume::create([ + 'name' => 'other-application-volume', + 'mount_path' => '/data', + 'resource_id' => $otherApplication->id, + 'resource_type' => $otherApplication->getMorphClass(), + ]); + + $this->withHeaders($this->headers) + ->putJson("/api/v1/applications/{$this->application->uuid}/storages/{$otherVolume->uuid}/backups", ['frequency' => 'daily']) + ->assertNotFound(); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); diff --git a/tests/Feature/BackupEditValidationTest.php b/tests/Feature/BackupEditValidationTest.php index fe396b5da..152bbc6da 100644 --- a/tests/Feature/BackupEditValidationTest.php +++ b/tests/Feature/BackupEditValidationTest.php @@ -1,5 +1,6 @@ $this->team]); }); +it('renders a highlighted enable backup button and a regular disable backup button', function () { + $view = file_get_contents(resource_path('views/livewire/project/database/backup-edit/general.blade.php')); + $s3View = file_get_contents(resource_path('views/livewire/project/database/backup-edit/s3.blade.php')); + + expect($view) + ->toContain('wire:target="toggleEnabled" isHighlighted>Enable Backup') + ->toContain('wire:target="toggleEnabled">Disable Backup') + ->not->toContain('label="Backup Enabled"') + ->and($s3View) + ->toContain('wire:target="toggleS3" isHighlighted') + ->toContain('wire:target="toggleS3">Disable S3') + ->not->toContain('label="S3 Enabled"'); +}); + +it('enables and disables S3 backups from the S3 title action', function () { + $s3 = createS3StorageForBackupEditValidationTest($this->team); + $backup = createBackupForEditValidationTest($this->team, [ + 'save_s3' => false, + 's3_storage_id' => $s3->id, + ]); + + $component = Livewire::test(BackupEdit::class, [ + 'backup' => $backup->fresh(), + 'availableS3Storages' => $this->team->s3s, + 'section' => 's3', + ]) + ->assertSee('Enable S3') + ->call('toggleS3') + ->assertSet('saveS3', true) + ->assertSee('Disable S3'); + + expect($backup->refresh()->save_s3)->toBeTruthy(); + + $component->call('toggleS3')->assertSet('saveS3', false); + + expect($backup->refresh()->save_s3)->toBeFalsy(); +}); + +it('shows and saves S3 retention while S3 backups are disabled', function () { + $backup = createBackupForEditValidationTest($this->team, [ + 'enabled' => false, + 'save_s3' => false, + 'database_backup_retention_amount_locally' => 0, + 'database_backup_retention_days_locally' => 0, + 'database_backup_retention_max_storage_locally' => 0, + 'database_backup_retention_amount_s3' => 0, + 'database_backup_retention_days_s3' => 0, + 'database_backup_retention_max_storage_s3' => 0, + 'dump_all' => false, + 'timeout' => 3600, + ]); + + Livewire::test(BackupEdit::class, [ + 'backup' => $backup, + 'availableS3Storages' => $this->team->s3s, + 'section' => 'retention', + ]) + ->assertSee('S3 Storage Retention') + ->set('databaseBackupRetentionAmountS3', 12) + ->set('databaseBackupRetentionDaysS3', 30) + ->set('databaseBackupRetentionMaxStorageS3', 4.5) + ->call('submit') + ->assertHasNoErrors(); + + expect($backup->refresh()->save_s3)->toBeFalsy() + ->and($backup->database_backup_retention_amount_s3)->toBe(12) + ->and($backup->database_backup_retention_days_s3)->toBe(30) + ->and($backup->database_backup_retention_max_storage_s3)->toBe(4.5); +}); + +it('splits standalone database backup settings and executions across dedicated urls', function () { + config(['cache.default' => 'array', 'app.maintenance.driver' => 'file']); + $backup = createBackupForEditValidationTest($this->team); + $database = $backup->database; + $parameters = [ + 'project_uuid' => $database->project()->uuid, + 'environment_uuid' => $database->environment->uuid, + 'database_uuid' => $database->uuid, + 'backup_uuid' => $backup->uuid, + ]; + $generalUrl = route('project.database.backup.execution', $parameters); + + $this->get($generalUrl) + ->assertOk() + ->assertSee('General') + ->assertSee('S3') + ->assertSee('Retention') + ->assertSee('Executions') + ->assertSee('Danger Zone') + ->assertSee('Frequency') + ->assertDontSee('S3 Enabled') + ->assertDontSee('Number of backups to keep') + ->assertDontSee('Cleanup Failed Backups') + ->assertDontSee('Delete Backups and Schedule'); + + $this->get($generalUrl.'/s3') + ->assertOk() + ->assertSeeText('No validated S3 available. Configure one here.') + ->assertDontSee('Disable Local Backup') + ->assertDontSee('Enable S3') + ->assertDontSee('Disable S3') + ->assertDontSee('S3 Storage Retention') + ->assertDontSee('Local Backup Retention') + ->assertDontSee('Frequency') + ->assertDontSee('Cleanup Failed Backups'); + + $s3View = file_get_contents(resource_path('views/livewire/project/database/backup-edit/s3.blade.php')); + expect(strpos($s3View, 'S3 Storage')) + ->toBeLessThan(strpos($s3View, 'label="Disable Local Backup"')); + + $this->get($generalUrl.'/retention') + ->assertOk() + ->assertSee('Local Backup Retention') + ->assertSee('S3 Storage Retention') + ->assertSee('Number of backups to keep') + ->assertDontSee('Frequency') + ->assertDontSee('Cleanup Failed Backups'); + + $this->get($generalUrl.'/executions') + ->assertOk() + ->assertSee('

Executions

', false) + ->assertDontSee('Executions assertSee('Cleanup Failed Backups') + ->assertDontSee('Frequency') + ->assertDontSee('Number of backups to keep'); + + $this->get($generalUrl.'/danger') + ->assertOk() + ->assertSee('Danger Zone') + ->assertSee('Delete Scheduled Backup') + ->assertSee('Delete Backups and Schedule') + ->assertDontSee('Frequency') + ->assertDontSee('Number of backups to keep') + ->assertDontSee('Cleanup Failed Backups'); +}); + +it('enables and disables a scheduled database backup from the title action', function () { + $backup = createBackupForEditValidationTest($this->team, [ + 'enabled' => false, + 'save_s3' => false, + ]); + + $component = Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) + ->assertSet('backupEnabled', false) + ->assertSee('Enable Backup') + ->call('toggleEnabled') + ->assertSet('backupEnabled', true) + ->assertSee('Disable Backup'); + + expect($backup->refresh()->enabled)->toBeTruthy(); + + $component->call('toggleEnabled')->assertSet('backupEnabled', false); + + expect($backup->refresh()->enabled)->toBeFalsy(); +}); + +it('redirects to executions after queuing a database backup with unusable S3 storage', function () { + Queue::fake(); + $s3Storage = createS3StorageForBackupEditValidationTest($this->team, 'Unavailable storage'); + $s3Storage->update(['is_usable' => false]); + $backup = createBackupForEditValidationTest($this->team, [ + 'enabled' => true, + 's3_storage_id' => $s3Storage->id, + 'database_backup_retention_amount_locally' => 7, + 'database_backup_retention_days_locally' => 0, + 'database_backup_retention_max_storage_locally' => 0, + 'database_backup_retention_amount_s3' => 7, + 'database_backup_retention_days_s3' => 0, + 'database_backup_retention_max_storage_s3' => 0, + 'dump_all' => false, + 'timeout' => 3600, + ]); + $database = $backup->database; + $parameters = [ + 'project_uuid' => $database->project()->uuid, + 'environment_uuid' => $database->environment->uuid, + 'database_uuid' => $database->uuid, + 'backup_uuid' => $backup->uuid, + ]; + + Livewire::test(BackupEdit::class, [ + 'backup' => $backup, + 'availableS3Storages' => $this->team->s3s, + ]) + ->call('backupNow') + ->assertRedirectToRoute('project.database.backup.executions', $parameters); + + Queue::assertPushed(DatabaseBackupJob::class); +}); + it('disables S3 backup when saved without a selected S3 storage', function () { $backup = createBackupForEditValidationTest($this->team); @@ -162,19 +354,52 @@ function createS3StorageForBackupEditValidationTest(Team|int $team, string $name 's3_storage_id' => null, ]); - Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) + Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3']) ->assertSee('First S3') ->assertSee('Second S3'); }); -it('shows disabled S3 storage dropdown when no storages are available', function () { +it('shows only an empty S3 state when no storages are available', function () { $backup = createBackupForEditValidationTest($this->team, [ 'save_s3' => false, 's3_storage_id' => null, ]); - Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) - ->assertSee('No S3 storage available'); + Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3']) + ->assertSeeHtml('

S3

') + ->assertSeeText('No validated S3 available. Configure one here.') + ->assertSeeHtml('href="'.route('storage.index').'"') + ->assertSeeHtml('>here') + ->assertDontSee('Save') + ->assertDontSee('Enable S3') + ->assertDontSee('Disable S3') + ->assertDontSee('S3 Storage') + ->assertDontSee('Disable Local Backup') + ->assertDontSee('No S3 storage available'); +}); + +it('allows S3 backups to be disabled when no usable storage remains', function () { + $backup = createBackupForEditValidationTest($this->team, [ + 'save_s3' => true, + 's3_storage_id' => null, + ]); + + $component = Livewire::test(BackupEdit::class, [ + 'backup' => $backup->fresh(), + 'availableS3Storages' => collect(), + 'section' => 's3', + ]) + ->assertSet('saveS3', true) + ->assertSeeText('No validated S3 available. Configure one here.') + ->assertDontSee('Disable S3') + ->call('toggleS3') + ->assertDispatched('success') + ->assertSet('saveS3', false) + ->assertDontSee('Enable S3'); + + expect($backup->refresh()->save_s3)->toBeFalsy() + ->and($backup->s3_storage_id)->toBeNull(); + }); it('shows when S3 backups are currently disabled', function () { @@ -184,7 +409,7 @@ function createS3StorageForBackupEditValidationTest(Team|int $team, string $name 's3_storage_id' => null, ]); - Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s]) + Livewire::test(BackupEdit::class, ['backup' => $backup->fresh(), 'availableS3Storages' => $this->team->s3s, 'section' => 's3']) ->assertSee('S3 Storage') ->assertSee('(currently disabled)'); }); @@ -205,3 +430,59 @@ function createS3StorageForBackupEditValidationTest(Team|int $team, string $name expect($backup->save_s3)->toBeFalsy(); expect($backup->s3_storage_id)->toBe($secondS3->id); }); + +it('subscribes to database status broadcasts so Backup Now can refresh without a full page reload', function () { + $component = app(BackupEdit::class); + $method = new ReflectionMethod($component, 'getListeners'); + $method->setAccessible(true); + $listeners = (array) $method->invoke($component); + + expect($listeners) + ->toHaveKey("echo-private:user.{$this->user->id},DatabaseStatusChanged") + ->toHaveKey("echo-private:team.{$this->team->id},ServiceChecked") + ->toHaveKey('databaseUpdated'); +}); + +it('shows Backup Now after refresh when the database becomes running', function () { + $backup = createBackupForEditValidationTest($this->team, [ + 'enabled' => true, + ]); + $database = $backup->database; + $database->update(['status' => 'exited:unhealthy']); + + $component = Livewire::test(BackupEdit::class, [ + 'backup' => $backup->fresh(), + 'availableS3Storages' => $this->team->s3s, + 'status' => 'exited:unhealthy', + ]) + ->assertDontSee('Backup Now') + ->assertSet('status', 'exited:unhealthy'); + + $database->update(['status' => 'running:healthy']); + + $component->call('refreshStatus') + ->assertSet('status', 'running:healthy') + ->assertSee('Backup Now'); +}); + +it('hides Backup Now after refresh when the database stops', function () { + $backup = createBackupForEditValidationTest($this->team, [ + 'enabled' => true, + ]); + $database = $backup->database; + $database->update(['status' => 'running:healthy']); + + $component = Livewire::test(BackupEdit::class, [ + 'backup' => $backup->fresh(), + 'availableS3Storages' => $this->team->s3s, + 'status' => 'running:healthy', + ]) + ->assertSee('Backup Now') + ->assertSet('status', 'running:healthy'); + + $database->update(['status' => 'exited:unhealthy']); + + $component->call('refreshStatus') + ->assertSet('status', 'exited:unhealthy') + ->assertDontSee('Backup Now'); +}); diff --git a/tests/Feature/BackupRetentionAndStaleDetectionTest.php b/tests/Feature/BackupRetentionAndStaleDetectionTest.php index cd4a76feb..97e45599f 100644 --- a/tests/Feature/BackupRetentionAndStaleDetectionTest.php +++ b/tests/Feature/BackupRetentionAndStaleDetectionTest.php @@ -2,12 +2,14 @@ use App\Jobs\CleanupInstanceStuffsJob; use App\Jobs\DatabaseBackupJob; +use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; use App\Models\ScheduledDatabaseBackupExecution; use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Queue\Middleware\WithoutOverlapping; use Illuminate\Support\Facades\Cache; +use Illuminate\Support\Facades\Storage; uses(RefreshDatabase::class); @@ -235,6 +237,55 @@ expect($successfulBackups->first()->uuid)->toBe('exec-uuid-1'); }); +test('database S3 retention remains best effort when deletion fails', function () { + $team = Team::factory()->create(); + $s3 = S3Storage::create([ + 'name' => 'Test S3', + 'region' => 'us-east-1', + 'key' => 'test-key', + 'secret' => 'test-secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $s3->id, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'database_backup_retention_amount_s3' => 1, + ]); + $newestExecution = ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'newest-s3-backup', + 'database_name' => 'test_db', + 'filename' => '/backup/newest.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + ]); + $oldExecution = ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'old-s3-backup', + 'database_name' => 'test_db', + 'filename' => '/backup/old.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + ]); + ScheduledDatabaseBackupExecution::whereKey($newestExecution->id)->update(['created_at' => now()]); + ScheduledDatabaseBackupExecution::whereKey($oldExecution->id)->update(['created_at' => now()->subDay()]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + + removeOldBackups($backup); + + expect($oldExecution->fresh()->s3_storage_deleted)->toBeFalse() + ->and($backup->executions()->count())->toBe(2); +}); + test('cleanup instance stuffs job throttles retention enforcement via cache', function () { Cache::forget('backup-retention-enforcement'); diff --git a/tests/Feature/BackupSearchTest.php b/tests/Feature/BackupSearchTest.php new file mode 100644 index 000000000..c9edadd97 --- /dev/null +++ b/tests/Feature/BackupSearchTest.php @@ -0,0 +1,178 @@ + 'file']); + InstanceSettings::forceCreate(['id' => 0]); + + $this->team = Team::factory()->create(); + $this->user = User::factory()->create(); + $this->user->teams()->attach($this->team, ['role' => 'owner']); + $this->actingAs($this->user); + session(['currentTeam' => $this->team]); +}); + +it('renders frontend-only application backup search data for volume names and frequencies', function () { + $application = createBackupSearchApplication($this->team); + $dailyVolume = createBackupSearchVolume($application, 'app-data'); + $weeklyVolume = createBackupSearchVolume($application, 'Cache-Data'); + + $dailyVolume->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + ]); + $weeklyVolume->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => '0 4 * * 0', + ]); + + $parameters = [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + ]; + + $this->get(route('project.application.backup.index', [...$parameters, 'search' => 'cache'])) + ->assertOk() + ->assertSee('Cache-Data') + ->assertSee('Volume: app-data') + ->assertSee("search: 'cache'", false) + ->assertSee('x-model="search"', false) + ->assertSee('x-show=', false) + ->assertSee('No scheduled backups match your search.'); +}); + +it('renders frontend-only database backup search data for database names and frequencies', function () { + $server = Server::factory()->create(['team_id' => $this->team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $database = StandalonePostgresql::create([ + 'name' => 'Orders-Primary', + 'image' => 'postgres:16-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $storage = S3Storage::create([ + 'name' => 'Archive-Bucket', + 'region' => 'us-east-1', + 'key' => 'test-key', + 'secret' => 'test-secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://s3.example.com', + 'is_usable' => true, + 'team_id' => $this->team->id, + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $storage->id, + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => '0 3 * * 1', + ]); + + $parameters = [ + 'project_uuid' => $project->uuid, + 'environment_uuid' => $environment->uuid, + 'database_uuid' => $database->uuid, + ]; + + $this->get(route('project.database.backup.index', [...$parameters, 'search' => '0 3'])) + ->assertOk() + ->assertSee('

0 3 * * 1

', false) + ->assertSee('

daily

', false) + ->assertSee('S3: Archive-Bucket') + ->assertSee('archive-bucket', false) + ->assertSee('backup.s3_storage.includes(query)', false) + ->assertSee('Search by database name, frequency, or S3 storage...') + ->assertSee('x-model="search"', false) + ->assertSee('x-show=', false) + ->assertSee('No scheduled backups match your search.'); +}); + +it('renders unavailable S3 storage only for S3-enabled database backups', function () { + $server = Server::factory()->create(['team_id' => $this->team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $this->team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $database = StandalonePostgresql::create([ + 'name' => 'Orders-Primary', + 'image' => 'postgres:16-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => null, + ]); + $database->scheduledBackups()->create([ + 'team_id' => $this->team->id, + 'frequency' => 'weekly', + 'save_s3' => false, + 's3_storage_id' => null, + ]); + + $parameters = [ + 'project_uuid' => $project->uuid, + 'environment_uuid' => $environment->uuid, + 'database_uuid' => $database->uuid, + ]; + + $response = $this->get(route('project.database.backup.index', $parameters)) + ->assertOk() + ->assertSee('S3: Storage unavailable'); + + expect(substr_count($response->getContent(), 'S3:'))->toBe(1); +}); + +function createBackupSearchApplication(Team $team): Application +{ + $server = Server::factory()->create(['team_id' => $team->id]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + + return $application; +} + +function createBackupSearchVolume(Application $application, string $name): LocalPersistentVolume +{ + return LocalPersistentVolume::create([ + 'name' => $name, + 'mount_path' => '/'.str($name)->lower(), + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); +} diff --git a/tests/Feature/CreateScheduledBackupValidationTest.php b/tests/Feature/CreateScheduledBackupValidationTest.php index 56a4cf479..3a5ea6a37 100644 --- a/tests/Feature/CreateScheduledBackupValidationTest.php +++ b/tests/Feature/CreateScheduledBackupValidationTest.php @@ -3,9 +3,10 @@ use App\Livewire\Project\Database\CreateScheduledBackup; use App\Models\Environment; use App\Models\Project; -use App\Models\S3Storage; use App\Models\ScheduledDatabaseBackup; use App\Models\Server; +use App\Models\Service; +use App\Models\ServiceDatabase; use App\Models\StandaloneClickhouse; use App\Models\StandaloneDocker; use App\Models\StandalonePostgresql; @@ -17,126 +18,80 @@ uses(RefreshDatabase::class); -function createDatabaseForScheduledBackupTest(Team $team): StandalonePostgresql -{ - $server = Server::factory()->create(['team_id' => $team->id]); - $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); - $project = Project::factory()->create(['team_id' => $team->id]); - $environment = Environment::factory()->create(['project_id' => $project->id]); - - return StandalonePostgresql::create([ - 'name' => 'pg-scheduled-backup-validation', - 'image' => 'postgres:16-alpine', - 'postgres_user' => 'postgres', - 'postgres_password' => 'password', - 'postgres_db' => 'postgres', - 'environment_id' => $environment->id, - 'destination_id' => $destination->id, - 'destination_type' => $destination->getMorphClass(), - ]); -} - -function createS3StorageForTeam(Team $team, string $name = 'Test S3'): S3Storage -{ - return S3Storage::create([ - 'name' => $name, - 'region' => 'us-east-1', - 'key' => 'test-key', - 'secret' => 'test-secret', - 'bucket' => 'test-bucket', - 'endpoint' => 'https://s3.example.com', - 'is_usable' => true, - 'team_id' => $team->id, - ]); -} - beforeEach(function () { $this->team = Team::factory()->create(); $this->user = User::factory()->create(); $this->user->teams()->attach($this->team, ['role' => 'owner']); $this->actingAs($this->user); session(['currentTeam' => $this->team]); + + $this->server = Server::factory()->create(['team_id' => $this->team->id]); + $this->destination = StandaloneDocker::where('server_id', $this->server->id)->firstOrFail(); + $this->project = Project::factory()->create(['team_id' => $this->team->id]); + $this->environment = Environment::factory()->create(['project_id' => $this->project->id]); }); -it('rejects enabling S3 backup without a selected S3 storage', function () { - $database = createDatabaseForScheduledBackupTest($this->team); - - Livewire::test(CreateScheduledBackup::class, ['database' => $database]) - ->set('frequency', '0 0 * * *') - ->set('saveToS3', true) - ->set('s3StorageId', null) - ->call('submit') - ->assertDispatched('error'); - - expect(ScheduledDatabaseBackup::count())->toBe(0); -}); - -it('rejects an S3 storage not owned by the current team', function () { - $database = createDatabaseForScheduledBackupTest($this->team); - - $foreignS3 = createS3StorageForTeam(Team::factory()->create(), 'Foreign S3'); - - Livewire::test(CreateScheduledBackup::class, ['database' => $database]) - ->set('frequency', '0 0 * * *') - ->set('saveToS3', true) - ->set('s3StorageId', $foreignS3->id) - ->call('submit') - ->assertDispatched('error'); - - expect(ScheduledDatabaseBackup::count())->toBe(0); -}); - -it('rejects an S3 storage that is reassigned after the component is mounted', function () { - $database = createDatabaseForScheduledBackupTest($this->team); - $s3 = createS3StorageForTeam($this->team); +it('creates a standalone database backup without S3 and opens its configuration', function () { + $database = StandalonePostgresql::create([ + 'name' => 'postgres', + 'image' => 'postgres:16-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); $component = Livewire::test(CreateScheduledBackup::class, ['database' => $database]) - ->set('frequency', '0 0 * * *') - ->set('saveToS3', true) - ->set('s3StorageId', $s3->id); + ->assertDontSee('Save to S3') + ->set('frequency', 'daily') + ->call('submit'); - $s3->update(['team_id' => Team::factory()->create()->id]); + $backup = ScheduledDatabaseBackup::query()->sole(); - $component - ->call('submit') - ->assertDispatched('error'); + $component->assertRedirectToRoute('project.database.backup.execution', [ + 'project_uuid' => $this->project->uuid, + 'environment_uuid' => $this->environment->uuid, + 'database_uuid' => $database->uuid, + 'backup_uuid' => $backup->uuid, + ]); - expect(ScheduledDatabaseBackup::count())->toBe(0); + expect($backup->save_s3)->toBeFalsy() + ->and($backup->s3_storage_id)->toBeNull(); }); -it('rejects an S3 storage that becomes unusable after the component is mounted', function () { - $database = createDatabaseForScheduledBackupTest($this->team); - $s3 = createS3StorageForTeam($this->team); +it('creates a service database backup without S3 and opens its configuration', function () { + $service = Service::factory()->create([ + 'server_id' => $this->server->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + 'environment_id' => $this->environment->id, + ]); + $database = ServiceDatabase::create([ + 'service_id' => $service->id, + 'name' => 'postgres', + 'image' => 'postgres:16-alpine', + 'custom_type' => 'postgresql', + ]); $component = Livewire::test(CreateScheduledBackup::class, ['database' => $database]) - ->set('frequency', '0 0 * * *') - ->set('saveToS3', true) - ->set('s3StorageId', $s3->id); + ->assertDontSee('Save to S3') + ->set('frequency', 'daily') + ->call('submit'); - $s3->update(['is_usable' => false]); + $backup = ScheduledDatabaseBackup::query()->sole(); - $component - ->call('submit') - ->assertDispatched('error'); + $component->assertRedirectToRoute('project.service.database.backup.show', [ + 'project_uuid' => $this->project->uuid, + 'environment_uuid' => $this->environment->uuid, + 'service_uuid' => $service->uuid, + 'stack_service_uuid' => $database->uuid, + 'backup_uuid' => $backup->uuid, + ]); - expect(ScheduledDatabaseBackup::count())->toBe(0); -}); - -it('creates a scheduled backup with a valid team-owned S3 storage', function () { - $database = createDatabaseForScheduledBackupTest($this->team); - $s3 = createS3StorageForTeam($this->team); - - Livewire::test(CreateScheduledBackup::class, ['database' => $database]) - ->set('frequency', '0 0 * * *') - ->set('saveToS3', true) - ->set('s3StorageId', $s3->id) - ->call('submit') - ->assertDispatched('refreshScheduledBackups'); - - $backup = ScheduledDatabaseBackup::first(); - expect($backup)->not->toBeNull(); - expect($backup->save_s3)->toBeTruthy(); - expect($backup->s3_storage_id)->toBe($s3->id); + expect($backup->save_s3)->toBeFalsy() + ->and($backup->s3_storage_id)->toBeNull(); }); it('creates a clickhouse backup for its configured database', function () { @@ -154,13 +109,19 @@ function createS3StorageForTeam(Team $team, string $name = 'Test S3'): S3Storage 'destination_type' => $destination->getMorphClass(), ]); - Livewire::test(CreateScheduledBackup::class, ['database' => $database]) + $component = Livewire::test(CreateScheduledBackup::class, ['database' => $database]) ->set('frequency', 'daily') - ->call('submit') - ->assertDispatched('refreshScheduledBackups'); + ->call('submit'); $backup = ScheduledDatabaseBackup::firstOrFail(); + $component->assertRedirectToRoute('project.database.backup.execution', [ + 'project_uuid' => $project->uuid, + 'environment_uuid' => $environment->uuid, + 'database_uuid' => $database->uuid, + 'backup_uuid' => $backup->uuid, + ]); + expect($backup->database_type)->toBe(StandaloneClickhouse::class) ->and($backup->databases_to_backup)->toBe('analytics'); }); diff --git a/tests/Feature/ExecutionViewStructureTest.php b/tests/Feature/ExecutionViewStructureTest.php new file mode 100644 index 000000000..ce613608b --- /dev/null +++ b/tests/Feature/ExecutionViewStructureTest.php @@ -0,0 +1,26 @@ +toContain('
') + ->toContain('

Executions

') + ->not->toContain('Executions toContain('class="flex flex-col gap-4 pt-2"'); +}); + +it('renders scheduled task executions as selectable status cards with expandable logs', function () { + $view = file_get_contents(resource_path('views/livewire/project/shared/scheduled-task/executions.blade.php')); + + expect($view) + ->toContain('
toContain('toContain('border-l-2 transition-colors p-4 cursor-pointer') + ->toContain("'success' => 'Success'") + ->toContain("'running' => 'In Progress'") + ->toContain("'failed' => 'Failed'") + ->toContain("data_get(\$execution, 'id') == \$selectedKey") + ->toContain('max-h-[600px] overflow-y-auto') + ->toContain('No executions found.'); +}); diff --git a/tests/Feature/FileStorageMountPathTest.php b/tests/Feature/FileStorageMountPathTest.php index 2a41b9075..aa277d37e 100644 --- a/tests/Feature/FileStorageMountPathTest.php +++ b/tests/Feature/FileStorageMountPathTest.php @@ -1,14 +1,19 @@ set('file_storage_path', '/etc/nginx/nginx.conf') ->set('file_storage_content', 'server {}') ->call('submitFileStorage') - ->assertDispatched('success'); + ->assertDispatched('success') + ->assertDispatched('configurationChanged'); $volume = LocalFileVolume::query()->sole(); @@ -109,7 +115,8 @@ ->set('host_file_storage_source', '/etc/nginx/nginx.conf') ->set('host_file_storage_destination', '/etc/nginx/nginx.conf') ->call('submitHostFileStorage') - ->assertDispatched('success'); + ->assertDispatched('success') + ->assertDispatched('configurationChanged'); $volume = LocalFileVolume::query()->sole(); @@ -121,3 +128,80 @@ Bus::assertNotDispatched(ServerStorageSaveJob::class); }); + +test('livewire volume storage refreshes the storage list and configuration warning', function () { + Livewire::test(Storage::class, ['resource' => $this->application]) + ->set('name', 'data') + ->set('mount_path', '/app/data') + ->call('submitPersistentVolume') + ->assertDispatched('success') + ->assertDispatched('refreshStorages') + ->assertDispatched('configurationChanged'); +}); + +test('volume storage list shows volumes added after it was mounted', function () { + $firstVolume = LocalPersistentVolume::create([ + 'name' => $this->application->uuid.'-first', + 'mount_path' => '/app/first', + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]); + + $storageList = Livewire::test(All::class, ['resource' => $this->application]) + ->assertSee($firstVolume->name); + + $secondVolume = LocalPersistentVolume::create([ + 'name' => $this->application->uuid.'-second', + 'mount_path' => '/app/second', + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]); + + $storageList + ->assertDontSee($secondVolume->name) + ->dispatch('refreshStorages') + ->assertSee($secondVolume->name); +}); + +test('deleting a file mount refreshes the configuration warning', function () { + $file = LocalFileVolume::create([ + 'fs_path' => '/etc/nginx/nginx.conf', + 'mount_path' => '/etc/nginx/nginx.conf', + 'is_host_file' => true, + 'is_based_on_git' => false, + 'is_preview_suffix_enabled' => true, + 'resource_id' => $this->application->id, + 'resource_type' => $this->application->getMorphClass(), + ]); + + Livewire::test(FileStorage::class, ['fileStorage' => $file]) + ->call('delete', 'password') + ->assertDispatched('configurationChanged'); + + expect($file->fresh())->toBeNull(); +}); + +test('deleting a volume mount refreshes the configuration warning', function () { + $database = StandalonePostgresql::create([ + 'name' => 'test-postgres', + 'image' => 'postgres:15-alpine', + 'postgres_user' => 'postgres', + 'postgres_password' => 'password', + 'postgres_db' => 'postgres', + 'environment_id' => $this->environment->id, + 'destination_id' => $this->destination->id, + 'destination_type' => $this->destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => $database->uuid.'-data', + 'mount_path' => '/var/lib/postgresql/data', + 'resource_id' => $database->id, + 'resource_type' => $database->getMorphClass(), + ]); + + Livewire::test(Show::class, ['storage' => $volume, 'resource' => $database]) + ->call('delete', 'password') + ->assertDispatched('configurationChanged'); + + expect($volume->fresh())->toBeNull(); +}); diff --git a/tests/Feature/Jobs/DatabaseBackupJobTest.php b/tests/Feature/Jobs/DatabaseBackupJobTest.php index fd55aabd2..c21e1be22 100644 --- a/tests/Feature/Jobs/DatabaseBackupJobTest.php +++ b/tests/Feature/Jobs/DatabaseBackupJobTest.php @@ -7,6 +7,7 @@ use App\Models\Team; use Illuminate\Foundation\Testing\RefreshDatabase; use Illuminate\Support\Facades\Schema; +use Illuminate\Support\Facades\Storage; uses(RefreshDatabase::class); @@ -238,6 +239,57 @@ expect($log->message)->toContain('Some real failure'); }); +test('retention cleanup failure does not fail a successful database backup', function () { + $team = Team::factory()->create(); + $s3 = S3Storage::create([ + 'name' => 'Test S3', + 'region' => 'us-east-1', + 'key' => 'test-key', + 'secret' => 'test-secret', + 'bucket' => 'test-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $backup = ScheduledDatabaseBackup::create([ + 'frequency' => '0 0 * * *', + 'save_s3' => true, + 'disable_local_backup' => true, + 's3_storage_id' => $s3->id, + 'database_type' => 'App\Models\StandalonePostgresql', + 'database_id' => 1, + 'team_id' => $team->id, + 'database_backup_retention_amount_s3' => 1, + ]); + $oldExecution = ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'old-backup', + 'database_name' => 'database', + 'filename' => '/backup/old.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + 'local_storage_deleted' => true, + 'created_at' => now()->subDay(), + ]); + ScheduledDatabaseBackupExecution::create([ + 'uuid' => 'new-backup', + 'database_name' => 'database', + 'filename' => '/backup/new.dmp', + 'scheduled_database_backup_id' => $backup->id, + 'status' => 'success', + 's3_uploaded' => true, + 'local_storage_deleted' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/backup/old.dmp'])->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + $job = new DatabaseBackupJob($backup); + + expect(fn () => (new ReflectionClass($job))->getMethod('removeExpiredBackups')->invoke($job)) + ->not->toThrow(Throwable::class); + + expect($oldExecution->fresh()->s3_storage_deleted)->toBeFalse(); +}); + test('s3 storage has scheduled backups relationship', function () { $team = Team::factory()->create(); diff --git a/tests/Feature/Livewire/ConfigurationCheckerTest.php b/tests/Feature/Livewire/ConfigurationCheckerTest.php index d9e6729c8..d6c3231df 100644 --- a/tests/Feature/Livewire/ConfigurationCheckerTest.php +++ b/tests/Feature/Livewire/ConfigurationCheckerTest.php @@ -5,6 +5,7 @@ use App\Models\ApplicationDeploymentQueue; use App\Models\Environment; use App\Models\EnvironmentVariable; +use App\Models\LocalFileVolume; use App\Models\Project; use App\Models\Team; use App\Models\User; @@ -87,6 +88,30 @@ function markConfigurationCheckerApplicationDeployed(Application $application): ->assertSee('Build command'); }); +it('shows an unapplied configuration warning after a directory mount is added', function () { + $application = configurationCheckerApplication($this->environment); + markConfigurationCheckerApplicationDeployed($application); + + $component = Livewire::test(ConfigurationChecker::class, ['resource' => $application->refresh()]) + ->assertSet('isConfigurationChanged', false); + + LocalFileVolume::withoutEvents(fn () => LocalFileVolume::forceCreate([ + 'uuid' => (string) Str::uuid(), + 'fs_path' => application_configuration_dir().'/'.$application->uuid.'/data', + 'mount_path' => '/app/data', + 'is_directory' => true, + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ])); + + $component + ->dispatch('configurationChanged') + ->assertSet('isConfigurationChanged', true) + ->assertSee('The latest configuration has not been applied') + ->assertSee('Directory mount') + ->assertSee('Please redeploy to apply the new configuration.'); +}); + it('refreshes stale modal configuration diff before opening changes', function () { $application = configurationCheckerApplication($this->environment); markConfigurationCheckerApplicationDeployed($application); diff --git a/tests/Feature/MobileResourceMenuTest.php b/tests/Feature/MobileResourceMenuTest.php index 7f4d6fe9d..47ef03510 100644 --- a/tests/Feature/MobileResourceMenuTest.php +++ b/tests/Feature/MobileResourceMenuTest.php @@ -131,6 +131,19 @@ function mobileActionsAreBeforeSelect(string $heading, string $actionsId, string return $actionsPosition !== false && $selectPosition !== false && $actionsPosition < $selectPosition; } +it('places database backups immediately after configuration in navigation menus', function () { + $databaseHeading = file_get_contents(resource_path('views/livewire/project/database/heading.blade.php')); + $mobileMenuItems = str($databaseHeading)->between('$databasePageItems = [', '$databaseConfigurationItems = [')->toString(); + $desktopMenuItems = str($databaseHeading)->between('class="scrollbar hidden', '')->toString(); + + foreach ([$mobileMenuItems, $desktopMenuItems] as $menuItems) { + expect(strpos($menuItems, 'Configuration')) + ->toBeLessThan(strpos($menuItems, 'Backups')) + ->and(strpos($menuItems, 'Backups'))->toBeLessThan(strpos($menuItems, 'Logs')) + ->and(strpos($menuItems, 'Logs'))->toBeLessThan(strpos($menuItems, 'Terminal')); + } +}); + it('keeps configuration sidebars hidden until desktop breakpoint', function () { expect(file_get_contents(resource_path('views/livewire/project/database/configuration.blade.php'))) ->toContain('sub-menu-wrapper hidden md:flex'); diff --git a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php b/tests/Feature/MutableLivewireComponentsAuthorizationTest.php index 962628298..f2890114c 100644 --- a/tests/Feature/MutableLivewireComponentsAuthorizationTest.php +++ b/tests/Feature/MutableLivewireComponentsAuthorizationTest.php @@ -28,3 +28,35 @@ ['AuthorizesRequests', "authorize('view'", "authorize('canAccessTerminal'"], ], ]); + +it('authorizes every volume backup form control', function (string $path, array $controlPatterns) { + $source = file_get_contents(base_path($path)); + + foreach ($controlPatterns as $controlPattern) { + expect($source)->toMatch($controlPattern); + } +})->with([ + 'retention controls' => [ + 'resources/views/livewire/project/shared/storages/volume-backups/retention.blade.php', + [ + '/]*type="submit")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*>Save<\/x-forms\.button>/s', + '/]*id="retentionAmountLocally")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + '/]*id="retentionDaysLocally")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + '/]*id="retentionMaxStorageLocally")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + '/]*id="retentionAmountS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + '/]*id="retentionDaysS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + '/]*id="retentionMaxStorageS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + ], + ], + 'S3 controls' => [ + 'resources/views/livewire/project/shared/storages/volume-backups/s3.blade.php', + [ + '/]*type="submit")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*>Save<\/x-forms\.button>/s', + '/]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*>Enable S3<\/x-forms\.button>/s', + '/]*wire:click="toggleS3")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*>Disable S3<\/x-forms\.button>/s', + '/]*id="s3StorageId")(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*>/s', + '/]*id="disableLocalBackup")(?=[^>]*instantSave)(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + '/]*id="disableLocalBackup")(?=[^>]*disabled)(?=[^>]*canGate="update")(?=[^>]*:canResource="\$backup")[^>]*\/\>/s', + ], + ], +]); diff --git a/tests/Feature/ServiceResourceRoutingTest.php b/tests/Feature/ServiceResourceRoutingTest.php index f27340330..3a4549046 100644 --- a/tests/Feature/ServiceResourceRoutingTest.php +++ b/tests/Feature/ServiceResourceRoutingTest.php @@ -5,6 +5,7 @@ use App\Models\Environment; use App\Models\InstanceSettings; use App\Models\Project; +use App\Models\ScheduledDatabaseBackup; use App\Models\Server; use App\Models\Service; use App\Models\ServiceApplication; @@ -139,3 +140,65 @@ ]) ->assertOk(); }); + +test('service database backup schedules use dedicated general retention and executions urls', function () { + $backup = ScheduledDatabaseBackup::create([ + 'team_id' => $this->teamA->id, + 'frequency' => 'daily', + 'database_id' => $this->ownServiceDatabase->id, + 'database_type' => $this->ownServiceDatabase->getMorphClass(), + 'save_s3' => true, + ]); + $listUrl = route('project.service.database.backups', [ + 'project_uuid' => $this->projectA->uuid, + 'environment_uuid' => $this->environmentA->uuid, + 'service_uuid' => $this->ownService->uuid, + 'stack_service_uuid' => $this->ownServiceDatabase->uuid, + ]); + $generalUrl = $listUrl.'/'.$backup->uuid; + + $this->get($listUrl) + ->assertOk() + ->assertSee('href="'.$generalUrl.'"', false); + + $this->get($generalUrl) + ->assertOk() + ->assertSee('Frequency') + ->assertDontSee('S3 Enabled') + ->assertDontSee('Number of backups to keep') + ->assertDontSee('Cleanup Failed Backups') + ->assertDontSee('Delete Backups and Schedule'); + + $this->get($generalUrl.'/s3') + ->assertOk() + ->assertSee('S3 Storage') + ->assertDontSee('S3 Storage Retention') + ->assertDontSee('Local Backup Retention') + ->assertDontSee('Frequency') + ->assertDontSee('Cleanup Failed Backups'); + + $this->get($generalUrl.'/retention') + ->assertOk() + ->assertSee('Local Backup Retention') + ->assertSee('S3 Storage Retention') + ->assertSee('Number of backups to keep') + ->assertDontSee('Frequency') + ->assertDontSee('Cleanup Failed Backups'); + + $this->get($generalUrl.'/executions') + ->assertOk() + ->assertSee('

Executions

', false) + ->assertDontSee('Executions assertSee('Cleanup Failed Backups') + ->assertDontSee('Frequency') + ->assertDontSee('Number of backups to keep'); + + $this->get($generalUrl.'/danger') + ->assertOk() + ->assertSee('Danger Zone') + ->assertSee('Delete Scheduled Backup') + ->assertSee('Delete Backups and Schedule') + ->assertDontSee('Frequency') + ->assertDontSee('Number of backups to keep') + ->assertDontSee('Cleanup Failed Backups'); +}); diff --git a/tests/Feature/StatusBadgeComponentsTest.php b/tests/Feature/StatusBadgeComponentsTest.php index 8f19c88f7..7506c97dc 100644 --- a/tests/Feature/StatusBadgeComponentsTest.php +++ b/tests/Feature/StatusBadgeComponentsTest.php @@ -59,23 +59,27 @@ public function __construct(public string $status) {} ->not->toContain(' 'running:unknown', + 'status' => $status, ])->render(); $runningStatus = file_get_contents(resource_path('views/components/status/running.blade.php')); expect($html) - ->toContain('No health check') + ->toContain($expectedText) + ->toContain('class="flex items-center gap-1 leading-none"') ->toContain('inline-flex h-5 max-w-full items-center gap-1 rounded-sm border') ->not->toContain('toContain('toContain('class="flex items-center gap-1"') + ->toContain('class="flex items-center gap-1 leading-none"') ->not->toContain('class="px-2"') ->not->toContain('viewBox="0 0 256 256"'); -}); +})->with([ + 'unknown health' => ['running:unknown', 'No health check'], + 'unhealthy' => ['running:unhealthy', 'Unhealthy'], +]); it('renders restart counts as warning badges', function () { $resource = new class diff --git a/tests/Feature/VolumeBackupTest.php b/tests/Feature/VolumeBackupTest.php new file mode 100644 index 000000000..8a8058ff4 --- /dev/null +++ b/tests/Feature/VolumeBackupTest.php @@ -0,0 +1,2210 @@ +toBeTrue() + ->and(class_exists(ScheduledVolumeBackupExecution::class))->toBeTrue() + ->and(class_exists(VolumeBackupJob::class))->toBeTrue() + ->and(class_exists(VolumeBackups::class))->toBeTrue() + ->and(method_exists(LocalPersistentVolume::class, 'scheduledBackups'))->toBeTrue() + ->and(method_exists(LocalFileVolume::class, 'scheduledBackups'))->toBeTrue(); +}); + +it('keeps the volume backup script inside the Livewire root element', function () { + $view = file_get_contents(resource_path('views/livewire/project/shared/storages/volume-backups.blade.php')); + + expect(strrpos($view, '@endscript'))->toBeLessThan(strrpos($view, '
')); +}); + +it('targets named volumes and application directory mounts through one backup relation', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $directory = LocalFileVolume::unguarded(fn () => LocalFileVolume::withoutEvents(fn () => LocalFileVolume::create([ + 'uuid' => new_public_id(), + 'fs_path' => './uploads', + 'mount_path' => '/app/uploads', + 'is_directory' => true, + 'is_based_on_git' => false, + 'is_preview_suffix_enabled' => true, + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]))); + + $volumeBackup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $directoryBackup = $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'weekly', + ]); + + expect($volumeBackup->backupable->is($volume))->toBeTrue() + ->and($volumeBackup->targetType())->toBe('Volume') + ->and($volumeBackup->targetName())->toBe('app-data') + ->and($volumeBackup->sourcePath())->toBe('app-data') + ->and($directoryBackup->backupable->is($directory))->toBeTrue() + ->and($directoryBackup->targetType())->toBe('Directory') + ->and($directoryBackup->targetName())->toBe('./uploads') + ->and($directoryBackup->sourcePath())->toBe($application->workdir().'/uploads') + ->and($directoryBackup->server()?->id)->toBe($application->destination->server->id); +}); + +it('provides application backup index and detail routes', function () { + expect(Route::has('project.application.backup.index'))->toBeTrue() + ->and(Route::has('project.application.backup.show'))->toBeTrue() + ->and(Route::has('download.volume-backup'))->toBeTrue(); +}); + +it('creates a scheduled backup with a preselected volume from the shared modal', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $component = Livewire::test(CreateScheduledVolumeBackup::class, [ + 'application' => $application, + 'selectedTargetKey' => 'volume:'.$volume->id, + ]) + ->assertSet('targetKey', 'volume:'.$volume->id) + ->assertSee($volume->name) + ->assertDontSee('Save to S3') + ->set('frequency', 'daily') + ->call('submit') + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + + $component->assertRedirectToRoute('project.application.backup.show', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'backup_uuid' => $backup->uuid, + ]); + + expect($backup->backupable->is($volume))->toBeTrue() + ->and($backup->frequency)->toBe('daily') + ->and($backup->enabled)->toBeTrue() + ->and($backup->save_s3)->toBeFalse() + ->and($backup->s3_storage_id)->toBeNull(); +}); + +it('handles scheduled backup persistence failures', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $shouldFail = true; + + ScheduledVolumeBackup::creating(function () use (&$shouldFail): void { + if ($shouldFail) { + $shouldFail = false; + + throw new RuntimeException('Scheduled backup persistence failed.'); + } + }); + + Livewire::test(CreateScheduledVolumeBackup::class, [ + 'application' => $application, + 'selectedTargetKey' => 'volume:'.$volume->id, + ]) + ->set('frequency', 'daily') + ->call('submit') + ->assertDispatched('error', 'Scheduled backup persistence failed.'); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('respects the instance wire navigate setting after creating a scheduled backup', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create([ + 'id' => 0, + 'is_wire_navigate_enabled' => false, + ])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $component = Livewire::test(CreateScheduledVolumeBackup::class, [ + 'application' => $application, + 'selectedTargetKey' => 'volume:'.$volume->id, + ]) + ->set('frequency', 'daily') + ->call('submit'); + + $backup = ScheduledVolumeBackup::query()->sole(); + + $component->assertRedirectToRoute('project.application.backup.show', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'backup_uuid' => $backup->uuid, + ]); + + expect($component->effects)->not->toHaveKey('redirectUsingNavigate'); +}); + +it('creates a scheduled backup for a preselected application directory', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application] = createVolumeBackupApplication($team); + $directory = createApplicationBackupDirectory($application); + + Livewire::test(CreateScheduledVolumeBackup::class, [ + 'application' => $application, + 'selectedTargetKey' => 'directory:'.$directory->id, + ]) + ->assertSet('targetKey', 'directory:'.$directory->id) + ->assertSee('Directory: '.$directory->fs_path) + ->set('frequency', 'daily') + ->call('submit') + ->assertDispatched('success'); + + expect(ScheduledVolumeBackup::query()->sole()->backupable->is($directory))->toBeTrue(); +}); + +it('rejects files and directory mounts owned by another application as backup targets', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application] = createVolumeBackupApplication($team); + $otherApplication = Application::factory()->create([ + 'environment_id' => $application->environment_id, + 'destination_id' => $application->destination_id, + 'destination_type' => $application->destination_type, + ]); + $foreignDirectory = createApplicationBackupDirectory($otherApplication); + $file = createApplicationBackupDirectory($application, './config.json'); + $file->update(['is_directory' => false]); + + Livewire::test(CreateScheduledVolumeBackup::class, ['application' => $application]) + ->set('targetKey', 'directory:'.$file->id) + ->call('submit') + ->assertHasErrors('targetKey') + ->set('targetKey', 'directory:'.$foreignDirectory->id) + ->call('submit') + ->assertHasErrors('targetKey'); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('shows volume backups on the application backups pages', function () { + config(['cache.default' => 'array', 'app.maintenance.driver' => 'file']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $parameters = [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + ]; + + $this->get(route('project.application.backup.index', $parameters)) + ->assertOk() + ->assertSee('Scheduled Backups') + ->assertSee($volume->name); + + $this->get(route('project.application.backup.show', [...$parameters, 'backup_uuid' => $backup->uuid])) + ->assertOk() + ->assertSee('

Backups

', false) + ->assertSee($volume->name); +}); + +it('shows directory backups on the application backup index and detail pages', function () { + config(['cache.default' => 'array', 'app.maintenance.driver' => 'file']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application] = createVolumeBackupApplication($team); + $directory = createApplicationBackupDirectory($application); + $backup = $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $parameters = [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + ]; + + $this->get(route('project.application.backup.index', $parameters)) + ->assertOk() + ->assertSee('Directory: '.$directory->fs_path); + + $this->get(route('project.application.backup.show', [...$parameters, 'backup_uuid' => $backup->uuid])) + ->assertOk() + ->assertSee('

Backups

', false) + ->assertSee($directory->fs_path); +}); + +it('splits scheduled backup settings and executions across dedicated urls', function () { + config(['cache.default' => 'array', 'app.maintenance.driver' => 'file']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'finished_at' => now(), + ]); + $parameters = [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'backup_uuid' => $backup->uuid, + ]; + $generalUrl = route('project.application.backup.show', $parameters); + + $this->get($generalUrl) + ->assertOk() + ->assertSee('General') + ->assertSee('S3') + ->assertSee('Retention') + ->assertSee('Executions') + ->assertSee('Danger Zone') + ->assertSee('Stop containers while creating the archive') + ->assertDontSee('S3 Enabled') + ->assertDontSee('Number of backups to keep') + ->assertDontSee('Backup Availability:') + ->assertDontSee('Delete Backups and Schedule'); + + $this->get($generalUrl.'/s3') + ->assertOk() + ->assertSeeText('No validated S3 available. Configure one here.') + ->assertDontSee('Disable Local Backup') + ->assertDontSee('Enable S3') + ->assertDontSee('Disable S3') + ->assertDontSee('S3 Storage Retention') + ->assertDontSee('Local Backup Retention') + ->assertDontSee('Frequency') + ->assertDontSee('Backup Availability:'); + + $s3View = file_get_contents(resource_path('views/livewire/project/shared/storages/volume-backups/s3.blade.php')); + expect(strpos($s3View, 'S3 Storage')) + ->toBeLessThan(strpos($s3View, 'label="Disable Local Backup"')); + + $this->get($generalUrl.'/retention') + ->assertOk() + ->assertSee('Local Backup Retention') + ->assertSee('S3 Storage Retention') + ->assertSee('Number of backups to keep') + ->assertDontSee('Stop containers while creating the archive') + ->assertDontSee('Backup Availability:'); + + $this->get($generalUrl.'/executions') + ->assertOk() + ->assertSee('Backup Availability:') + ->assertDontSee('Stop containers while creating the archive') + ->assertDontSee('Number of backups to keep'); + + $this->get($generalUrl.'/danger') + ->assertOk() + ->assertSee('Danger Zone') + ->assertSee('Delete Scheduled Backup') + ->assertSee('Delete Backups and Schedule') + ->assertDontSee('Stop containers while creating the archive') + ->assertDontSee('Number of backups to keep') + ->assertDontSee('Backup Availability:'); +}); + +it('shows the configure backup modal trigger inside the volume card instead of inline backup settings', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $component = Livewire::test(Show::class, [ + 'storage' => $volume, + 'resource' => $application, + ]) + ->set('isReadOnly', true) + ->assertSee('Configure Backup') + ->assertDontSee('Backups made while the application is writing'); + + $html = $component->html(); + + expect(strpos($html, 'Configure Backup')) + ->toBeGreaterThan(strpos($html, 'toBeLessThan(strpos($html, '')); +}); + +it('only shows the backup enabled badge for an enabled volume backup', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => false, + ]); + + $component = Livewire::test(Show::class, [ + 'storage' => $volume, + 'resource' => $application, + ])->assertDontSee('Backup enabled'); + + $backup->update(['enabled' => true]); + + $backupUrl = route('project.application.backup.show', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'backup_uuid' => $backup->uuid, + ]); + + $component + ->dispatch('refreshVolumeBackups') + ->assertSeeInOrder(['Volume Name', 'Backup enabled']) + ->assertSee('href="'.$backupUrl.'"', false); + + Livewire::test(Show::class, [ + 'storage' => $volume, + 'resource' => $application, + 'isFirst' => false, + ])->assertSeeInOrder(['Volume Name', 'Backup enabled']); +}); + +it('links the backup enabled badge to a filtered backup list when the application has multiple schedules', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + ]); + createApplicationBackupDirectory($application)->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'weekly', + 'enabled' => true, + ]); + + $backupUrl = route('project.application.backup.index', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'search' => $volume->name, + ]); + + Livewire::test(Show::class, [ + 'storage' => $volume, + 'resource' => $application, + ]) + ->assertSee('Backup enabled') + ->assertSee('href="'.$backupUrl.'"', false); +}); + +it('offers backup configuration and status on application directory mounts', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application] = createVolumeBackupApplication($team); + $directory = createApplicationBackupDirectory($application); + + $component = Livewire::test(FileStorage::class, ['fileStorage' => $directory]) + ->assertSee('Configure Backup') + ->assertSeeInOrder(['Convert to file', 'Configure Backup', 'Delete']) + ->assertDontSee('Backup enabled'); + + $backup = $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + ]); + + $backupUrl = route('project.application.backup.show', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'backup_uuid' => $backup->uuid, + ]); + + $component + ->dispatch('refreshVolumeBackups') + ->assertSee('Backup enabled') + ->assertSee('href="'.$backupUrl.'"', false); +}); + +it('links the backup enabled badge to service database backups for database directory mounts', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application] = createVolumeBackupApplication($team); + $service = Service::factory()->create([ + 'environment_id' => $application->environment_id, + 'destination_id' => $application->destination_id, + 'destination_type' => $application->destination_type, + ]); + $database = ServiceDatabase::create([ + 'uuid' => new_public_id(), + 'name' => 'postgres', + 'image' => 'postgres:17-alpine', + 'service_id' => $service->id, + ]); + $directory = LocalFileVolume::unguarded(fn () => LocalFileVolume::withoutEvents(fn () => LocalFileVolume::create([ + 'uuid' => new_public_id(), + 'fs_path' => './postgres-data', + 'mount_path' => '/var/lib/postgresql/data', + 'is_directory' => true, + 'is_based_on_git' => false, + 'is_preview_suffix_enabled' => true, + 'resource_id' => $database->id, + 'resource_type' => $database->getMorphClass(), + ]))); + $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + ]); + + $backupUrl = route('project.service.database.backups', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'service_uuid' => $service->uuid, + 'stack_service_uuid' => $database->uuid, + ]); + + Livewire::test(FileStorage::class, ['fileStorage' => $directory]) + ->assertSee('Backup enabled') + ->assertSee('href="'.$backupUrl.'"', false); +}); + +it('prevents a backed up directory from being converted or deleted', function () { + Process::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application] = createVolumeBackupApplication($team); + $directory = createApplicationBackupDirectory($application); + $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + + Livewire::test(FileStorage::class, ['fileStorage' => $directory]) + ->call('convertToFile') + ->assertDispatched('error') + ->call('delete', 'password') + ->assertDispatched('error'); + + expect($directory->fresh())->not->toBeNull() + ->and($directory->fresh()->is_directory)->toBeTrue(); +}); + +it('stores volume backup schedules and executions', function () { + expect(Schema::hasColumns('scheduled_volume_backups', [ + 'uuid', + 'backupable_type', + 'backupable_id', + 'team_id', + 's3_storage_id', + 'frequency', + 'enabled', + 'save_s3', + 'disable_local_backup', + 'stop_during_backup', + 'retention_amount_locally', + 'retention_days_locally', + 'retention_max_storage_locally', + 'retention_amount_s3', + 'retention_days_s3', + 'retention_max_storage_s3', + 'timeout', + ]))->toBeTrue() + ->and(Schema::hasColumns('scheduled_volume_backup_executions', [ + 'uuid', + 'scheduled_volume_backup_id', + 'status', + 'message', + 'size', + 'filename', + 'stop_container_ids', + 'stop_recovery_pending', + 's3_cleanup_pending', + 'finished_at', + 'local_storage_deleted', + 's3_storage_deleted', + 's3_uploaded', + ]))->toBeTrue(); +}); + +it('records the S3 storage used by each volume backup execution', function () { + expect(Schema::hasColumn('scheduled_volume_backup_executions', 's3_storage_id'))->toBeTrue() + ->and((new ScheduledVolumeBackupExecution)->s3())->not->toBeNull(); +}); + +it('exposes actions to manage and run volume backups', function () { + expect(method_exists(VolumeBackups::class, 'save'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'backupNow'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'toggleEnabled'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'delete'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'cleanupFailed'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'cleanupDeleted'))->toBeTrue() + ->and(method_exists(VolumeBackups::class, 'deleteBackup'))->toBeTrue(); +}); + +it('declares the volume backup action return types', function () { + $backupNowReturnType = (new ReflectionMethod(VolumeBackups::class, 'backupNow'))->getReturnType(); + $deleteReturnType = (new ReflectionMethod(VolumeBackups::class, 'delete'))->getReturnType(); + + expect($backupNowReturnType)->toBeInstanceOf(ReflectionUnionType::class) + ->and(collect($backupNowReturnType->getTypes())->map->getName()->all())->toEqualCanonicalizing([ + RedirectResponse::class, + Redirector::class, + 'null', + ]) + ->and($deleteReturnType)->toBeInstanceOf(ReflectionUnionType::class) + ->and(collect($deleteReturnType->getTypes())->map->getName()->all())->toEqualCanonicalizing(['bool', 'string']); +}); + +it('renders volume backup executions like database backup executions', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $server->settings->update(['server_timezone' => 'Europe/Budapest']); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'size' => 1024, + 'finished_at' => now(), + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 'executions']) + ->assertSet('timezone', 'Europe/Budapest') + ->assertSeeInOrder([ + 'Executions', + 'Success', + 'Volume: app-data', + 'Backup Availability:', + ]) + ->assertSee('Executions') + ->assertSee('Page 1 of 1') + ->assertSee('Cleanup Failed Backups') + ->assertSee('Cleanup Deleted') + ->assertSee('Backup Availability:') + ->assertSee('Local Storage') + ->assertSee('Location: /data/coolify/backups/volumes/test/archive.tar.gz') + ->assertSee('Download') + ->assertSee('Delete') + ->assertDontSee('border border-neutral-200', false); +}); + +it('cleans up failed and fully deleted volume backup execution records', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $failed = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + ]); + $deleted = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'local_storage_deleted' => true, + 's3_storage_deleted' => true, + 's3_uploaded' => true, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('cleanupFailed') + ->assertDispatched('success') + ->call('cleanupDeleted') + ->assertDispatched('success'); + + expect($failed->fresh())->toBeNull() + ->and($deleted->fresh())->toBeNull(); +}); + +it('deletes an individual volume backup archive and execution', function () { + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + Process::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'finished_at' => now(), + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('deleteBackup', $execution->id, 'password') + ->assertDispatched('success'); + + expect($execution->fresh())->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'archive.tar.gz')); +}); + +it('deletes an individual S3 archive from the storage recorded on its execution', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz', + 'local_storage_deleted' => true, + 's3_uploaded' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete') + ->once() + ->with(['/data/coolify/backups/volumes/test/historical.tar.gz']) + ->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->set('delete_backup_s3', true) + ->call('deleteBackup', $execution->id, 'password') + ->assertDispatched('success'); + + expect($execution->fresh())->toBeNull(); +}); + +it('prevents another team from downloading a volume backup', function () { + config(['app.maintenance.driver' => 'file']); + $backupTeam = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($backupTeam); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $backupTeam->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + ]); + $otherTeam = Team::factory()->create(); + signInForVolumeBackups($this, $otherTeam); + + $this->get(route('download.volume-backup', $execution->id))->assertForbidden(); +}); + +it('enables and disables volume backups from the title action', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSet('enabled', false) + ->assertSee('Enable Backup') + ->call('toggleEnabled') + ->assertSet('enabled', true) + ->assertSee('Disable Backup'); + + expect(ScheduledVolumeBackup::query()->sole()->enabled)->toBeTrue(); + + $component->call('toggleEnabled')->assertSet('enabled', false); + + expect(ScheduledVolumeBackup::query()->sole()->enabled)->toBeFalse(); +}); + +it('enables and disables volume S3 backups from the S3 title action', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 's3_storage_id' => $s3Storage->id, + ]); + + $component = Livewire::test(VolumeBackups::class, [ + 'storage' => $volume, + 'resource' => $application, + 'section' => 's3', + ]) + ->assertSee('Enable S3') + ->call('toggleS3') + ->assertSet('saveToS3', true) + ->assertSee('Disable S3'); + + expect($backup->refresh()->save_s3)->toBeTrue(); + + $component->call('toggleS3')->assertSet('saveToS3', false); + + expect($backup->refresh()->save_s3)->toBeFalse(); +}); + +it('shows and saves volume S3 retention while S3 backups are disabled', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => false, + ]); + + Livewire::test(VolumeBackups::class, [ + 'storage' => $volume, + 'resource' => $application, + 'section' => 'retention', + ]) + ->assertSee('S3 Storage Retention') + ->set('retentionAmountS3', 12) + ->set('retentionDaysS3', 30) + ->set('retentionMaxStorageS3', 4.5) + ->call('save') + ->assertHasNoErrors(); + + expect($backup->refresh()->save_s3)->toBeFalse() + ->and($backup->retention_amount_s3)->toBe(12) + ->and($backup->retention_days_s3)->toBe(30) + ->and($backup->retention_max_storage_s3)->toBe(4.5); +}); + +it('only updates S3 fields when toggling volume S3 backups', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 's3_storage_id' => $s3Storage->id, + ]); + + Livewire::test(VolumeBackups::class, [ + 'storage' => $volume, + 'resource' => $application, + 'section' => 's3', + ]) + ->set('frequency', 'not a valid schedule') + ->call('toggleS3') + ->assertDispatched('success'); + + expect($backup->refresh()->save_s3)->toBeTrue() + ->and($backup->frequency)->toBe('daily'); +}); + +function createVolumeBackupApplication(Team $team): array +{ + InstanceSettings::unguarded(fn () => InstanceSettings::firstOrCreate(['id' => 0])); + $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]); + $server = Server::factory()->create([ + 'team_id' => $team->id, + 'private_key_id' => $privateKey->id, + 'ip' => '203.0.113.10', + ]); + $destination = StandaloneDocker::where('server_id', $server->id)->firstOrFail(); + $project = Project::factory()->create(['team_id' => $team->id]); + $environment = Environment::factory()->create(['project_id' => $project->id]); + $application = Application::factory()->create([ + 'environment_id' => $environment->id, + 'destination_id' => $destination->id, + 'destination_type' => $destination->getMorphClass(), + ]); + $volume = LocalPersistentVolume::create([ + 'name' => 'app-data', + 'mount_path' => '/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + + return [$application, $volume, $server]; +} + +function createApplicationBackupDirectory(Application $application, string $path = './uploads'): LocalFileVolume +{ + return LocalFileVolume::unguarded(fn () => LocalFileVolume::withoutEvents(fn () => LocalFileVolume::create([ + 'uuid' => new_public_id(), + 'fs_path' => $path, + 'mount_path' => '/app/uploads', + 'is_directory' => true, + 'is_based_on_git' => false, + 'is_preview_suffix_enabled' => true, + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]))); +} + +function signInForVolumeBackups($testCase, Team $team): User +{ + $user = User::factory()->create(); + $user->teams()->attach($team, ['role' => 'owner']); + $testCase->actingAs($user); + session(['currentTeam' => $team]); + + return $user; +} + +it('creates a local scheduled backup for a persistent volume', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSee('General') + ->assertSee('Volume:') + ->assertSee('inconsistent or corrupted') + ->assertSee('gracefully stop containers') + ->set('frequency', 'daily') + ->set('retentionAmountLocally', 5) + ->set('retentionDaysLocally', 14) + ->set('retentionMaxStorageLocally', 1.5) + ->set('stopDuringBackup', true) + ->call('save') + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + + expect($backup->backupable->is($volume))->toBeTrue() + ->and($backup->team_id)->toBe($team->id) + ->and($backup->frequency)->toBe('daily') + ->and($backup->retention_amount_locally)->toBe(5) + ->and($backup->retention_days_locally)->toBe(14) + ->and($backup->retention_max_storage_locally)->toBe(1.5) + ->and($backup->stop_during_backup)->toBeTrue() + ->and($backup->save_s3)->toBeFalse(); +}); + +it('only accepts a usable S3 storage owned by the current team', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $foreignStorage = S3Storage::create([ + 'name' => 'Foreign S3', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => Team::factory()->create()->id, + 'is_usable' => true, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->set('frequency', 'daily') + ->set('saveToS3', true) + ->set('s3StorageId', $foreignStorage->id) + ->call('save') + ->assertHasErrors('s3StorageId'); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); +}); + +it('saves the database-style S3 backup controls immediately', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + + $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->assertSet('s3StorageId', $s3Storage->id) + ->set('saveToS3', true) + ->call('instantSave') + ->set('disableLocalBackup', true) + ->set('retentionAmountS3', 10) + ->set('retentionDaysS3', 30) + ->set('retentionMaxStorageS3', 5.5) + ->call('instantSave'); + + $backup = ScheduledVolumeBackup::query()->sole(); + expect($backup->save_s3)->toBeTrue() + ->and($backup->s3_storage_id)->toBe($s3Storage->id) + ->and($backup->disable_local_backup)->toBeTrue() + ->and($backup->retention_amount_s3)->toBe(10) + ->and($backup->retention_days_s3)->toBe(30) + ->and($backup->retention_max_storage_s3)->toBe(5.5); + + $component->set('saveToS3', false)->call('instantSave'); + + expect($backup->fresh()->save_s3)->toBeFalse() + ->and($backup->fresh()->s3_storage_id)->toBe($s3Storage->id) + ->and($backup->fresh()->disable_local_backup)->toBeFalse(); +}); + +it('saves the selected volume backup S3 storage immediately while S3 is disabled', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $firstS3Storage = S3Storage::create([ + 'name' => 'First storage', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $secondS3Storage = S3Storage::create([ + 'name' => 'Second storage', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => false, + 's3_storage_id' => $firstS3Storage->id, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application, 'section' => 's3']) + ->set('s3StorageId', $secondS3Storage->id) + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + expect($backup->save_s3)->toBeFalse() + ->and($backup->s3_storage_id)->toBe($secondS3Storage->id); +}); + +it('saves each editable volume backup checkbox immediately', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + + $generalComponent = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]); + preg_match('/]*wire:model=(?:"stopDuringBackup"|stopDuringBackup))[^>]*>/', $generalComponent->html(), $matches); + expect($matches[0] ?? null)->not->toBeNull() + ->and($matches[0])->toContain("wire:click='instantSave'"); + + $s3Component = Livewire::test(VolumeBackups::class, [ + 'storage' => $volume, + 'resource' => $application, + 'section' => 's3', + ]); + foreach (['disableLocalBackup'] as $property) { + preg_match('/]*wire:model=(?:"'.$property.'"|'.$property.'))[^>]*>/', $s3Component->html(), $matches); + expect($matches[0] ?? null)->not->toBeNull() + ->and($matches[0])->toContain("wire:click='instantSave'"); + } + + $generalComponent->set('stopDuringBackup', true)->call('instantSave')->assertDispatched('success'); + expect($backup->refresh()->stop_during_backup)->toBeTrue(); + + $generalComponent->set('stopDuringBackup', false)->call('instantSave')->assertDispatched('success'); + expect($backup->refresh()->stop_during_backup)->toBeFalse(); +}); + +it('allows volume S3 backups to be disabled when no usable storage remains', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => null, + ]); + + $component = Livewire::test(VolumeBackups::class, [ + 'storage' => $volume, + 'resource' => $application, + 'section' => 's3', + ]) + ->assertSet('saveToS3', true) + ->assertSeeHtml('

S3

') + ->assertSeeText('No validated S3 available. Configure one here.') + ->assertSeeHtml('href="'.route('storage.index').'"') + ->assertSeeHtml('>here') + ->assertDontSee('Save') + ->assertDontSee('Disable S3') + ->assertDontSee('S3 Storage') + ->assertDontSee('Disable Local Backup') + ->call('toggleS3') + ->assertDispatched('success') + ->assertSet('saveToS3', false) + ->assertDontSee('Enable S3'); + + expect($backup->refresh()->save_s3)->toBeFalse() + ->and($backup->s3_storage_id)->toBeNull(); + +}); + +it('disables S3 volume backups when the storage is deleted', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/s3.tar.gz', + 's3_uploaded' => true, + 's3_cleanup_pending' => true, + ]); + + $s3Storage->delete(); + + expect($backup->fresh()->save_s3)->toBeFalse() + ->and($backup->fresh()->s3_storage_id)->toBeNull() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue() + ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse(); +}); + +it('marks historical executions deleted when their recorded S3 storage is removed', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz', + 's3_uploaded' => true, + 's3_cleanup_pending' => true, + ]); + + $originalStorage->delete(); + + expect($backup->fresh()->s3_storage_id)->toBe($newStorage->id) + ->and($execution->fresh()->s3_storage_id)->toBeNull() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue() + ->and($execution->fresh()->s3_cleanup_pending)->toBeFalse(); +}); + +it('queues a manual backup before a schedule has been saved', function () { + Queue::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + + $component = Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('backupNow') + ->assertDispatched('success'); + + $backup = ScheduledVolumeBackup::query()->sole(); + + $component->assertRedirectToRoute('project.application.backup.executions', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'backup_uuid' => $backup->uuid, + ]); + + expect($backup->enabled)->toBeFalse(); + Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup)); +}); + +it('queues a manual backup when its S3 storage has become unusable', function () { + Queue::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Unavailable storage', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => false, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $parameters = [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + 'backup_uuid' => $backup->uuid, + ]; + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('backupNow') + ->assertRedirectToRoute('project.application.backup.executions', $parameters); + + Queue::assertPushed(VolumeBackupJob::class, fn (VolumeBackupJob $job) => $job->backup->is($backup)); +}); + +it('deletes local archives before deleting a volume backup schedule', function () { + Process::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + 'size' => 128, + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('success') + ->assertRedirectToRoute('project.application.backup.index', [ + 'project_uuid' => $application->project()->uuid, + 'environment_uuid' => $application->environment->uuid, + 'application_uuid' => $application->uuid, + ]); + + expect(ScheduledVolumeBackup::query()->count())->toBe(0); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'archive.tar.gz')); +}); + +it('deletes a volume backup schedule without a password when two-step confirmation is disabled', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + InstanceSettings::get()->update(['disable_two_step_confirmation' => true]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', '') + ->assertDispatched('success'); + + expect($backup->fresh())->toBeNull(); +}); + +it('deletes S3 archives from the storage recorded on each execution', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/historical.tar.gz', + 'local_storage_deleted' => true, + 's3_uploaded' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete') + ->once() + ->with(['/data/coolify/backups/volumes/test/historical.tar.gz']) + ->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('success'); + + expect($backup->fresh())->toBeNull(); +}); + +it('refuses to delete a schedule while its backup is running', function () { + Process::fake(); + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'running', + ]); + + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('error'); + + expect($backup->fresh())->not->toBeNull(); + Process::assertNothingRan(); +}); + +it('uses the backup operation lock while deleting a schedule', function () { + $team = Team::factory()->create(); + signInForVolumeBackups($this, $team); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $lock = Cache::lock(VolumeBackupJob::lockKey($backup->id), 60); + $lock->get(); + + try { + Livewire::test(VolumeBackups::class, ['storage' => $volume, 'resource' => $application]) + ->call('delete', 'password') + ->assertDispatched('error'); + + expect($backup->fresh())->not->toBeNull(); + } finally { + $lock->release(); + } +}); + +it('prevents a volume with tracked backup archives from being deleted', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + + expect(fn () => $volume->delete())->toThrow(RuntimeException::class, 'Delete this volume backup schedule'); + expect($volume->fresh())->not->toBeNull(); +}); + +it('aborts storage deletion when scheduled backups exist', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $directory = createApplicationBackupDirectory($application); + + $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + + expect(fn () => $volume->abortIfScheduledBackupsExist()) + ->toThrow(HttpException::class, 'Delete this volume backup schedule and its archives before deleting the volume.') + ->and(fn () => $directory->abortIfScheduledBackupsExist()) + ->toThrow(HttpException::class, 'Delete this directory backup schedule and its archives before deleting the directory.'); +}); + +it('deletes disabled volume backup archives before deleting their application', function () { + Process::fake(); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => false, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/application-delete.tar.gz', + ]); + $application->delete(); + + (new DeleteResourceJob($application))->handle(); + + expect($backup->fresh())->toBeNull() + ->and($execution->fresh())->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'application-delete.tar.gz')); +}); + +it('marks a running execution failed even when the job instance lost its execution state', function () { + Process::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'running', + 'filename' => '/data/coolify/backups/volumes/test/timed-out.tar.gz', + ]); + + (new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out')); + + expect($execution->fresh()->status)->toBe('failed') + ->and($execution->fresh()->message)->toBe('Worker timed out') + ->and($execution->fresh()->finished_at)->not->toBeNull() + ->and($execution->fresh()->filename)->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'timed-out.tar.gz')); +}); + +it('archives a named volume on its server', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'enabled' => true, + 'retention_amount_locally' => 7, + 'retention_amount_s3' => 7, + 'timeout' => 3600, + ]); + + Process::fake([ + '*docker ps -q*' => 'abc123', + '*du -b*' => '128', + '*' => '', + ]); + + $job = new VolumeBackupJob($backup); + $middleware = $job->middleware(); + $job->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + + expect($middleware)->toHaveCount(1) + ->and($middleware[0])->toBeInstanceOf(WithoutOverlapping::class) + ->and($job->queue)->toBe(crons_queue()) + ->and($execution->status)->toBe('success') + ->and($execution->size)->toBe(128) + ->and($execution->filename)->toEndWith('.tar.gz'); + + Process::assertRan(fn ($process) => str_contains($process->command, 'docker volume inspect') + && str_contains($process->command, 'docker run --rm --name ') + && str_contains($process->command, 'app-data:/volume:ro') + && str_contains($process->command, 'tar -czf -') + && str_contains($process->command, '> ') + && str_contains($process->command, '.tar.gz') + && ! str_contains($process->command, ':/backup')); +}); + +it('keeps the upload destination on the volume backup execution', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Execution destination', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => true, + 's3_storage_id' => $s3Storage->id, + ]); + $sshDisk = Storage::fake('ssh-keys'); + $disk = Mockery::mock(); + $disk->shouldReceive('files')->zeroOrMoreTimes()->andReturn([]); + $disk->shouldReceive('delete')->zeroOrMoreTimes()->andReturnTrue(); + Storage::shouldReceive('disk')->with('ssh-keys')->andReturn($sshDisk); + Storage::shouldReceive('build')->zeroOrMoreTimes()->andReturn($disk); + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + + expect($execution->s3_storage_id)->toBe($s3Storage->id) + ->and($execution->s3->is($s3Storage))->toBeTrue(); +}); + +it('removes local volume backups older than the configured retention days', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'retention_amount_locally' => 0, + 'retention_days_locally' => 7, + 'retention_max_storage_locally' => 0, + ]); + $expiredExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/expired.tar.gz', + 'size' => 64, + ]); + ScheduledVolumeBackupExecution::query()->whereKey($expiredExecution)->update(['created_at' => now()->subDays(8)]); + $recentExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/recent.tar.gz', + 'size' => 64, + ]); + ScheduledVolumeBackupExecution::query()->whereKey($recentExecution)->update(['created_at' => now()->subDay()]); + + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + expect($expiredExecution->fresh())->toBeNull() + ->and($recentExecution->fresh())->not->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'expired.tar.gz')); +}); + +it('removes oldest local volume backups over the configured storage limit', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0.00000015, + ]); + $oldExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/over-limit.tar.gz', + 'size' => 64, + ]); + ScheduledVolumeBackupExecution::query()->whereKey($oldExecution)->update(['created_at' => now()->subDay()]); + + Process::fake([ + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + expect($oldExecution->fresh())->toBeNull() + ->and($backup->executions()->where('status', 'success')->count())->toBe(1); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, 'over-limit.tar.gz')); +}); + +it('removes retained S3 archives from the storage recorded on each execution', function () { + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0, + 'retention_amount_s3' => 1, + 'retention_days_s3' => 0, + 'retention_max_storage_s3' => 0, + ]); + $oldExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/old.tar.gz', + 's3_uploaded' => true, + 'created_at' => now()->subDay(), + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $newStorage->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/new.tar.gz', + 's3_uploaded' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->with(['/data/coolify/backups/volumes/test/old.tar.gz'])->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + $job = new VolumeBackupJob($backup); + $method = (new ReflectionClass($job))->getMethod('removeExpiredBackups'); + + $method->invoke($job, $server); + + expect($oldExecution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + +it('does not query execution history when local retention is unlimited', function () { + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'save_s3' => false, + 'retention_amount_locally' => 0, + 'retention_days_locally' => 0, + 'retention_max_storage_locally' => 0, + ]); + ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/unlimited.tar.gz', + ]); + $job = new VolumeBackupJob($backup); + $method = (new ReflectionClass($job))->getMethod('removeExpiredBackups'); + DB::enableQueryLog(); + DB::flushQueryLog(); + + $method->invoke($job, $server); + + $executionQueries = collect(DB::getQueryLog()) + ->filter(fn (array $query): bool => str_contains($query['query'], 'scheduled_volume_backup_executions')) + ->filter(fn (array $query): bool => str_starts_with(strtolower(ltrim($query['query'])), 'select')); + + expect($executionQueries)->toBeEmpty(); +}); + +it('keeps a successful backup successful when retention cleanup fails', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'disable_local_backup' => true, + 'retention_amount_locally' => 1, + ]); + $oldExecution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/old.tar.gz', + 'size' => 64, + 'created_at' => now()->subDay(), + ]); + + Process::fake([ + '*rm -f*' => Process::result(errorOutput: 'permission denied', exitCode: 1), + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + $latestExecution = $backup->executions()->first(); + + expect($latestExecution->status)->toBe('success') + ->and($latestExecution->filename)->not->toBeNull() + ->and($oldExecution->fresh()->local_storage_deleted)->toBeFalse(); +}); + +it('stops and restarts containers that use the volume when requested', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'stop_during_backup' => true, + ]); + + Process::fake([ + '*docker ps -q*' => "abc123\ndef456", + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + Process::assertRan(fn ($process) => str_contains($process->command, 'trap cleanup EXIT') + && str_contains($process->command, 'exit 143') + && str_contains($process->command, 'TERM') + && str_contains($process->command, 'docker stop') + && str_contains($process->command, 'docker start') + && str_contains($process->command, 'state_file=')); + Process::assertRan(fn ($process) => str_contains($process->command, '{{println .Source}}{{println .Name}}') + && str_contains($process->command, "grep -Fqx -- 'app-data'") + && str_contains($process->command, '{{.State.Paused}}') + && str_contains($process->command, 'continue')); +}); + +it('finds containers using a bind mounted host path before stopping', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $volume->update(['host_path' => '/srv/app-data']); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'stop_during_backup' => true, + ]); + + Process::fake([ + '*docker ps -q*' => 'abc123', + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + Process::assertRan(fn ($process) => str_contains($process->command, '{{println .Source}}{{println .Name}}') + && str_contains($process->command, "grep -Fqx -- '/srv/app-data'")); +}); + +it('archives an application directory mount from its resolved host path', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application] = createVolumeBackupApplication($team); + $directory = LocalFileVolume::unguarded(fn () => LocalFileVolume::withoutEvents(fn () => LocalFileVolume::create([ + 'uuid' => new_public_id(), + 'fs_path' => './uploads', + 'mount_path' => '/app/uploads', + 'is_directory' => true, + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]))); + $backup = $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'stop_during_backup' => true, + ]); + $source = $application->workdir().'/uploads'; + + Process::fake([ + '*docker ps -q*' => 'abc123', + '*du -b*' => '128', + '*' => '', + ]); + + (new VolumeBackupJob($backup))->handle(); + + expect(ScheduledVolumeBackupExecution::query()->sole()->status)->toBe('success'); + Process::assertRan(fn ($process) => str_contains($process->command, 'test -d '.escapeshellarg($source))); + Process::assertRan(fn ($process) => str_contains($process->command, $source) + && str_contains($process->command, ':/volume:ro') + && str_contains($process->command, 'directory-uploads-')); + Process::assertRan(fn ($process) => str_contains($process->command, "grep -Fqx -- '".$source."'")); + Process::assertRan(fn ($process) => str_contains($process->command, 'docker stop') + && str_contains($process->command, 'docker start')); +}); + +it('retries container recovery when a timed out backup left a container stopped', function () { + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'running', + 'stop_recovery_pending' => true, + ]); + Process::fake([ + '*cat *coolify-volume-backup-*' => 'abc123', + '*docker start*' => Process::result(errorOutput: 'daemon unavailable', exitCode: 1), + ]); + + (new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out')); + + expect($execution->fresh()->status)->toBe('failed') + ->and($execution->fresh()->message)->toContain('Container recovery failed') + ->and($execution->fresh()->stop_container_ids)->toBe(['abc123']) + ->and($execution->fresh()->stop_recovery_pending)->toBeTrue(); + Queue::assertPushed(VolumeBackupRecoveryJob::class); +}); + +it('restarts containers and removes a partial archive when archiving fails', function () { + config(['broadcasting.default' => 'null']); + InstanceSettings::unguarded(fn () => InstanceSettings::create(['id' => 0])); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => 'daily', + 'stop_during_backup' => true, + ]); + Process::fake([ + '*docker ps -q*' => 'abc123', + '*trap cleanup EXIT*' => Process::result(errorOutput: 'archive failed', exitCode: 1), + '*' => '', + ]); + + expect(fn () => (new VolumeBackupJob($backup))->handle())->toThrow(RuntimeException::class); + + $execution = ScheduledVolumeBackupExecution::query()->sole(); + expect($execution->status)->toBe('failed') + ->and($execution->stop_container_ids)->toBeNull(); + Process::assertRan(fn ($process) => str_contains($process->command, 'rm -f') + && str_contains($process->command, '.tar.gz')); +}); + +it('throws when the S3 filesystem reports that deletion failed', function () { + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + $s3 = new S3Storage([ + 'key' => 'key', + 'secret' => 'secret', + 'region' => 'us-east-1', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + ]); + + expect(fn () => deleteBackupsS3('archive.tar.gz', $s3)) + ->toThrow(RuntimeException::class, 'could not be deleted'); +}); + +it('cleans an interrupted S3 upload and coordinates recovery with the backup lock', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('build')->once()->andReturn($disk); + $job = new VolumeBackupRecoveryJob($execution); + + expect($job->middleware()[0]->getLockKey($job))->toBe(VolumeBackupJob::lockKey($backup->id)); + $job->handle(); + + expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + +it('cleans an interrupted upload from the execution S3 storage after a schedule switch', function () { + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $originalStorage = S3Storage::create([ + 'name' => 'Original storage', + 'region' => 'us-east-1', + 'key' => 'original-key', + 'secret' => 'secret', + 'bucket' => 'original-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $newStorage = S3Storage::create([ + 'name' => 'New storage', + 'region' => 'us-east-1', + 'key' => 'new-key', + 'secret' => 'secret', + 'bucket' => 'new-bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $newStorage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $originalStorage->id, + 'status' => 'failed', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnTrue(); + Storage::shouldReceive('build') + ->once() + ->with(Mockery::on(fn (array $config): bool => $config['key'] === 'original-key')) + ->andReturn($disk); + + VolumeBackupRecoveryJob::cleanupS3Upload($execution); + + expect($execution->fresh()->s3_cleanup_pending)->toBeFalse() + ->and($execution->fresh()->s3_storage_deleted)->toBeTrue(); +}); + +it('keeps the S3 key tracked when interrupted upload cleanup must be retried', function () { + Queue::fake(); + Process::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $s3Storage = S3Storage::create([ + 'name' => 'Volume backups', + 'region' => 'us-east-1', + 'key' => 'key', + 'secret' => 'secret', + 'bucket' => 'bucket', + 'endpoint' => 'https://s3.example.com', + 'team_id' => $team->id, + 'is_usable' => true, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 's3_storage_id' => $s3Storage->id, + 'frequency' => 'daily', + 'save_s3' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 's3_storage_id' => $s3Storage->id, + 'status' => 'running', + 'filename' => '/data/coolify/backups/volumes/test/interrupted.tar.gz', + 's3_cleanup_pending' => true, + ]); + $disk = Mockery::mock(); + $disk->shouldReceive('delete')->once()->andReturnFalse(); + Storage::shouldReceive('build')->once()->andReturn($disk); + + (new VolumeBackupJob($backup))->failed(new RuntimeException('Worker timed out')); + + expect($execution->fresh()->filename)->toBe('/data/coolify/backups/volumes/test/interrupted.tar.gz') + ->and($execution->fresh()->s3_cleanup_pending)->toBeTrue(); + Queue::assertPushed(VolumeBackupRecoveryJob::class); +}); + +it('dispatches due scheduled volume backups', function () { + config(['constants.coolify.self_hosted' => true]); + Carbon::setTestNow('2026-07-15 12:00:00'); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $server->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + ]); + expect($server->fresh()->isFunctional())->toBeTrue(); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => '* * * * *', + 'enabled' => true, + ]); + Cache::forget("scheduled-volume-backup:{$backup->id}"); + expect($backup->server()?->isFunctional())->toBeTrue(); + + (new ScheduledJobManager)->handle(); + + Queue::assertPushed( + VolumeBackupJob::class, + fn (VolumeBackupJob $job) => $job->backup->is($backup), + ); +}); + +it('dispatches due scheduled directory backups', function () { + config(['constants.coolify.self_hosted' => true]); + Carbon::setTestNow('2026-07-15 12:00:00'); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $directory = createApplicationBackupDirectory($application); + $server->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + ]); + $backup = $directory->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => '* * * * *', + 'enabled' => true, + ]); + Cache::forget("scheduled-volume-backup:{$backup->id}"); + + (new ScheduledJobManager)->handle(); + + Queue::assertPushed( + VolumeBackupJob::class, + fn (VolumeBackupJob $job) => $job->backup->is($backup), + ); +}); + +it('retains scheduled volume backups and archive metadata when the server is missing', function () { + config(['constants.coolify.self_hosted' => true]); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume] = createVolumeBackupApplication($team); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => '* * * * *', + 'enabled' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'success', + 'filename' => '/data/coolify/backups/volumes/test/archive.tar.gz', + ]); + $application->update([ + 'destination_id' => null, + 'destination_type' => null, + ]); + + (new ScheduledJobManager)->handle(); + + expect($backup->fresh())->not->toBeNull() + ->and($execution->fresh())->not->toBeNull() + ->and($execution->fresh()->filename)->toBe('/data/coolify/backups/volumes/test/archive.tar.gz'); + Queue::assertNotPushed(VolumeBackupJob::class); +}); + +it('dispatches pending recovery without starting another volume backup', function () { + config(['constants.coolify.self_hosted' => true]); + Queue::fake(); + $team = Team::factory()->create(); + [$application, $volume, $server] = createVolumeBackupApplication($team); + $server->settings()->update([ + 'is_reachable' => true, + 'is_usable' => true, + 'force_disabled' => false, + ]); + $backup = $volume->scheduledBackups()->create([ + 'team_id' => $team->id, + 'frequency' => '* * * * *', + 'enabled' => true, + ]); + $execution = ScheduledVolumeBackupExecution::create([ + 'scheduled_volume_backup_id' => $backup->id, + 'status' => 'failed', + 'stop_recovery_pending' => true, + ]); + + (new ScheduledJobManager)->handle(); + + Queue::assertPushed( + VolumeBackupRecoveryJob::class, + fn (VolumeBackupRecoveryJob $job) => $job->execution->is($execution), + ); + Queue::assertNotPushed(VolumeBackupJob::class); +}); diff --git a/tests/Unit/BackupDownloadHelperTest.php b/tests/Unit/BackupDownloadHelperTest.php new file mode 100644 index 000000000..8ed499df6 --- /dev/null +++ b/tests/Unit/BackupDownloadHelperTest.php @@ -0,0 +1,72 @@ +forceFill([ + 'ip' => '192.0.2.10', + 'port' => 2222, + 'user' => 'root', + ]); + + $privateKey = Mockery::mock(PrivateKey::class); + $privateKey->shouldReceive('getKeyLocation')->andReturn('/tmp/private-key'); + $server->setRelation('privateKey', $privateKey); + + return $server; +} + +it('throws when the backup file does not exist on the server', function () { + $disk = Mockery::mock(); + $disk->shouldReceive('exists') + ->once() + ->with('/backups/archive.tar.gz') + ->andReturnFalse(); + + Storage::shouldReceive('build') + ->once() + ->with([ + 'driver' => 'sftp', + 'host' => '192.0.2.10', + 'port' => 2222, + 'username' => 'root', + 'privateKey' => '/tmp/private-key', + 'root' => '/', + ]) + ->andReturn($disk); + + streamBackupFromServer(backupDownloadServer(), '/backups/archive.tar.gz', 'application/gzip'); +})->throws(FileNotFoundException::class); + +it('streams a backup file with the requested content type', function () { + $stream = fopen('php://memory', 'r+'); + fwrite($stream, 'backup contents'); + rewind($stream); + + $disk = Mockery::mock(); + $disk->shouldReceive('exists')->once()->andReturnTrue(); + $disk->shouldReceive('readStream') + ->once() + ->with('/backups/archive.tar.gz') + ->andReturn($stream); + Storage::shouldReceive('build')->once()->andReturn($disk); + + $response = streamBackupFromServer(backupDownloadServer(), '/backups/archive.tar.gz', 'application/gzip'); + + expect($response)->toBeInstanceOf(StreamedResponse::class) + ->and($response->headers->get('content-type'))->toBe('application/gzip') + ->and($response->headers->get('content-disposition'))->toBe('attachment; filename="archive.tar.gz"'); + + ob_start(); + ob_start(); + $response->sendContent(); + $contents = ob_get_clean(); + + expect($contents)->toBe('backup contents'); +}); diff --git a/tests/Unit/DeploymentConfiguration/ApplicationConfigurationSnapshotTest.php b/tests/Unit/DeploymentConfiguration/ApplicationConfigurationSnapshotTest.php index a00c00b4b..afad0593f 100644 --- a/tests/Unit/DeploymentConfiguration/ApplicationConfigurationSnapshotTest.php +++ b/tests/Unit/DeploymentConfiguration/ApplicationConfigurationSnapshotTest.php @@ -4,6 +4,8 @@ use App\Models\ApplicationDeploymentQueue; use App\Models\Environment; use App\Models\EnvironmentVariable; +use App\Models\LocalFileVolume; +use App\Models\LocalPersistentVolume; use App\Models\Project; use App\Models\Team; use App\Services\DeploymentConfiguration\ConfigurationDiffer; @@ -79,6 +81,35 @@ function markSnapshotTestApplicationDeployed(Application $application): Applicat ->and($change['new_full_value'])->toBe($domains); }); +it('detects added storage mounts as redeploy-only changes', function (string $type) { + $application = snapshotTestApplication(); + markSnapshotTestApplicationDeployed($application); + + if ($type === 'volume') { + LocalPersistentVolume::create([ + 'name' => $application->uuid.'-data', + 'mount_path' => '/app/data', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ]); + } else { + LocalFileVolume::withoutEvents(fn () => LocalFileVolume::forceCreate([ + 'uuid' => (string) Str::uuid(), + 'fs_path' => application_configuration_dir().'/'.$application->uuid.'/data', + 'mount_path' => '/app/data', + 'is_directory' => $type === 'directory', + 'resource_id' => $application->id, + 'resource_type' => $application->getMorphClass(), + ])); + } + + $diff = $application->refresh()->pendingDeploymentConfigurationDiff(); + + expect($diff->isChanged())->toBeTrue() + ->and($diff->requiresBuild())->toBeFalse() + ->and(collect($diff->changes())->pluck('section'))->toContain('storage'); +})->with(['volume', 'directory', 'file']); + it('detects Docker image reference changes as redeploy-only changes', function (string $field, string $label, string $newValue) { $application = snapshotTestApplication([ 'build_pack' => 'dockerimage', diff --git a/tests/Unit/ProductionImageWorkflowTest.php b/tests/Unit/ProductionImageWorkflowTest.php index 06085a33f..557e86b64 100644 --- a/tests/Unit/ProductionImageWorkflowTest.php +++ b/tests/Unit/ProductionImageWorkflowTest.php @@ -32,3 +32,15 @@ ->toContain('if: ${{ ! github.event.release.prerelease }}') ->toContain('--tag "${IMAGE}:latest"'); }); + +it('documents the sha image release process', function () { + $releaseGuide = file_get_contents(dirname(__DIR__, 2).'/RELEASE.md'); + + expect($releaseGuide) + ->toContain('Merge the release commit into `v4.x`') + ->toContain('`sha-`') + ->toContain('targeting the exact commit that produced the SHA image') + ->toContain('promotes the existing SHA image without rebuilding it') + ->toContain('Update the CDN') + ->not->toContain('Merging to `main`'); +});