fix(backups): support long-running volume backups (#11358)
This commit is contained in:
commit
8aa2cbe462
8 changed files with 80 additions and 9 deletions
|
|
@ -34,7 +34,7 @@
|
||||||
new OA\Property(property: 'retention_amount_s3', type: 'integer', default: 7, minimum: 0, maximum: 10000),
|
new OA\Property(property: 'retention_amount_s3', type: 'integer', default: 7, minimum: 0, maximum: 10000),
|
||||||
new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),
|
new OA\Property(property: 'retention_days_s3', type: 'integer', default: 0, maximum: 2147483647, minimum: 0),
|
||||||
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),
|
new OA\Property(property: 'retention_max_storage_s3', type: 'number', format: 'float', default: 0, maximum: 9999999999, minimum: 0),
|
||||||
new OA\Property(property: 'timeout', type: 'integer', default: 3600, minimum: 60, maximum: 36000),
|
new OA\Property(property: 'timeout', type: 'integer', default: ScheduledVolumeBackup::DEFAULT_TIMEOUT, minimum: 60, maximum: 36000),
|
||||||
],
|
],
|
||||||
type: 'object',
|
type: 'object',
|
||||||
additionalProperties: false,
|
additionalProperties: false,
|
||||||
|
|
@ -261,7 +261,7 @@ private function persistSchedule(
|
||||||
string $resourceType,
|
string $resourceType,
|
||||||
Model $resource,
|
Model $resource,
|
||||||
): JsonResponse {
|
): JsonResponse {
|
||||||
$backup = $storage->scheduledBackups()->updateOrCreate([], [
|
$attributes = [
|
||||||
'team_id' => $teamId,
|
'team_id' => $teamId,
|
||||||
'frequency' => $request->string('frequency')->toString(),
|
'frequency' => $request->string('frequency')->toString(),
|
||||||
'enabled' => $request->boolean('enabled', true),
|
'enabled' => $request->boolean('enabled', true),
|
||||||
|
|
@ -275,8 +275,12 @@ private function persistSchedule(
|
||||||
'retention_amount_s3' => $request->integer('retention_amount_s3', 7),
|
'retention_amount_s3' => $request->integer('retention_amount_s3', 7),
|
||||||
'retention_days_s3' => $request->integer('retention_days_s3'),
|
'retention_days_s3' => $request->integer('retention_days_s3'),
|
||||||
'retention_max_storage_s3' => $request->float('retention_max_storage_s3'),
|
'retention_max_storage_s3' => $request->float('retention_max_storage_s3'),
|
||||||
'timeout' => $request->integer('timeout', 3600),
|
];
|
||||||
]);
|
if ($request->has('timeout')) {
|
||||||
|
$attributes['timeout'] = $request->integer('timeout');
|
||||||
|
}
|
||||||
|
|
||||||
|
$backup = $storage->scheduledBackups()->updateOrCreate([], $attributes);
|
||||||
$created = $backup->wasRecentlyCreated;
|
$created = $backup->wasRecentlyCreated;
|
||||||
|
|
||||||
auditLog('api.volume_backup.schedule_set', [
|
auditLog('api.volume_backup.schedule_set', [
|
||||||
|
|
|
||||||
|
|
@ -28,14 +28,14 @@ class VolumeBackupJob implements ShouldBeEncrypted, ShouldQueue
|
||||||
|
|
||||||
public int $maxExceptions = 1;
|
public int $maxExceptions = 1;
|
||||||
|
|
||||||
public int $timeout = 3600;
|
public int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT;
|
||||||
|
|
||||||
private ?ScheduledVolumeBackupExecution $execution = null;
|
private ?ScheduledVolumeBackupExecution $execution = null;
|
||||||
|
|
||||||
public function __construct(public ScheduledVolumeBackup $backup)
|
public function __construct(public ScheduledVolumeBackup $backup)
|
||||||
{
|
{
|
||||||
$this->onQueue(crons_queue());
|
$this->onQueue(crons_queue());
|
||||||
$this->timeout = $backup->timeout ?? 3600;
|
$this->timeout = $backup->timeout ?? ScheduledVolumeBackup::DEFAULT_TIMEOUT;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function middleware(): array
|
public function middleware(): array
|
||||||
|
|
|
||||||
|
|
@ -56,7 +56,7 @@ class VolumeBackups extends Component
|
||||||
|
|
||||||
public string $timezone = '';
|
public string $timezone = '';
|
||||||
|
|
||||||
public int $timeout = 3600;
|
public int $timeout = ScheduledVolumeBackup::DEFAULT_TIMEOUT;
|
||||||
|
|
||||||
public int $perPage = 10;
|
public int $perPage = 10;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,8 @@
|
||||||
|
|
||||||
class ScheduledVolumeBackup extends BaseModel
|
class ScheduledVolumeBackup extends BaseModel
|
||||||
{
|
{
|
||||||
|
public const int DEFAULT_TIMEOUT = 36000;
|
||||||
|
|
||||||
protected $fillable = [
|
protected $fillable = [
|
||||||
'uuid',
|
'uuid',
|
||||||
'backupable_type',
|
'backupable_type',
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
|
use App\Models\ScheduledVolumeBackup;
|
||||||
use Illuminate\Support\Str;
|
use Illuminate\Support\Str;
|
||||||
|
|
||||||
return [
|
return [
|
||||||
|
|
@ -202,7 +203,10 @@
|
||||||
'tries' => 1,
|
'tries' => 1,
|
||||||
'nice' => 0,
|
'nice' => 0,
|
||||||
'sleep' => 3,
|
'sleep' => 3,
|
||||||
'timeout' => env('HORIZON_TIMEOUT', 36000),
|
'timeout' => min(
|
||||||
|
max((int) env('HORIZON_TIMEOUT', 39600), ScheduledVolumeBackup::DEFAULT_TIMEOUT + 600),
|
||||||
|
85800,
|
||||||
|
),
|
||||||
],
|
],
|
||||||
|
|
||||||
],
|
],
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
use Illuminate\Database\Migrations\Migration;
|
||||||
|
use Illuminate\Database\Schema\Blueprint;
|
||||||
|
use Illuminate\Support\Facades\Schema;
|
||||||
|
|
||||||
|
return new class extends Migration
|
||||||
|
{
|
||||||
|
public function up(): void
|
||||||
|
{
|
||||||
|
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
|
||||||
|
$table->unsignedInteger('timeout')->default(36000)->change();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public function down(): void
|
||||||
|
{
|
||||||
|
Schema::table('scheduled_volume_backups', function (Blueprint $table) {
|
||||||
|
$table->unsignedInteger('timeout')->default(3600)->change();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
@ -130,6 +130,7 @@ function createVolumeBackupApiToken($context, User $user, array $abilities): str
|
||||||
'save_s3' => true,
|
'save_s3' => true,
|
||||||
'disable_local_backup' => true,
|
'disable_local_backup' => true,
|
||||||
's3_storage_id' => $this->s3Storage->id,
|
's3_storage_id' => $this->s3Storage->id,
|
||||||
|
'timeout' => 7200,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
$response = $this->withHeaders($this->headers)
|
$response = $this->withHeaders($this->headers)
|
||||||
|
|
@ -146,7 +147,8 @@ function createVolumeBackupApiToken($context, User $user, array $abilities): str
|
||||||
->and($backup->enabled)->toBeFalse()
|
->and($backup->enabled)->toBeFalse()
|
||||||
->and($backup->save_s3)->toBeFalse()
|
->and($backup->save_s3)->toBeFalse()
|
||||||
->and($backup->disable_local_backup)->toBeFalse()
|
->and($backup->disable_local_backup)->toBeFalse()
|
||||||
->and($backup->s3_storage_id)->toBeNull();
|
->and($backup->s3_storage_id)->toBeNull()
|
||||||
|
->and($backup->timeout)->toBe(7200);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('sets a directory backup schedule through the API', function () {
|
it('sets a directory backup schedule through the API', function () {
|
||||||
|
|
|
||||||
|
|
@ -51,6 +51,43 @@
|
||||||
->and(method_exists(LocalFileVolume::class, 'scheduledBackups'))->toBeTrue();
|
->and(method_exists(LocalFileVolume::class, 'scheduledBackups'))->toBeTrue();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('allows large volume backups to run for ten hours by default', function () {
|
||||||
|
$backup = new ScheduledVolumeBackup;
|
||||||
|
$job = new VolumeBackupJob($backup);
|
||||||
|
|
||||||
|
expect($job->timeout)->toBe(36000)
|
||||||
|
->and((new VolumeBackups)->timeout)->toBe(36000)
|
||||||
|
->and(config('horizon.defaults.s6.timeout'))->toBeGreaterThan($job->timeout)
|
||||||
|
->and(config('queue.connections.redis.retry_after'))->toBeGreaterThan(config('horizon.defaults.s6.timeout'));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('changes the default volume backup timeout without changing existing timeouts', function () {
|
||||||
|
$team = Team::factory()->create();
|
||||||
|
[$application, $defaultVolume] = createVolumeBackupApplication($team);
|
||||||
|
$customVolume = LocalPersistentVolume::create([
|
||||||
|
'name' => 'custom-timeout-data',
|
||||||
|
'mount_path' => '/custom-data',
|
||||||
|
'resource_id' => $application->id,
|
||||||
|
'resource_type' => $application->getMorphClass(),
|
||||||
|
]);
|
||||||
|
$defaultBackup = $defaultVolume->scheduledBackups()->create([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'frequency' => 'daily',
|
||||||
|
'timeout' => 3600,
|
||||||
|
]);
|
||||||
|
$customBackup = $customVolume->scheduledBackups()->create([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'frequency' => 'daily',
|
||||||
|
'timeout' => 7200,
|
||||||
|
]);
|
||||||
|
|
||||||
|
$migration = require database_path('migrations/2026_08_15_000000_increase_default_volume_backup_timeout.php');
|
||||||
|
$migration->up();
|
||||||
|
|
||||||
|
expect($defaultBackup->fresh()->timeout)->toBe(3600)
|
||||||
|
->and($customBackup->fresh()->timeout)->toBe(7200);
|
||||||
|
});
|
||||||
|
|
||||||
it('includes parallel gzip support in the Coolify helper image', function () {
|
it('includes parallel gzip support in the Coolify helper image', function () {
|
||||||
$dockerfile = file_get_contents(base_path('docker/coolify-helper/Dockerfile'));
|
$dockerfile = file_get_contents(base_path('docker/coolify-helper/Dockerfile'));
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue