| Time |
Type |
+ Resource |
Reason |
- Details |
@@ -214,6 +235,17 @@ class="border-b border-gray-200 dark:border-coolgray-400">
{{ ucfirst(str_replace('_', ' ', $skip['type'])) }}
+
+ @if($skip['link'] ?? null)
+
+ {{ $skip['resource_name'] }}
+
+ @elseif($skip['resource_name'] ?? null)
+ {{ $skip['resource_name'] }}
+ @else
+ {{ $skip['context']['task_name'] ?? $skip['context']['server_name'] ?? 'Deleted' }}
+ @endif
+ |
@php
$reasonLabel = match($skip['reason']) {
@@ -235,15 +267,6 @@ class="border-b border-gray-200 dark:border-coolgray-400">
@endphp
{{ $reasonLabel }}
|
-
- @php
- $details = collect($skip['context'])
- ->except(['type', 'skip_reason', 'execution_time'])
- ->map(fn($v, $k) => str_replace('_', ' ', $k) . ': ' . $v)
- ->implode(', ');
- @endphp
- {{ $details }}
- |
@empty
diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh
index 648849d5c..f32db9b8d 100644
--- a/scripts/upgrade.sh
+++ b/scripts/upgrade.sh
@@ -141,6 +141,15 @@ else
log "Network 'coolify' already exists"
fi
+# Fix SSH directory ownership if not owned by container user UID 9999 (fixes #6621)
+# Only changes owner — preserves existing group to respect custom setups
+SSH_OWNER=$(stat -c '%u' /data/coolify/ssh 2>/dev/null || echo "unknown")
+if [ "$SSH_OWNER" != "9999" ]; then
+ log "Fixing SSH directory ownership (was owned by UID $SSH_OWNER)"
+ chown -R 9999 /data/coolify/ssh
+ chmod -R 700 /data/coolify/ssh
+fi
+
# Check if Docker config file exists
DOCKER_CONFIG_MOUNT=""
if [ -f /root/.docker/config.json ]; then
diff --git a/tests/Feature/CleanupUnreachableServersTest.php b/tests/Feature/CleanupUnreachableServersTest.php
new file mode 100644
index 000000000..edfd0511c
--- /dev/null
+++ b/tests/Feature/CleanupUnreachableServersTest.php
@@ -0,0 +1,73 @@
+= 3 after 7 days', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'unreachable_count' => 50,
+ 'unreachable_notification_sent' => true,
+ 'updated_at' => now()->subDays(8),
+ ]);
+
+ $this->artisan('cleanup:unreachable-servers')->assertSuccessful();
+
+ $server->refresh();
+ expect($server->ip)->toBe('1.2.3.4');
+});
+
+it('does not clean up servers with unreachable_count less than 3', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'unreachable_count' => 2,
+ 'unreachable_notification_sent' => true,
+ 'updated_at' => now()->subDays(8),
+ ]);
+
+ $originalIp = $server->ip;
+
+ $this->artisan('cleanup:unreachable-servers')->assertSuccessful();
+
+ $server->refresh();
+ expect($server->ip)->toBe($originalIp);
+});
+
+it('does not clean up servers updated within 7 days', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'unreachable_count' => 10,
+ 'unreachable_notification_sent' => true,
+ 'updated_at' => now()->subDays(3),
+ ]);
+
+ $originalIp = $server->ip;
+
+ $this->artisan('cleanup:unreachable-servers')->assertSuccessful();
+
+ $server->refresh();
+ expect($server->ip)->toBe($originalIp);
+});
+
+it('does not clean up servers without notification sent', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create([
+ 'team_id' => $team->id,
+ 'unreachable_count' => 10,
+ 'unreachable_notification_sent' => false,
+ 'updated_at' => now()->subDays(8),
+ ]);
+
+ $originalIp = $server->ip;
+
+ $this->artisan('cleanup:unreachable-servers')->assertSuccessful();
+
+ $server->refresh();
+ expect($server->ip)->toBe($originalIp);
+});
diff --git a/tests/Feature/PushServerUpdateJobOptimizationTest.php b/tests/Feature/PushServerUpdateJobOptimizationTest.php
new file mode 100644
index 000000000..eb51059db
--- /dev/null
+++ b/tests/Feature/PushServerUpdateJobOptimizationTest.php
@@ -0,0 +1,150 @@
+create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
+
+ $data = [
+ 'containers' => [],
+ 'filesystem_usage_root' => ['used_percentage' => 45],
+ ];
+
+ $job = new PushServerUpdateJob($server, $data);
+ $job->handle();
+
+ Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
+ return $job->server->id === $server->id && $job->percentage === 45;
+ });
+});
+
+it('does not dispatch storage check when disk percentage is unchanged', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
+
+ // Simulate a previous push that cached the percentage
+ Cache::put('storage-check:'.$server->id, 45, 600);
+
+ $data = [
+ 'containers' => [],
+ 'filesystem_usage_root' => ['used_percentage' => 45],
+ ];
+
+ $job = new PushServerUpdateJob($server, $data);
+ $job->handle();
+
+ Queue::assertNotPushed(ServerStorageCheckJob::class);
+});
+
+it('dispatches storage check when disk percentage changes from cached value', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
+
+ // Simulate a previous push that cached 45%
+ Cache::put('storage-check:'.$server->id, 45, 600);
+
+ $data = [
+ 'containers' => [],
+ 'filesystem_usage_root' => ['used_percentage' => 50],
+ ];
+
+ $job = new PushServerUpdateJob($server, $data);
+ $job->handle();
+
+ Queue::assertPushed(ServerStorageCheckJob::class, function ($job) use ($server) {
+ return $job->server->id === $server->id && $job->percentage === 50;
+ });
+});
+
+it('rate-limits ConnectProxyToNetworksJob dispatch to every 10 minutes', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
+ $server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+
+ // First push: should dispatch ConnectProxyToNetworksJob
+ $containersWithProxy = [
+ [
+ 'name' => 'coolify-proxy',
+ 'state' => 'running',
+ 'health_status' => 'healthy',
+ 'labels' => ['coolify.managed' => true],
+ ],
+ ];
+
+ $data = [
+ 'containers' => $containersWithProxy,
+ 'filesystem_usage_root' => ['used_percentage' => 10],
+ ];
+
+ $job = new PushServerUpdateJob($server, $data);
+ $job->handle();
+
+ Queue::assertPushed(ConnectProxyToNetworksJob::class, 1);
+
+ // Second push: should NOT dispatch ConnectProxyToNetworksJob (rate-limited)
+ Queue::fake();
+ $job2 = new PushServerUpdateJob($server, $data);
+ $job2->handle();
+
+ Queue::assertNotPushed(ConnectProxyToNetworksJob::class);
+});
+
+it('dispatches ConnectProxyToNetworksJob again after cache expires', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
+ $server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+
+ $containersWithProxy = [
+ [
+ 'name' => 'coolify-proxy',
+ 'state' => 'running',
+ 'health_status' => 'healthy',
+ 'labels' => ['coolify.managed' => true],
+ ],
+ ];
+
+ $data = [
+ 'containers' => $containersWithProxy,
+ 'filesystem_usage_root' => ['used_percentage' => 10],
+ ];
+
+ // First push
+ $job = new PushServerUpdateJob($server, $data);
+ $job->handle();
+
+ Queue::assertPushed(ConnectProxyToNetworksJob::class, 1);
+
+ // Clear cache to simulate expiration
+ Cache::forget('connect-proxy:'.$server->id);
+
+ // Next push: should dispatch again
+ Queue::fake();
+ $job2 = new PushServerUpdateJob($server, $data);
+ $job2->handle();
+
+ Queue::assertPushed(ConnectProxyToNetworksJob::class, 1);
+});
+
+it('uses default queue for PushServerUpdateJob', function () {
+ $team = Team::factory()->create();
+ $server = Server::factory()->create(['team_id' => $team->id]);
+
+ $job = new PushServerUpdateJob($server, ['containers' => []]);
+
+ expect($job->queue)->toBeNull();
+});
diff --git a/tests/Feature/ScheduledJobManagerShouldRunNowTest.php b/tests/Feature/ScheduledJobManagerShouldRunNowTest.php
new file mode 100644
index 000000000..f820c3777
--- /dev/null
+++ b/tests/Feature/ScheduledJobManagerShouldRunNowTest.php
@@ -0,0 +1,222 @@
+getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1');
+
+ expect($result)->toBeTrue();
+});
+
+it('catches delayed job when cache has a baseline from previous run', function () {
+ // Simulate a previous dispatch yesterday at 02:00
+ Cache::put('test-backup:1', Carbon::create(2026, 2, 27, 2, 0, 0, 'UTC')->toIso8601String(), 86400);
+
+ // Freeze time at 02:07 — job was delayed 7 minutes past today's 02:00 cron
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 7, 0, 'UTC'));
+
+ $job = new ScheduledJobManager;
+
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ // isDue() would return false at 02:07, but getPreviousRunDate() = 02:00 today
+ // lastDispatched = 02:00 yesterday → 02:00 today > yesterday → fires
+ $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:1');
+
+ expect($result)->toBeTrue();
+});
+
+it('does not double-dispatch on subsequent runs within same cron window', function () {
+ // First run at 02:00 — dispatches and sets cache
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
+
+ $job = new ScheduledJobManager;
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ $first = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2');
+ expect($first)->toBeTrue();
+
+ // Second run at 02:01 — should NOT dispatch (previousDue=02:00, lastDispatched=02:00)
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $second = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:2');
+ expect($second)->toBeFalse();
+});
+
+it('fires every_minute cron correctly on consecutive minutes', function () {
+ $job = new ScheduledJobManager;
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ // Minute 1
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
+ $executionTimeProp->setValue($job, Carbon::now());
+ $result1 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
+ expect($result1)->toBeTrue();
+
+ // Minute 2
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 1, 0, 'UTC'));
+ $executionTimeProp->setValue($job, Carbon::now());
+ $result2 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
+ expect($result2)->toBeTrue();
+
+ // Minute 3
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 2, 0, 'UTC'));
+ $executionTimeProp->setValue($job, Carbon::now());
+ $result3 = $method->invoke($job, '* * * * *', 'UTC', 'test-backup:3');
+ expect($result3)->toBeTrue();
+});
+
+it('does not fire non-due jobs on restart when cache is empty', function () {
+ // Time is 10:00, cron is daily at 02:00 — NOT due right now
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
+
+ $job = new ScheduledJobManager;
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ // Cache is empty (fresh restart) — should NOT fire daily backup at 10:00
+ $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4');
+ expect($result)->toBeFalse();
+});
+
+it('fires due jobs on restart when cache is empty', function () {
+ // Time is exactly 02:00, cron is daily at 02:00 — IS due right now
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
+
+ $job = new ScheduledJobManager;
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ // Cache is empty (fresh restart) — but cron IS due → should fire
+ $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:4b');
+ expect($result)->toBeTrue();
+});
+
+it('does not dispatch when cron is not due and was not recently due', function () {
+ // Time is 10:00, cron is daily at 02:00 — last due was 8 hours ago
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 10, 0, 0, 'UTC'));
+
+ $job = new ScheduledJobManager;
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ // previousDue = 02:00, but lastDispatched was set at 02:00 (simulate)
+ Cache::put('test-backup:5', Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC')->toIso8601String(), 86400);
+
+ $result = $method->invoke($job, '0 2 * * *', 'UTC', 'test-backup:5');
+ expect($result)->toBeFalse();
+});
+
+it('falls back to isDue when no dedup key is provided', function () {
+ // Time is exactly 02:00, cron is "0 2 * * *" — should be due
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 0, 0, 'UTC'));
+
+ $job = new ScheduledJobManager;
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ // No dedup key → simple isDue check
+ $result = $method->invoke($job, '0 2 * * *', 'UTC');
+ expect($result)->toBeTrue();
+
+ // At 02:01 without dedup key → isDue returns false
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 2, 1, 0, 'UTC'));
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $result2 = $method->invoke($job, '0 2 * * *', 'UTC');
+ expect($result2)->toBeFalse();
+});
+
+it('respects server timezone for cron evaluation', function () {
+ // UTC time is 22:00 Feb 28, which is 06:00 Mar 1 in Asia/Singapore (+8)
+ Carbon::setTestNow(Carbon::create(2026, 2, 28, 22, 0, 0, 'UTC'));
+
+ $job = new ScheduledJobManager;
+ $reflection = new ReflectionClass($job);
+
+ $executionTimeProp = $reflection->getProperty('executionTime');
+ $executionTimeProp->setAccessible(true);
+ $executionTimeProp->setValue($job, Carbon::now());
+
+ $method = $reflection->getMethod('shouldRunNow');
+ $method->setAccessible(true);
+
+ // Simulate that today's 06:00 UTC run was already dispatched (at 06:00 UTC)
+ Cache::put('test-backup:7', Carbon::create(2026, 2, 28, 6, 0, 0, 'UTC')->toIso8601String(), 86400);
+
+ // Cron "0 6 * * *" in Asia/Singapore: local time is 06:00 Mar 1 → previousDue = 06:00 Mar 1 SGT
+ // That's a NEW cron window (Mar 1) that hasn't been dispatched → should fire
+ $resultSingapore = $method->invoke($job, '0 6 * * *', 'Asia/Singapore', 'test-backup:6');
+ expect($resultSingapore)->toBeTrue();
+
+ // Cron "0 6 * * *" in UTC: previousDue = 06:00 Feb 28 UTC, already dispatched at 06:00 → should NOT fire
+ $resultUtc = $method->invoke($job, '0 6 * * *', 'UTC', 'test-backup:7');
+ expect($resultUtc)->toBeFalse();
+});
diff --git a/tests/Feature/ScheduledJobMonitoringTest.php b/tests/Feature/ScheduledJobMonitoringTest.php
index 1348375d4..036c3b638 100644
--- a/tests/Feature/ScheduledJobMonitoringTest.php
+++ b/tests/Feature/ScheduledJobMonitoringTest.php
@@ -173,6 +173,42 @@
@unlink($logPath);
});
+test('scheduler log parser excludes started events from runs', function () {
+ $logPath = storage_path('logs/scheduled-test-started-filter.log');
+ $logDir = dirname($logPath);
+ if (! is_dir($logDir)) {
+ mkdir($logDir, 0755, true);
+ }
+
+ // Temporarily rename existing logs so they don't interfere
+ $existingLogs = glob(storage_path('logs/scheduled-*.log'));
+ $renamed = [];
+ foreach ($existingLogs as $log) {
+ $tmp = $log.'.bak';
+ rename($log, $tmp);
+ $renamed[$tmp] = $log;
+ }
+
+ $logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log');
+ $lines = [
+ '['.now()->format('Y-m-d H:i:s').'] production.INFO: ScheduledJobManager started {}',
+ '['.now()->format('Y-m-d H:i:s').'] production.INFO: ScheduledJobManager completed {"duration_ms":74,"dispatched":1,"skipped":13}',
+ ];
+ file_put_contents($logPath, implode("\n", $lines)."\n");
+
+ $parser = new SchedulerLogParser;
+ $runs = $parser->getRecentRuns();
+
+ expect($runs)->toHaveCount(1);
+ expect($runs->first()['message'])->toContain('completed');
+
+ // Cleanup
+ @unlink($logPath);
+ foreach ($renamed as $tmp => $original) {
+ rename($tmp, $original);
+ }
+});
+
test('scheduler log parser filters by team id', function () {
$logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log');
$logDir = dirname($logPath);
@@ -198,3 +234,39 @@
// Cleanup
@unlink($logPath);
});
+
+test('skipped jobs show fallback when resource is deleted', function () {
+ $this->actingAs($this->rootUser);
+ session(['currentTeam' => $this->rootTeam]);
+
+ $logPath = storage_path('logs/scheduled-'.now()->format('Y-m-d').'.log');
+ $logDir = dirname($logPath);
+ if (! is_dir($logDir)) {
+ mkdir($logDir, 0755, true);
+ }
+
+ // Temporarily rename existing logs so they don't interfere
+ $existingLogs = glob(storage_path('logs/scheduled-*.log'));
+ $renamed = [];
+ foreach ($existingLogs as $log) {
+ $tmp = $log.'.bak';
+ rename($log, $tmp);
+ $renamed[$tmp] = $log;
+ }
+
+ $lines = [
+ '['.now()->format('Y-m-d H:i:s').'] production.INFO: Task skipped {"type":"task","skip_reason":"application_not_running","task_id":99999,"task_name":"my-cron-job","team_id":0}',
+ ];
+ file_put_contents($logPath, implode("\n", $lines)."\n");
+
+ Livewire::test(ScheduledJobs::class)
+ ->assertStatus(200)
+ ->assertSee('my-cron-job')
+ ->assertSee('Application not running');
+
+ // Cleanup
+ @unlink($logPath);
+ foreach ($renamed as $tmp => $original) {
+ rename($tmp, $original);
+ }
+});
diff --git a/tests/Unit/PrivateKeyStorageTest.php b/tests/Unit/PrivateKeyStorageTest.php
index 00f39e3df..09472604b 100644
--- a/tests/Unit/PrivateKeyStorageTest.php
+++ b/tests/Unit/PrivateKeyStorageTest.php
@@ -112,7 +112,7 @@ public function it_throws_exception_when_storage_directory_is_not_writable()
);
$this->expectException(\Exception::class);
- $this->expectExceptionMessage('SSH keys storage directory is not writable');
+ $this->expectExceptionMessage('SSH keys storage directory is not writable. Run on the host: sudo chown -R 9999 /data/coolify/ssh && sudo chmod -R 700 /data/coolify/ssh && docker restart coolify');
PrivateKey::createAndStore([
'name' => 'Test Key',
diff --git a/tests/Unit/ServerManagerJobSentinelCheckTest.php b/tests/Unit/ServerManagerJobSentinelCheckTest.php
index 0f2613f11..d8449adc3 100644
--- a/tests/Unit/ServerManagerJobSentinelCheckTest.php
+++ b/tests/Unit/ServerManagerJobSentinelCheckTest.php
@@ -1,6 +1,7 @@
instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
- // Create a mock server with sentinel enabled
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(true);
+ $server->shouldReceive('isSentinelLive')->andReturn(true);
$server->id = 1;
$server->name = 'test-server';
$server->ip = '192.168.1.100';
@@ -34,29 +34,76 @@
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
- // Mock the Server query
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
- // Execute the job
$job = new ServerManagerJob;
$job->handle();
- // Assert CheckAndStartSentinelJob was dispatched for the sentinel-enabled server
- Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
- return $job->server->id === $server->id;
- });
+ // Hourly CheckAndStartSentinelJob dispatch was removed — ServerCheckJob handles it when Sentinel is out of sync
+ Queue::assertNotPushed(CheckAndStartSentinelJob::class);
});
-it('does not dispatch CheckAndStartSentinelJob for servers without sentinel enabled', function () {
- // Mock InstanceSettings
+it('skips ServerConnectionCheckJob when sentinel is live', function () {
+ $settings = Mockery::mock(InstanceSettings::class);
+ $settings->instance_timezone = 'UTC';
+ $this->app->instance(InstanceSettings::class, $settings);
+
+ $server = Mockery::mock(Server::class)->makePartial();
+ $server->shouldReceive('isSentinelEnabled')->andReturn(true);
+ $server->shouldReceive('isSentinelLive')->andReturn(true);
+ $server->id = 1;
+ $server->name = 'test-server';
+ $server->ip = '192.168.1.100';
+ $server->sentinel_updated_at = Carbon::now();
+ $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
+ $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
+
+ Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
+ Server::shouldReceive('get')->andReturn(collect([$server]));
+
+ $job = new ServerManagerJob;
+ $job->handle();
+
+ // Sentinel is healthy so SSH connection check is skipped
+ Queue::assertNotPushed(ServerConnectionCheckJob::class);
+});
+
+it('dispatches ServerConnectionCheckJob when sentinel is not live', function () {
+ $settings = Mockery::mock(InstanceSettings::class);
+ $settings->instance_timezone = 'UTC';
+ $this->app->instance(InstanceSettings::class, $settings);
+
+ $server = Mockery::mock(Server::class)->makePartial();
+ $server->shouldReceive('isSentinelEnabled')->andReturn(true);
+ $server->shouldReceive('isSentinelLive')->andReturn(false);
+ $server->id = 1;
+ $server->name = 'test-server';
+ $server->ip = '192.168.1.100';
+ $server->sentinel_updated_at = Carbon::now()->subMinutes(10);
+ $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
+ $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
+
+ Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
+ Server::shouldReceive('get')->andReturn(collect([$server]));
+
+ $job = new ServerManagerJob;
+ $job->handle();
+
+ // Sentinel is out of sync so SSH connection check is needed
+ Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
+ return $job->server->id === $server->id;
+ });
+});
+
+it('dispatches ServerConnectionCheckJob when sentinel is not enabled', function () {
$settings = Mockery::mock(InstanceSettings::class);
$settings->instance_timezone = 'UTC';
$this->app->instance(InstanceSettings::class, $settings);
- // Create a mock server with sentinel disabled
$server = Mockery::mock(Server::class)->makePartial();
$server->shouldReceive('isSentinelEnabled')->andReturn(false);
+ $server->shouldReceive('isSentinelLive')->never();
$server->id = 2;
$server->name = 'test-server-no-sentinel';
$server->ip = '192.168.1.101';
@@ -64,78 +111,14 @@
$server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
$server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
- // Mock the Server query
Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
Server::shouldReceive('get')->andReturn(collect([$server]));
- // Execute the job
$job = new ServerManagerJob;
$job->handle();
- // Assert CheckAndStartSentinelJob was NOT dispatched
- Queue::assertNotPushed(CheckAndStartSentinelJob::class);
-});
-
-it('respects server timezone when scheduling sentinel checks', function () {
- // Mock InstanceSettings
- $settings = Mockery::mock(InstanceSettings::class);
- $settings->instance_timezone = 'UTC';
- $this->app->instance(InstanceSettings::class, $settings);
-
- // Set test time to top of hour in America/New_York (which is 17:00 UTC)
- Carbon::setTestNow('2025-01-15 17:00:00'); // 12:00 PM EST (top of hour in EST)
-
- // Create a mock server with sentinel enabled and America/New_York timezone
- $server = Mockery::mock(Server::class)->makePartial();
- $server->shouldReceive('isSentinelEnabled')->andReturn(true);
- $server->id = 3;
- $server->name = 'test-server-est';
- $server->ip = '192.168.1.102';
- $server->sentinel_updated_at = Carbon::now();
- $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'America/New_York']);
- $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
-
- // Mock the Server query
- Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
- Server::shouldReceive('get')->andReturn(collect([$server]));
-
- // Execute the job
- $job = new ServerManagerJob;
- $job->handle();
-
- // Assert CheckAndStartSentinelJob was dispatched (should run at top of hour in server's timezone)
- Queue::assertPushed(CheckAndStartSentinelJob::class, function ($job) use ($server) {
+ // Sentinel is not enabled so SSH connection check must run
+ Queue::assertPushed(ServerConnectionCheckJob::class, function ($job) use ($server) {
return $job->server->id === $server->id;
});
});
-
-it('does not dispatch sentinel check when not at top of hour', function () {
- // Mock InstanceSettings
- $settings = Mockery::mock(InstanceSettings::class);
- $settings->instance_timezone = 'UTC';
- $this->app->instance(InstanceSettings::class, $settings);
-
- // Set test time to middle of the hour (not top of hour)
- Carbon::setTestNow('2025-01-15 12:30:00');
-
- // Create a mock server with sentinel enabled
- $server = Mockery::mock(Server::class)->makePartial();
- $server->shouldReceive('isSentinelEnabled')->andReturn(true);
- $server->id = 4;
- $server->name = 'test-server-mid-hour';
- $server->ip = '192.168.1.103';
- $server->sentinel_updated_at = Carbon::now();
- $server->shouldReceive('getAttribute')->with('settings')->andReturn((object) ['server_timezone' => 'UTC']);
- $server->shouldReceive('waitBeforeDoingSshCheck')->andReturn(120);
-
- // Mock the Server query
- Server::shouldReceive('where')->with('ip', '!=', '1.2.3.4')->andReturnSelf();
- Server::shouldReceive('get')->andReturn(collect([$server]));
-
- // Execute the job
- $job = new ServerManagerJob;
- $job->handle();
-
- // Assert CheckAndStartSentinelJob was NOT dispatched (not top of hour)
- Queue::assertNotPushed(CheckAndStartSentinelJob::class);
-});
-
diff --git a/app/Console/Commands/CleanupUnreachableServers.php b/app/Console/Commands/CleanupUnreachableServers.php
index def01b265..09563a2c3 100644
--- a/app/Console/Commands/CleanupUnreachableServers.php
+++ b/app/Console/Commands/CleanupUnreachableServers.php
@@ -14,7 +14,7 @@ class CleanupUnreachableServers extends Command
public function handle()
{
echo "Running unreachable server cleanup...\n";
- $servers = Server::where('unreachable_count', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
+ $servers = Server::where('unreachable_count', '>=', 3)->where('unreachable_notification_sent', true)->where('updated_at', '<', now()->subDays(7))->get();
if ($servers->count() > 0) {
foreach ($servers as $server) {
echo "Cleanup unreachable server ($server->id) with name $server->name";
diff --git a/app/Console/Commands/Cloud/SyncStripeSubscriptions.php b/app/Console/Commands/Cloud/SyncStripeSubscriptions.php
index e64f86926..46f6b4edd 100644
--- a/app/Console/Commands/Cloud/SyncStripeSubscriptions.php
+++ b/app/Console/Commands/Cloud/SyncStripeSubscriptions.php
@@ -36,7 +36,14 @@ public function handle(): int
$this->newLine();
$job = new SyncStripeSubscriptionsJob($fix);
- $result = $job->handle();
+ $fetched = 0;
+ $result = $job->handle(function (int $count) use (&$fetched): void {
+ $fetched = $count;
+ $this->output->write("\r Fetching subscriptions from Stripe... {$fetched}");
+ });
+ if ($fetched > 0) {
+ $this->output->write("\r".str_repeat(' ', 60)."\r");
+ }
if (isset($result['error'])) {
$this->error($result['error']);
@@ -68,6 +75,19 @@ public function handle(): int
$this->info('No discrepancies found. All subscriptions are in sync.');
}
+ if (count($result['resubscribed']) > 0) {
+ $this->newLine();
+ $this->warn('Resubscribed users (same email, different customer): '.count($result['resubscribed']));
+ $this->newLine();
+
+ foreach ($result['resubscribed'] as $resub) {
+ $this->line(" - Team ID: {$resub['team_id']} | Email: {$resub['email']}");
+ $this->line(" Old: {$resub['old_stripe_subscription_id']} (cus: {$resub['old_stripe_customer_id']})");
+ $this->line(" New: {$resub['new_stripe_subscription_id']} (cus: {$resub['new_stripe_customer_id']}) [{$resub['new_status']}]");
+ $this->newLine();
+ }
+ }
+
if (count($result['errors']) > 0) {
$this->newLine();
$this->error('Errors encountered: '.count($result['errors']));
diff --git a/app/Jobs/DatabaseBackupJob.php b/app/Jobs/DatabaseBackupJob.php
index a585baa69..5fc9f6cd8 100644
--- a/app/Jobs/DatabaseBackupJob.php
+++ b/app/Jobs/DatabaseBackupJob.php
@@ -111,6 +111,12 @@ public function handle(): void
$status = str(data_get($this->database, 'status'));
if (! $status->startsWith('running') && $this->database->id !== 0) {
+ Log::info('DatabaseBackupJob skipped: database not running', [
+ 'backup_id' => $this->backup->id,
+ 'database_id' => $this->database->id,
+ 'status' => (string) $status,
+ ]);
+
return;
}
if (data_get($this->backup, 'database_type') === \App\Models\ServiceDatabase::class) {
@@ -472,7 +478,7 @@ private function backup_standalone_mongodb(string $databaseWithCollections): voi
throw new \Exception('MongoDB credentials not found. Ensure MONGO_INITDB_ROOT_USERNAME and MONGO_INITDB_ROOT_PASSWORD environment variables are available in the container.');
}
}
- \Log::info('MongoDB backup URL configured', ['has_url' => filled($url), 'using_env_vars' => blank($this->database->internal_db_url)]);
+ Log::info('MongoDB backup URL configured', ['has_url' => filled($url), 'using_env_vars' => blank($this->database->internal_db_url)]);
if ($databaseWithCollections === 'all') {
$commands[] = 'mkdir -p '.$this->backup_dir;
if (str($this->database->image)->startsWith('mongo:4')) {
diff --git a/app/Jobs/DockerCleanupJob.php b/app/Jobs/DockerCleanupJob.php
index f3f3a2ae4..78ef7f3a2 100644
--- a/app/Jobs/DockerCleanupJob.php
+++ b/app/Jobs/DockerCleanupJob.php
@@ -91,6 +91,8 @@ public function handle(): void
$this->server->team?->notify(new DockerCleanupSuccess($this->server, $message));
event(new DockerCleanupDone($this->execution_log));
+
+ return;
}
if ($this->usageBefore >= $this->server->settings->docker_cleanup_threshold) {
diff --git a/app/Jobs/PushServerUpdateJob.php b/app/Jobs/PushServerUpdateJob.php
index 5d018cf19..85684ff19 100644
--- a/app/Jobs/PushServerUpdateJob.php
+++ b/app/Jobs/PushServerUpdateJob.php
@@ -24,6 +24,7 @@
use Illuminate\Queue\Middleware\WithoutOverlapping;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\Cache;
use Laravel\Horizon\Contracts\Silenced;
class PushServerUpdateJob implements ShouldBeEncrypted, ShouldQueue, Silenced
@@ -130,7 +131,14 @@ public function handle()
$this->containers = collect(data_get($data, 'containers'));
$filesystemUsageRoot = data_get($data, 'filesystem_usage_root.used_percentage');
- ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
+
+ // Only dispatch storage check when disk percentage actually changes
+ $storageCacheKey = 'storage-check:'.$this->server->id;
+ $lastPercentage = Cache::get($storageCacheKey);
+ if ($lastPercentage === null || (string) $lastPercentage !== (string) $filesystemUsageRoot) {
+ Cache::put($storageCacheKey, $filesystemUsageRoot, 600);
+ ServerStorageCheckJob::dispatch($this->server, $filesystemUsageRoot);
+ }
if ($this->containers->isEmpty()) {
return;
@@ -207,7 +215,7 @@ public function handle()
$serviceId = $labels->get('coolify.serviceId');
$subType = $labels->get('coolify.service.subType');
$subId = $labels->get('coolify.service.subId');
- if (empty($subId)) {
+ if (empty(trim((string) $subId))) {
continue;
}
if ($subType === 'application') {
@@ -327,6 +335,10 @@ private function aggregateServiceContainerStatuses()
// Parse key: serviceId:subType:subId
[$serviceId, $subType, $subId] = explode(':', $key);
+ if (empty($subId)) {
+ continue;
+ }
+
$service = $this->services->where('id', $serviceId)->first();
if (! $service) {
continue;
@@ -335,9 +347,9 @@ private function aggregateServiceContainerStatuses()
// Get the service sub-resource (ServiceApplication or ServiceDatabase)
$subResource = null;
if ($subType === 'application') {
- $subResource = $service->applications()->where('id', $subId)->first();
+ $subResource = $service->applications->where('id', $subId)->first();
} elseif ($subType === 'database') {
- $subResource = $service->databases()->where('id', $subId)->first();
+ $subResource = $service->databases->where('id', $subId)->first();
}
if (! $subResource) {
@@ -476,8 +488,13 @@ private function updateProxyStatus()
} catch (\Throwable $e) {
}
} else {
- // Connect proxy to networks asynchronously to avoid blocking the status update
- ConnectProxyToNetworksJob::dispatch($this->server);
+ // Connect proxy to networks periodically (every 10 min) to avoid excessive job dispatches.
+ // On-demand triggers (new network, service deploy) use dispatchSync() and bypass this.
+ $proxyCacheKey = 'connect-proxy:'.$this->server->id;
+ if (! Cache::has($proxyCacheKey)) {
+ Cache::put($proxyCacheKey, true, 600);
+ ConnectProxyToNetworksJob::dispatch($this->server);
+ }
}
}
}
@@ -545,7 +562,7 @@ private function updateServiceSubStatus(string $serviceId, string $subType, stri
return;
}
if ($subType === 'application') {
- $application = $service->applications()->where('id', $subId)->first();
+ $application = $service->applications->where('id', $subId)->first();
if ($application) {
if ($application->status !== $containerStatus) {
$application->status = $containerStatus;
@@ -553,7 +570,7 @@ private function updateServiceSubStatus(string $serviceId, string $subType, stri
}
}
} elseif ($subType === 'database') {
- $database = $service->databases()->where('id', $subId)->first();
+ $database = $service->databases->where('id', $subId)->first();
if ($database) {
if ($database->status !== $containerStatus) {
$database->status = $containerStatus;
diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php
index de782b96d..e68e3b613 100644
--- a/app/Jobs/ScheduledJobManager.php
+++ b/app/Jobs/ScheduledJobManager.php
@@ -160,7 +160,8 @@ private function processScheduledBackups(): void
foreach ($backups as $backup) {
try {
- $skipReason = $this->getBackupSkipReason($backup);
+ $server = $backup->server();
+ $skipReason = $this->getBackupSkipReason($backup, $server);
if ($skipReason !== null) {
$this->skippedCount++;
$this->logSkip('backup', $skipReason, [
@@ -173,7 +174,6 @@ private function processScheduledBackups(): void
continue;
}
- $server = $backup->server();
$serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
if (validate_timezone($serverTimezone) === false) {
@@ -185,7 +185,7 @@ private function processScheduledBackups(): void
$frequency = VALID_CRON_STRINGS[$frequency];
}
- if ($this->shouldRunNow($frequency, $serverTimezone)) {
+ if ($this->shouldRunNow($frequency, $serverTimezone, "scheduled-backup:{$backup->id}")) {
DatabaseBackupJob::dispatch($backup);
$this->dispatchedCount++;
Log::channel('scheduled')->info('Backup dispatched', [
@@ -213,19 +213,21 @@ private function processScheduledTasks(): void
foreach ($tasks as $task) {
try {
- $skipReason = $this->getTaskSkipReason($task);
- if ($skipReason !== null) {
+ $server = $task->server();
+
+ // Phase 1: Critical checks (always — cheap, handles orphans and infra issues)
+ $criticalSkip = $this->getTaskCriticalSkipReason($task, $server);
+ if ($criticalSkip !== null) {
$this->skippedCount++;
- $this->logSkip('task', $skipReason, [
+ $this->logSkip('task', $criticalSkip, [
'task_id' => $task->id,
'task_name' => $task->name,
- 'team_id' => $task->server()?->team_id,
+ 'team_id' => $server?->team_id,
]);
continue;
}
- $server = $task->server();
$serverTimezone = data_get($server->settings, 'server_timezone', config('app.timezone'));
if (validate_timezone($serverTimezone) === false) {
@@ -237,16 +239,31 @@ private function processScheduledTasks(): void
$frequency = VALID_CRON_STRINGS[$frequency];
}
- if ($this->shouldRunNow($frequency, $serverTimezone)) {
- ScheduledTaskJob::dispatch($task);
- $this->dispatchedCount++;
- Log::channel('scheduled')->info('Task dispatched', [
+ if (! $this->shouldRunNow($frequency, $serverTimezone, "scheduled-task:{$task->id}")) {
+ continue;
+ }
+
+ // Phase 2: Runtime checks (only when cron is due — avoids noise for stopped resources)
+ $runtimeSkip = $this->getTaskRuntimeSkipReason($task);
+ if ($runtimeSkip !== null) {
+ $this->skippedCount++;
+ $this->logSkip('task', $runtimeSkip, [
'task_id' => $task->id,
'task_name' => $task->name,
'team_id' => $server->team_id,
- 'server_id' => $server->id,
]);
+
+ continue;
}
+
+ ScheduledTaskJob::dispatch($task);
+ $this->dispatchedCount++;
+ Log::channel('scheduled')->info('Task dispatched', [
+ 'task_id' => $task->id,
+ 'task_name' => $task->name,
+ 'team_id' => $server->team_id,
+ 'server_id' => $server->id,
+ ]);
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Error processing task', [
'task_id' => $task->id,
@@ -256,7 +273,7 @@ private function processScheduledTasks(): void
}
}
- private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
+ private function getBackupSkipReason(ScheduledDatabaseBackup $backup, ?Server $server): ?string
{
if (blank(data_get($backup, 'database'))) {
$backup->delete();
@@ -264,7 +281,6 @@ private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
return 'database_deleted';
}
- $server = $backup->server();
if (blank($server)) {
$backup->delete();
@@ -282,12 +298,8 @@ private function getBackupSkipReason(ScheduledDatabaseBackup $backup): ?string
return null;
}
- private function getTaskSkipReason(ScheduledTask $task): ?string
+ private function getTaskCriticalSkipReason(ScheduledTask $task, ?Server $server): ?string
{
- $service = $task->service;
- $application = $task->application;
-
- $server = $task->server();
if (blank($server)) {
$task->delete();
@@ -302,33 +314,71 @@ private function getTaskSkipReason(ScheduledTask $task): ?string
return 'subscription_unpaid';
}
- if (! $service && ! $application) {
+ if (! $task->service && ! $task->application) {
$task->delete();
return 'resource_deleted';
}
- if ($application && str($application->status)->contains('running') === false) {
+ return null;
+ }
+
+ private function getTaskRuntimeSkipReason(ScheduledTask $task): ?string
+ {
+ if ($task->application && str($task->application->status)->contains('running') === false) {
return 'application_not_running';
}
- if ($service && str($service->status)->contains('running') === false) {
+ if ($task->service && str($task->service->status)->contains('running') === false) {
return 'service_not_running';
}
return null;
}
- private function shouldRunNow(string $frequency, string $timezone): bool
+ /**
+ * Determine if a cron schedule should run now.
+ *
+ * When a dedupKey is provided, uses getPreviousRunDate() + last-dispatch tracking
+ * instead of isDue(). This is resilient to queue delays — even if the job is delayed
+ * by minutes, it still catches the missed cron window. Without dedupKey, falls back
+ * to simple isDue() check.
+ */
+ private function shouldRunNow(string $frequency, string $timezone, ?string $dedupKey = null): bool
{
$cron = new CronExpression($frequency);
-
- // Use the frozen execution time, not the current time
- // Fallback to current time if execution time is not set (shouldn't happen)
$baseTime = $this->executionTime ?? Carbon::now();
$executionTime = $baseTime->copy()->setTimezone($timezone);
- return $cron->isDue($executionTime);
+ // No dedup key → simple isDue check (used by docker cleanups)
+ if ($dedupKey === null) {
+ return $cron->isDue($executionTime);
+ }
+
+ // Get the most recent time this cron was due (including current minute)
+ $previousDue = Carbon::instance($cron->getPreviousRunDate($executionTime, allowCurrentDate: true));
+
+ $lastDispatched = Cache::get($dedupKey);
+
+ if ($lastDispatched === null) {
+ // First run after restart or cache loss: only fire if actually due right now.
+ // Seed the cache so subsequent runs can use tolerance/catch-up logic.
+ $isDue = $cron->isDue($executionTime);
+ if ($isDue) {
+ Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
+ }
+
+ return $isDue;
+ }
+
+ // Subsequent runs: fire if there's been a due time since last dispatch
+ if ($previousDue->gt(Carbon::parse($lastDispatched))) {
+ Cache::put($dedupKey, $executionTime->toIso8601String(), 86400);
+
+ return true;
+ }
+
+ return false;
}
private function processDockerCleanups(): void
diff --git a/app/Jobs/ScheduledTaskJob.php b/app/Jobs/ScheduledTaskJob.php
index b21bc11a1..49b9b9702 100644
--- a/app/Jobs/ScheduledTaskJob.php
+++ b/app/Jobs/ScheduledTaskJob.php
@@ -14,13 +14,14 @@
use App\Notifications\ScheduledTask\TaskSuccess;
use Carbon\Carbon;
use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
-class ScheduledTaskJob implements ShouldQueue
+class ScheduledTaskJob implements ShouldBeEncrypted, ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php
index a4619354d..c8219a2ea 100644
--- a/app/Jobs/ServerManagerJob.php
+++ b/app/Jobs/ServerManagerJob.php
@@ -64,11 +64,11 @@ public function handle(): void
private function getServers(): Collection
{
- $allServers = Server::where('ip', '!=', '1.2.3.4');
+ $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
if (isCloud()) {
$servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
- $own = Team::find(0)->servers;
+ $own = Team::find(0)->servers()->with('settings')->get();
return $servers->merge($own);
} else {
@@ -82,6 +82,10 @@ private function dispatchConnectionChecks(Collection $servers): void
if ($this->shouldRunNow($this->checkFrequency)) {
$servers->each(function (Server $server) {
try {
+ // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
+ if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
+ return;
+ }
ServerConnectionCheckJob::dispatch($server);
} catch (\Exception $e) {
Log::channel('scheduled-errors')->error('Failed to dispatch ServerConnectionCheck', [
@@ -134,9 +138,7 @@ private function processServerTasks(Server $server): void
// Dispatch Sentinel restart if due (daily for Sentinel-enabled servers)
if ($shouldRestartSentinel) {
- dispatch(function () use ($server) {
- $server->restartContainer('coolify-sentinel');
- });
+ CheckAndStartSentinelJob::dispatch($server);
}
// Dispatch ServerStorageCheckJob if due (only when Sentinel is out of sync or disabled)
@@ -160,11 +162,8 @@ private function processServerTasks(Server $server): void
ServerPatchCheckJob::dispatch($server);
}
- // Sentinel update checks (hourly) - check for updates to Sentinel version
- // No timezone needed for hourly - runs at top of every hour
- if ($isSentinelEnabled && $this->shouldRunNow('0 * * * *')) {
- CheckAndStartSentinelJob::dispatch($server);
- }
+ // Note: CheckAndStartSentinelJob is only dispatched daily (line above) for version updates.
+ // Crash recovery is handled by sentinelOutOfSync → ServerCheckJob → CheckAndStartSentinelJob.
}
private function shouldRunNow(string $frequency, ?string $timezone = null): bool
diff --git a/app/Jobs/SyncStripeSubscriptionsJob.php b/app/Jobs/SyncStripeSubscriptionsJob.php
index 9eb946e4d..4301a80d1 100644
--- a/app/Jobs/SyncStripeSubscriptionsJob.php
+++ b/app/Jobs/SyncStripeSubscriptionsJob.php
@@ -22,7 +22,7 @@ public function __construct(public bool $fix = false)
$this->onQueue('high');
}
- public function handle(): array
+ public function handle(?\Closure $onProgress = null): array
{
if (! isCloud() || ! isStripe()) {
return ['error' => 'Not running on Cloud or Stripe not configured'];
@@ -33,48 +33,73 @@ public function handle(): array
->get();
$stripe = new \Stripe\StripeClient(config('subscription.stripe_api_key'));
+
+ // Bulk fetch all valid subscription IDs from Stripe (active + past_due)
+ $validStripeIds = $this->fetchValidStripeSubscriptionIds($stripe, $onProgress);
+
+ // Find DB subscriptions not in the valid set
+ $staleSubscriptions = $subscriptions->filter(
+ fn (Subscription $sub) => ! in_array($sub->stripe_subscription_id, $validStripeIds)
+ );
+
+ // For each stale subscription, get the exact Stripe status and check for resubscriptions
$discrepancies = [];
+ $resubscribed = [];
$errors = [];
- foreach ($subscriptions as $subscription) {
+ foreach ($staleSubscriptions as $subscription) {
try {
$stripeSubscription = $stripe->subscriptions->retrieve(
$subscription->stripe_subscription_id
);
+ $stripeStatus = $stripeSubscription->status;
- // Check if Stripe says cancelled but we think it's active
- if (in_array($stripeSubscription->status, ['canceled', 'incomplete_expired', 'unpaid'])) {
- $discrepancies[] = [
- 'subscription_id' => $subscription->id,
- 'team_id' => $subscription->team_id,
- 'stripe_subscription_id' => $subscription->stripe_subscription_id,
- 'stripe_status' => $stripeSubscription->status,
- ];
-
- // Only fix if --fix flag is passed
- if ($this->fix) {
- $subscription->update([
- 'stripe_invoice_paid' => false,
- 'stripe_past_due' => false,
- ]);
-
- if ($stripeSubscription->status === 'canceled') {
- $subscription->team?->subscriptionEnded();
- }
- }
- }
-
- // Small delay to avoid Stripe rate limits
- usleep(100000); // 100ms
+ usleep(100000); // 100ms rate limit delay
} catch (\Exception $e) {
$errors[] = [
'subscription_id' => $subscription->id,
'error' => $e->getMessage(),
];
+
+ continue;
+ }
+
+ // Check if this user resubscribed under a different customer/subscription
+ $activeSub = $this->findActiveSubscriptionByEmail($stripe, $stripeSubscription->customer);
+ if ($activeSub) {
+ $resubscribed[] = [
+ 'subscription_id' => $subscription->id,
+ 'team_id' => $subscription->team_id,
+ 'email' => $activeSub['email'],
+ 'old_stripe_subscription_id' => $subscription->stripe_subscription_id,
+ 'old_stripe_customer_id' => $stripeSubscription->customer,
+ 'new_stripe_subscription_id' => $activeSub['subscription_id'],
+ 'new_stripe_customer_id' => $activeSub['customer_id'],
+ 'new_status' => $activeSub['status'],
+ ];
+
+ continue;
+ }
+
+ $discrepancies[] = [
+ 'subscription_id' => $subscription->id,
+ 'team_id' => $subscription->team_id,
+ 'stripe_subscription_id' => $subscription->stripe_subscription_id,
+ 'stripe_status' => $stripeStatus,
+ ];
+
+ if ($this->fix) {
+ $subscription->update([
+ 'stripe_invoice_paid' => false,
+ 'stripe_past_due' => false,
+ ]);
+
+ if ($stripeStatus === 'canceled') {
+ $subscription->team?->subscriptionEnded();
+ }
}
}
- // Only notify if discrepancies found and fixed
if ($this->fix && count($discrepancies) > 0) {
send_internal_notification(
'SyncStripeSubscriptionsJob: Fixed '.count($discrepancies)." discrepancies:\n".
@@ -85,8 +110,88 @@ public function handle(): array
return [
'total_checked' => $subscriptions->count(),
'discrepancies' => $discrepancies,
+ 'resubscribed' => $resubscribed,
'errors' => $errors,
'fixed' => $this->fix,
];
}
+
+ /**
+ * Given a Stripe customer ID, get their email and search for other customers
+ * with the same email that have an active subscription.
+ *
+ * @return array{email: string, customer_id: string, subscription_id: string, status: string}|null
+ */
+ private function findActiveSubscriptionByEmail(\Stripe\StripeClient $stripe, string $customerId): ?array
+ {
+ try {
+ $customer = $stripe->customers->retrieve($customerId);
+ $email = $customer->email;
+
+ if (! $email) {
+ return null;
+ }
+
+ usleep(100000);
+
+ $customers = $stripe->customers->all([
+ 'email' => $email,
+ 'limit' => 10,
+ ]);
+
+ usleep(100000);
+
+ foreach ($customers->data as $matchingCustomer) {
+ if ($matchingCustomer->id === $customerId) {
+ continue;
+ }
+
+ $subs = $stripe->subscriptions->all([
+ 'customer' => $matchingCustomer->id,
+ 'limit' => 10,
+ ]);
+
+ usleep(100000);
+
+ foreach ($subs->data as $sub) {
+ if (in_array($sub->status, ['active', 'past_due'])) {
+ return [
+ 'email' => $email,
+ 'customer_id' => $matchingCustomer->id,
+ 'subscription_id' => $sub->id,
+ 'status' => $sub->status,
+ ];
+ }
+ }
+ }
+ } catch (\Exception $e) {
+ // Silently skip — will fall through to normal discrepancy
+ }
+
+ return null;
+ }
+
+ /**
+ * Bulk fetch all active and past_due subscription IDs from Stripe.
+ *
+ * @return array