From c8a332a3bc935064dcbb2f7703b1edeb2becfaae Mon Sep 17 00:00:00 2001
From: Andras Bacsai <5845193+andrasbacsai@users.noreply.github.com>
Date: Sat, 11 Jul 2026 22:52:10 +0200
Subject: [PATCH] fix(servers): isolate cloud status checks from SSH checks

Track provider state independently, skip SSH work for placeholder IPs, and clean up failed cloud server provisioning.
---
 app/Actions/Server/DeleteServer.php           |  18 +--
 .../Api/DigitalOceanController.php            |  78 ++++++---
 app/Http/Controllers/Api/VultrController.php  |   6 +-
 .../CleanupOrphanedPreviewContainersJob.php   |   9 +-
 app/Jobs/ScheduledJobManager.php              |   4 +-
 .../ServerCloudProviderStatusCheckJob.php     |  59 +++++++
 app/Jobs/ServerConnectionCheckJob.php         |  88 +----------
 app/Jobs/ServerManagerJob.php                 |  34 +++-
 app/Livewire/Server/New/ByDigitalOcean.php    |  53 +++++--
 app/Livewire/Server/Show.php                  |   1 +
 app/Models/Server.php                         |  99 +++++++++---
 tests/Feature/DigitalOceanApiTest.php         | 104 ++++++++++++
 .../DigitalOceanServerCreationTest.php        |  48 ++++++
 .../ServerCloudProviderStatusCheckJobTest.php | 149 ++++++++++++++++++
 .../ServerConnectionCheckIsolationTest.php    |  80 ++++++++++
 tests/Feature/VultrApiTest.php                |  60 +++++++
 tests/Feature/VultrServerLifecycleTest.php    |  26 ++-
 tests/Unit/DigitalOceanDeleteServerTest.php   |  57 +++++++
 tests/Unit/DigitalOceanServerStateTest.php    |  37 ++++-
 tests/Unit/ServerBackoffTest.php              |  16 +-
 tests/Unit/ServerPlaceholderIpTest.php        |  58 +++++++
 tests/Unit/VultrDeleteServerTest.php          |  50 ++++++
 22 files changed, 954 insertions(+), 180 deletions(-)
 create mode 100644 app/Jobs/ServerCloudProviderStatusCheckJob.php
 create mode 100644 tests/Feature/ServerCloudProviderStatusCheckJobTest.php
 create mode 100644 tests/Feature/ServerConnectionCheckIsolationTest.php

diff --git a/app/Actions/Server/DeleteServer.php b/app/Actions/Server/DeleteServer.php
index aab889479..c6f032013 100644
--- a/app/Actions/Server/DeleteServer.php
+++ b/app/Actions/Server/DeleteServer.php
@@ -122,12 +122,7 @@ private function deleteFromVultrById(string $vultrInstanceId, ?int $cloudProvide
             }
 
             if (! $token) {
-                logger()->debug('No Vultr token found for team, skipping Vultr deletion', [
-                    'team_id' => $teamId,
-                    'vultr_instance_id' => $vultrInstanceId,
-                ]);
-
-                return;
+                throw new \RuntimeException('No Vultr token found for the server team.');
             }
 
             $vultrService = new VultrService($token->token);
@@ -143,6 +138,8 @@ private function deleteFromVultrById(string $vultrInstanceId, ?int $cloudProvide
                 'vultr_instance_id' => $vultrInstanceId,
                 'team_id' => $teamId,
             ]);
+
+            throw $e;
         }
     }
 
@@ -165,12 +162,7 @@ private function deleteFromDigitalOceanById(int $digitalOceanDropletId, ?int $cl
             }
 
             if (! $token) {
-                logger()->debug('No DigitalOcean token found for team, skipping droplet deletion', [
-                    'team_id' => $teamId,
-                    'digitalocean_droplet_id' => $digitalOceanDropletId,
-                ]);
-
-                return;
+                throw new \RuntimeException('No DigitalOcean token found for the server team.');
             }
 
             $digitalOceanService = new DigitalOceanService($token->token);
@@ -186,6 +178,8 @@ private function deleteFromDigitalOceanById(int $digitalOceanDropletId, ?int $cl
                 'digitalocean_droplet_id' => $digitalOceanDropletId,
                 'team_id' => $teamId,
             ]);
+
+            throw $e;
         }
     }
 }
diff --git a/app/Http/Controllers/Api/DigitalOceanController.php b/app/Http/Controllers/Api/DigitalOceanController.php
index fc51e7c01..5bd9d2392 100644
--- a/app/Http/Controllers/Api/DigitalOceanController.php
+++ b/app/Http/Controllers/Api/DigitalOceanController.php
@@ -15,6 +15,7 @@
 use App\Services\DigitalOceanService;
 use Illuminate\Http\JsonResponse;
 use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
 use OpenApi\Attributes as OA;
 
 class DigitalOceanController extends Controller
@@ -283,6 +284,10 @@ public function createServer(Request $request): JsonResponse
             return response()->json(['message' => 'Private key not found.'], 404);
         }
 
+        $digitalOceanService = null;
+        $dropletId = null;
+        $server = null;
+
         try {
             $digitalOceanService = new DigitalOceanService($token->token);
             $sshKeyId = $this->getOrCreateSshKey($digitalOceanService, $privateKey);
@@ -309,29 +314,41 @@ public function createServer(Request $request): JsonResponse
 
             $droplet = $digitalOceanService->createDroplet($params);
             $dropletId = (int) $droplet['id'];
-            $droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);
-            $ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);
 
-            if (! $ipAddress) {
-                throw new \Exception('No public IP address available for the new droplet.');
+            $server = DB::transaction(function () use ($normalizedServerName, $teamId, $privateKey, $token, $dropletId, $droplet): Server {
+                $server = Server::create([
+                    'name' => $normalizedServerName,
+                    'ip' => Server::PLACEHOLDER_IP,
+                    'user' => 'root',
+                    'port' => 22,
+                    'team_id' => $teamId,
+                    'private_key_id' => $privateKey->id,
+                    'cloud_provider_token_id' => $token->id,
+                    'digitalocean_droplet_id' => $dropletId,
+                    'digitalocean_droplet_status' => $droplet['status'] ?? null,
+                ]);
+
+                $server->proxy->set('status', 'exited');
+                $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
+                $server->save();
+
+                return $server;
+            });
+
+            try {
+                $droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);
+                $ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);
+
+                if ($ipAddress) {
+                    $server->update([
+                        'ip' => $ipAddress,
+                        'digitalocean_droplet_status' => $droplet['status'] ?? $server->digitalocean_droplet_status,
+                    ]);
+                }
+            } catch (\Throwable $e) {
+                report($e);
             }
 
-            $server = Server::create([
-                'name' => $normalizedServerName,
-                'ip' => $ipAddress,
-                'user' => 'root',
-                'port' => 22,
-                'team_id' => $teamId,
-                'private_key_id' => $privateKey->id,
-                'cloud_provider_token_id' => $token->id,
-                'digitalocean_droplet_id' => $dropletId,
-                'digitalocean_droplet_status' => $droplet['status'] ?? null,
-            ]);
-
-            $server->proxy->set('status', 'exited');
-            $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
-            $server->save();
-
             if ($request->instant_validate) {
                 ValidateServer::dispatch($server);
             }
@@ -341,15 +358,17 @@ public function createServer(Request $request): JsonResponse
                 'server_uuid' => $server->uuid,
                 'server_name' => $server->name,
                 'digitalocean_droplet_id' => $dropletId,
-                'ip' => $ipAddress,
+                'ip' => $server->ip,
             ]);
 
             return response()->json([
                 'uuid' => $server->uuid,
                 'digitalocean_droplet_id' => $dropletId,
-                'ip' => $ipAddress,
+                'ip' => $server->ip,
             ])->setStatusCode(201);
         } catch (RateLimitException $e) {
+            $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
+
             $response = response()->json(['message' => $e->getMessage()], 429);
             if ($e->retryAfter !== null) {
                 $response->header('Retry-After', $e->retryAfter);
@@ -357,6 +376,8 @@ public function createServer(Request $request): JsonResponse
 
             return $response;
         } catch (\Throwable $e) {
+            $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
+
             logger()->error('Failed to create DigitalOcean server', [
                 'error' => $e->getMessage(),
             ]);
@@ -365,6 +386,19 @@ public function createServer(Request $request): JsonResponse
         }
     }
 
+    private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void
+    {
+        if (! $digitalOceanService || ! $dropletId || $server) {
+            return;
+        }
+
+        try {
+            $digitalOceanService->deleteDroplet($dropletId);
+        } catch (\Throwable $e) {
+            report($e);
+        }
+    }
+
     private function getOrCreateSshKey(DigitalOceanService $digitalOceanService, PrivateKey $privateKey): int
     {
         $md5Fingerprint = PrivateKey::generateMd5Fingerprint($privateKey->private_key);
diff --git a/app/Http/Controllers/Api/VultrController.php b/app/Http/Controllers/Api/VultrController.php
index 7aaba568e..60b9d0a0a 100644
--- a/app/Http/Controllers/Api/VultrController.php
+++ b/app/Http/Controllers/Api/VultrController.php
@@ -52,6 +52,8 @@ private function getVultrToken(Request $request): CloudProviderToken|JsonRespons
             return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
         }
 
+        $this->authorize('view', $token);
+
         return $token;
     }
 
@@ -277,6 +279,8 @@ public function createServer(Request $request): JsonResponse
             return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
         }
 
+        $this->authorize('view', $token);
+
         $privateKey = PrivateKey::whereTeamId($teamId)->whereUuid($request->private_key_uuid)->first();
         if (! $privateKey) {
             return response()->json(['message' => 'Private key not found.'], 404);
@@ -313,7 +317,7 @@ public function createServer(Request $request): JsonResponse
             }
 
             $vultrInstance = $vultrService->createInstance($params);
-            $ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? '0.0.0.0';
+            $ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? Server::PLACEHOLDER_IP;
 
             $server = Server::create([
                 'name' => $normalizedServerName,
diff --git a/app/Jobs/CleanupOrphanedPreviewContainersJob.php b/app/Jobs/CleanupOrphanedPreviewContainersJob.php
index 5d3bed457..e74cba554 100644
--- a/app/Jobs/CleanupOrphanedPreviewContainersJob.php
+++ b/app/Jobs/CleanupOrphanedPreviewContainersJob.php
@@ -12,6 +12,7 @@
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Log;
 
 /**
@@ -53,11 +54,13 @@ public function handle(): void
     /**
      * Get all functional servers to check for orphaned containers.
      */
-    private function getServersToCheck(): \Illuminate\Support\Collection
+    private function getServersToCheck(): Collection
     {
         $query = Server::whereRelation('settings', 'is_usable', true)
             ->whereRelation('settings', 'is_reachable', true)
-            ->where('ip', '!=', '1.2.3.4');
+            ->whereNotNull('ip')
+            ->where('ip', '!=', '')
+            ->whereNotIn('ip', Server::PLACEHOLDER_IPS);
 
         if (isCloud()) {
             $query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true);
@@ -99,7 +102,7 @@ private function cleanupOrphanedContainersOnServer(Server $server): void
     /**
      * Get all PR containers on a server (containers with pullRequestId > 0).
      */
-    private function getPRContainersOnServer(Server $server): \Illuminate\Support\Collection
+    private function getPRContainersOnServer(Server $server): Collection
     {
         try {
             $output = instant_remote_process([
diff --git a/app/Jobs/ScheduledJobManager.php b/app/Jobs/ScheduledJobManager.php
index e7a21949c..46bc89d42 100644
--- a/app/Jobs/ScheduledJobManager.php
+++ b/app/Jobs/ScheduledJobManager.php
@@ -457,7 +457,9 @@ private function processDockerCleanup(Server $server): void
     private function getServersForCleanupQuery(): Builder
     {
         $query = Server::with('settings')
-            ->where('ip', '!=', '1.2.3.4');
+            ->whereNotNull('ip')
+            ->where('ip', '!=', '')
+            ->whereNotIn('ip', Server::PLACEHOLDER_IPS);
 
         if (isCloud()) {
             $query
diff --git a/app/Jobs/ServerCloudProviderStatusCheckJob.php b/app/Jobs/ServerCloudProviderStatusCheckJob.php
new file mode 100644
index 000000000..cd2e51df7
--- /dev/null
+++ b/app/Jobs/ServerCloudProviderStatusCheckJob.php
@@ -0,0 +1,59 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\Server;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeEncrypted;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Log;
+
+class ServerCloudProviderStatusCheckJob implements ShouldBeEncrypted, ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 1;
+
+    public $timeout = 120;
+
+    public function __construct(public Server $server)
+    {
+        $this->onQueue('high');
+    }
+
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping('server-cloud-provider-status-'.$this->server->uuid))->expireAfter(130)->dontRelease()];
+    }
+
+    public function handle(): void
+    {
+        try {
+            if (! $this->server->cloudProviderToken) {
+                return;
+            }
+
+            match ($this->server->cloudProviderToken->provider) {
+                'hetzner' => $this->server->hetzner_server_id
+                    ? $this->server->refreshHetznerState()
+                    : null,
+                'vultr' => $this->server->vultr_instance_id
+                    ? $this->server->refreshVultrState()
+                    : null,
+                'digitalocean' => $this->server->digitalocean_droplet_id
+                    ? $this->server->refreshDigitalOceanState()
+                    : null,
+                default => null,
+            };
+        } catch (\Throwable $e) {
+            Log::debug('Cloud provider status check failed', [
+                'server_id' => $this->server->id,
+                'error' => $e->getMessage(),
+            ]);
+        }
+    }
+}
diff --git a/app/Jobs/ServerConnectionCheckJob.php b/app/Jobs/ServerConnectionCheckJob.php
index 60adf8c18..f86686df7 100644
--- a/app/Jobs/ServerConnectionCheckJob.php
+++ b/app/Jobs/ServerConnectionCheckJob.php
@@ -6,7 +6,6 @@
 use App\Helpers\SshMultiplexingHelper;
 use App\Models\Server;
 use App\Services\ConfigurationRepository;
-use App\Services\HetznerService;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldBeEncrypted;
 use Illuminate\Contracts\Queue\ShouldQueue;
@@ -42,8 +41,12 @@ private function disableSshMux(): void
         $configRepository->disableSshMux();
     }
 
-    public function handle()
+    public function handle(): void
     {
+        if ($this->server->hasPlaceholderIp()) {
+            return;
+        }
+
         $wasReachable = (bool) $this->server->settings->is_reachable;
         $wasNotified = (bool) $this->server->unreachable_notification_sent;
 
@@ -62,19 +65,6 @@ public function handle()
                 return;
             }
 
-            // Check Hetzner server status if applicable
-            if ($this->server->hetzner_server_id && $this->server->cloudProviderToken) {
-                $this->checkHetznerStatus();
-            }
-
-            if ($this->server->vultr_instance_id && $this->server->cloudProviderToken) {
-                $this->checkVultrStatus();
-            }
-
-            if ($this->server->digitalocean_droplet_id && $this->server->cloudProviderToken) {
-                $this->checkDigitalOceanStatus();
-            }
-
             // Temporarily disable mux if requested
             if ($this->disableMux) {
                 $this->disableSshMux();
@@ -136,17 +126,6 @@ public function handle()
     public function failed(?\Throwable $exception): void
     {
         if ($exception instanceof TimeoutExceededException) {
-            $wasReachable = (bool) $this->server->settings->is_reachable;
-            $wasNotified = (bool) $this->server->unreachable_notification_sent;
-
-            $this->server->settings->update([
-                'is_reachable' => false,
-                'is_usable' => false,
-            ]);
-            $this->server->increment('unreachable_count');
-
-            $this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
-
             // Delete the queue job so it doesn't appear in Horizon's failed list.
             $this->job?->delete();
         }
@@ -171,63 +150,6 @@ private function dispatchReachabilityChangedIfNeeded(bool $wasReachable, bool $w
         }
     }
 
-    private function checkHetznerStatus(): void
-    {
-        $status = null;
-
-        try {
-            $hetznerService = new HetznerService($this->server->cloudProviderToken->token);
-            $serverData = $hetznerService->getServer($this->server->hetzner_server_id);
-            $status = $serverData['status'] ?? null;
-
-        } catch (\Throwable) {
-            // Silently ignore — server may have been deleted from Hetzner.
-        }
-        if ($this->server->hetzner_server_status !== $status) {
-            $this->server->update(['hetzner_server_status' => $status]);
-            $this->server->hetzner_server_status = $status;
-            if ($status === 'off') {
-                throw new \Exception('Server is powered off');
-            }
-        }
-
-    }
-
-    private function checkVultrStatus(): void
-    {
-        try {
-            $status = $this->server->refreshVultrState();
-        } catch (\Throwable) {
-            // Silently ignore transient Vultr API errors.
-
-            return;
-        }
-
-        if (in_array($status, ['stopped', 'suspended', 'deleted'], true)) {
-            throw new \Exception('Vultr instance is not running');
-        }
-    }
-
-    private function checkDigitalOceanStatus(): void
-    {
-        try {
-            $status = $this->server->refreshDigitalOceanState();
-        } catch (\Throwable $e) {
-            Log::debug('ServerConnectionCheck: DigitalOcean status check failed', [
-                'server_id' => $this->server->id,
-                'error' => $e->getMessage(),
-            ]);
-
-            return;
-        }
-
-        $this->server->digitalocean_droplet_status = $status;
-
-        if (in_array($status, ['off', 'archive', 'deleted'], true)) {
-            throw new \Exception('DigitalOcean droplet is not running');
-        }
-    }
-
     private function checkConnection(): bool
     {
         try {
diff --git a/app/Jobs/ServerManagerJob.php b/app/Jobs/ServerManagerJob.php
index 9532282cc..67c222c24 100644
--- a/app/Jobs/ServerManagerJob.php
+++ b/app/Jobs/ServerManagerJob.php
@@ -55,6 +55,13 @@ public function handle(): void
         // Get all servers to process
         $servers = $this->getServers();
 
+        // Provider state checks run independently so slow APIs cannot block SSH checks.
+        $this->dispatchCloudProviderStatusChecks($servers);
+
+        $servers = $servers
+            ->reject(fn (Server $server) => $server->hasPlaceholderIp())
+            ->values();
+
         // Dispatch ServerConnectionCheck for all servers efficiently
         $this->dispatchConnectionChecks($servers);
 
@@ -64,24 +71,45 @@ public function handle(): void
 
     private function getServers(): Collection
     {
-        $allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
+        $allServers = Server::with(['settings', 'cloudProviderToken']);
 
         if (isCloud()) {
             $servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
-            $own = Team::find(0)->servers()->with('settings')->get();
+            $own = Team::find(0)->servers()->with(['settings', 'cloudProviderToken'])->get();
 
-            return $servers->merge($own);
+            return $servers->merge($own)->unique('id')->values();
         } else {
             return $allServers->get();
         }
     }
 
+    private function dispatchCloudProviderStatusChecks(Collection $servers): void
+    {
+        if (! shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-cloud-provider-status-checks', $this->executionTime)) {
+            return;
+        }
+
+        $servers->each(function (Server $server) {
+            $hasCloudResource = $server->hetzner_server_id
+                || $server->vultr_instance_id
+                || $server->digitalocean_droplet_id;
+
+            if ($hasCloudResource && $server->cloudProviderToken) {
+                ServerCloudProviderStatusCheckJob::dispatch($server);
+            }
+        });
+    }
+
     private function dispatchConnectionChecks(Collection $servers): void
     {
 
         if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {
             $servers->each(function (Server $server) {
                 try {
+                    if ($server->hasPlaceholderIp()) {
+                        return;
+                    }
+
                     // Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
                     if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
                         return;
diff --git a/app/Livewire/Server/New/ByDigitalOcean.php b/app/Livewire/Server/New/ByDigitalOcean.php
index a59f23efb..cf27f0f08 100644
--- a/app/Livewire/Server/New/ByDigitalOcean.php
+++ b/app/Livewire/Server/New/ByDigitalOcean.php
@@ -14,6 +14,7 @@
 use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
 use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Collection;
+use Illuminate\Support\Facades\DB;
 use Livewire\Attributes\Locked;
 use Livewire\Component;
 
@@ -449,6 +450,10 @@ public function submit()
     {
         $this->validate();
 
+        $digitalOceanService = null;
+        $dropletId = null;
+        $server = null;
+
         try {
             $this->authorize('create', Server::class);
 
@@ -468,24 +473,29 @@ public function submit()
 
             $digitalOceanService = new DigitalOceanService($this->getDigitalOceanToken());
             $droplet = $this->createDigitalOceanDroplet($digitalOceanService);
+            $dropletId = (int) $droplet['id'];
 
             // Persist the server immediately so the droplet is always tracked
             // in Coolify, even if waiting for the public IP fails below.
-            $server = Server::create([
-                'name' => strtolower(trim($this->server_name)),
-                'ip' => Server::PLACEHOLDER_IP,
-                'user' => 'root',
-                'port' => 22,
-                'team_id' => currentTeam()->id,
-                'private_key_id' => $this->private_key_id,
-                'cloud_provider_token_id' => $this->selected_token_id,
-                'digitalocean_droplet_id' => $droplet['id'],
-                'digitalocean_droplet_status' => $droplet['status'] ?? null,
-            ]);
+            $server = DB::transaction(function () use ($dropletId, $droplet): Server {
+                $server = Server::create([
+                    'name' => strtolower(trim($this->server_name)),
+                    'ip' => Server::PLACEHOLDER_IP,
+                    'user' => 'root',
+                    'port' => 22,
+                    'team_id' => currentTeam()->id,
+                    'private_key_id' => $this->private_key_id,
+                    'cloud_provider_token_id' => $this->selected_token_id,
+                    'digitalocean_droplet_id' => $dropletId,
+                    'digitalocean_droplet_status' => $droplet['status'] ?? null,
+                ]);
 
-            $server->proxy->set('status', 'exited');
-            $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
-            $server->save();
+                $server->proxy->set('status', 'exited');
+                $server->proxy->set('type', ProxyTypes::TRAEFIK->value);
+                $server->save();
+
+                return $server;
+            });
 
             try {
                 $droplet = $digitalOceanService->waitForPublicIp($droplet, true, $this->enable_ipv6);
@@ -510,10 +520,25 @@ public function submit()
 
             return redirectRoute($this, 'server.show', [$server->uuid]);
         } catch (\Throwable $e) {
+            $this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
+
             return handleError($e, $this);
         }
     }
 
+    private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void
+    {
+        if (! $digitalOceanService || ! $dropletId || $server) {
+            return;
+        }
+
+        try {
+            $digitalOceanService->deleteDroplet($dropletId);
+        } catch (\Throwable $e) {
+            report($e);
+        }
+    }
+
     public function render()
     {
         return view('livewire.server.new.by-digital-ocean');
diff --git a/app/Livewire/Server/Show.php b/app/Livewire/Server/Show.php
index 83f4f83d9..6cd004362 100644
--- a/app/Livewire/Server/Show.php
+++ b/app/Livewire/Server/Show.php
@@ -611,6 +611,7 @@ public function startHetznerServer()
     public function startVultrInstance()
     {
         try {
+            $this->authorize('update', $this->server);
             if (! $this->server->vultr_instance_id || ! $this->server->cloudProviderToken) {
                 $this->dispatch('error', 'This server is not associated with a Vultr instance or token.');
 
diff --git a/app/Models/Server.php b/app/Models/Server.php
index 928f80fed..d55df0179 100644
--- a/app/Models/Server.php
+++ b/app/Models/Server.php
@@ -18,6 +18,7 @@
 use App\Notifications\Server\Unreachable;
 use App\Services\ConfigurationRepository;
 use App\Services\DigitalOceanService;
+use App\Services\HetznerService;
 use App\Services\VultrService;
 use App\Support\ValidationPatterns;
 use App\Traits\ClearsGlobalSearchCache;
@@ -119,6 +120,8 @@ class Server extends BaseModel
      */
     public const PLACEHOLDER_IP = '1.2.3.4';
 
+    public const PLACEHOLDER_IPS = [self::PLACEHOLDER_IP, '0.0.0.0', '::'];
+
     public static $batch_counter = 0;
 
     /**
@@ -317,9 +320,12 @@ public function type()
     public function hasPlaceholderIp(): bool
     {
         // Cast: the saving hook stores the ip as a Stringable in memory.
-        $ip = (string) $this->ip;
+        return self::isPlaceholderIp((string) $this->ip);
+    }
 
-        return blank($ip) || in_array($ip, [self::PLACEHOLDER_IP, '0.0.0.0', '::'], true);
+    public static function isPlaceholderIp(?string $ip): bool
+    {
+        return blank($ip) || in_array($ip, self::PLACEHOLDER_IPS, true);
     }
 
     /**
@@ -328,13 +334,66 @@ public function hasPlaceholderIp(): bool
      */
     public function backfillPlaceholderIp(?string $ip): bool
     {
-        if ($ip && $this->hasPlaceholderIp()) {
-            $this->update(['ip' => $ip]);
-
-            return true;
+        if (self::isPlaceholderIp($ip)) {
+            return false;
         }
 
-        return false;
+        $updated = static::query()
+            ->whereKey($this->getKey())
+            ->where(function (Builder $query): void {
+                $query->whereNull('ip')
+                    ->orWhere('ip', '')
+                    ->orWhereIn('ip', self::PLACEHOLDER_IPS);
+            })
+            ->update(['ip' => $ip]);
+
+        if ($updated === 0) {
+            return false;
+        }
+
+        $this->forceFill(['ip' => $ip]);
+        $this->syncOriginalAttribute('ip');
+        static::flushIdentityMap();
+
+        return true;
+    }
+
+    /**
+     * Persist provider status without saving a stale in-memory IP value.
+     *
+     * @param  array<string, mixed>  $updates
+     */
+    private function persistProviderState(array $updates): void
+    {
+        if (empty($updates)) {
+            return;
+        }
+
+        static::query()->whereKey($this->getKey())->update($updates);
+        $this->forceFill($updates);
+        $this->syncOriginalAttributes(array_keys($updates));
+        static::flushIdentityMap();
+    }
+
+    public function refreshHetznerState(): ?string
+    {
+        if (! $this->hetzner_server_id || ! $this->cloudProviderToken || $this->cloudProviderToken->provider !== 'hetzner') {
+            return $this->hetzner_server_status;
+        }
+
+        $hetznerService = new HetznerService($this->cloudProviderToken->token);
+        $server = $hetznerService->getServer($this->hetzner_server_id);
+        $status = $server['status'] ?? null;
+        $assignedIp = data_get($server, 'public_net.ipv4.ip') ?? data_get($server, 'public_net.ipv6.ip');
+
+        $updates = [];
+        if ($this->hetzner_server_status !== $status) {
+            $updates['hetzner_server_status'] = $status;
+        }
+        $this->persistProviderState($updates);
+        $this->backfillPlaceholderIp($assignedIp);
+
+        return $status;
     }
 
     public function refreshVultrState(): ?string
@@ -352,8 +411,7 @@ public function refreshVultrState(): ?string
             }
 
             if ($this->vultr_instance_status !== 'deleted') {
-                $this->update(['vultr_instance_status' => 'deleted']);
-                $this->forceFill(['vultr_instance_status' => 'deleted']);
+                $this->persistProviderState(['vultr_instance_status' => 'deleted']);
             }
 
             return 'deleted';
@@ -368,15 +426,8 @@ public function refreshVultrState(): ?string
         if ($this->vultr_instance_status !== $status) {
             $updates['vultr_instance_status'] = $status;
         }
-
-        if ($this->hasPlaceholderIp() && $publicIp) {
-            $updates['ip'] = $publicIp;
-        }
-
-        if (! empty($updates)) {
-            $this->update($updates);
-            $this->forceFill($updates);
-        }
+        $this->persistProviderState($updates);
+        $this->backfillPlaceholderIp($publicIp);
 
         return $status;
     }
@@ -393,7 +444,7 @@ public function refreshDigitalOceanState(): ?string
             $droplet = $digitalOceanService->getDroplet((int) $this->digitalocean_droplet_id);
         } catch (RequestException $e) {
             if ($e->response?->status() === 404) {
-                $this->update(['digitalocean_droplet_status' => 'deleted']);
+                $this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
 
                 return 'deleted';
             }
@@ -401,7 +452,7 @@ public function refreshDigitalOceanState(): ?string
             throw $e;
         } catch (\Throwable $e) {
             if ((int) $e->getCode() === 404) {
-                $this->update(['digitalocean_droplet_status' => 'deleted']);
+                $this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
 
                 return 'deleted';
             }
@@ -416,12 +467,8 @@ public function refreshDigitalOceanState(): ?string
         $status = $droplet['status'] ?? null;
         $ip = $digitalOceanService->getPublicIpAddress($droplet);
 
-        $updates = ['digitalocean_droplet_status' => $status];
-        if ($ip && $this->hasPlaceholderIp()) {
-            $updates['ip'] = $ip;
-        }
-
-        $this->update($updates);
+        $this->persistProviderState(['digitalocean_droplet_status' => $status]);
+        $this->backfillPlaceholderIp($ip);
 
         return $status;
     }
diff --git a/tests/Feature/DigitalOceanApiTest.php b/tests/Feature/DigitalOceanApiTest.php
index 04c1e16be..826899957 100644
--- a/tests/Feature/DigitalOceanApiTest.php
+++ b/tests/Feature/DigitalOceanApiTest.php
@@ -3,6 +3,7 @@
 use App\Models\CloudProviderToken;
 use App\Models\InstanceSettings;
 use App\Models\PrivateKey;
+use App\Models\Server;
 use App\Models\Team;
 use App\Models\User;
 use Illuminate\Foundation\Testing\RefreshDatabase;
@@ -133,6 +134,109 @@
             && $request['user_data'] === '#cloud-config');
     });
 
+    test('tracks the droplet with a placeholder IP when waiting for its IP fails', function () {
+        Http::fake([
+            'https://api.digitalocean.com/v2/account/keys' => Http::response([
+                'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
+            ], 201),
+            'https://api.digitalocean.com/v2/account/keys*' => Http::response([
+                'ssh_keys' => [],
+                'links' => ['pages' => []],
+            ], 200),
+            'https://api.digitalocean.com/v2/droplets' => Http::response([
+                'droplet' => [
+                    'id' => 987,
+                    'name' => 'waiting-for-ip',
+                    'status' => 'new',
+                    'networks' => ['v4' => [], 'v6' => []],
+                ],
+            ], 202),
+            'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+                'message' => 'temporary provider failure',
+            ], 500),
+        ]);
+
+        $response = $this->withHeaders([
+            'Authorization' => 'Bearer '.$this->bearerToken,
+            'Content-Type' => 'application/json',
+        ])->postJson('/api/v1/servers/digitalocean', [
+            'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
+            'region' => 'nyc1',
+            'size' => 's-1vcpu-1gb',
+            'image' => 'ubuntu-24-04-x64',
+            'name' => 'waiting-for-ip',
+            'private_key_uuid' => $this->privateKey->uuid,
+        ]);
+
+        $response->assertCreated();
+        $response->assertJsonFragment([
+            'digitalocean_droplet_id' => 987,
+            'ip' => Server::PLACEHOLDER_IP,
+        ]);
+
+        $this->assertDatabaseHas('servers', [
+            'name' => 'waiting-for-ip',
+            'ip' => Server::PLACEHOLDER_IP,
+            'team_id' => $this->team->id,
+            'digitalocean_droplet_id' => 987,
+            'digitalocean_droplet_status' => 'new',
+        ]);
+    });
+
+    test('deletes the droplet when local server persistence fails', function () {
+        Http::fake([
+            'https://api.digitalocean.com/v2/account/keys' => Http::response([
+                'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
+            ], 201),
+            'https://api.digitalocean.com/v2/account/keys*' => Http::response([
+                'ssh_keys' => [],
+                'links' => ['pages' => []],
+            ], 200),
+            'https://api.digitalocean.com/v2/droplets' => Http::response([
+                'droplet' => [
+                    'id' => 987,
+                    'name' => 'persistence-fails',
+                    'status' => 'active',
+                    'networks' => [
+                        'v4' => [
+                            ['ip_address' => '203.0.113.10', 'type' => 'public'],
+                        ],
+                    ],
+                ],
+            ], 202),
+            'https://api.digitalocean.com/v2/droplets/987' => Http::response(null, 204),
+        ]);
+
+        $eventDispatcher = Server::getEventDispatcher();
+        Server::setEventDispatcher(clone $eventDispatcher);
+        Server::created(function (): void {
+            throw new RuntimeException('local persistence failed');
+        });
+
+        try {
+            $response = $this->withHeaders([
+                'Authorization' => 'Bearer '.$this->bearerToken,
+                'Content-Type' => 'application/json',
+            ])->postJson('/api/v1/servers/digitalocean', [
+                'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
+                'region' => 'nyc1',
+                'size' => 's-1vcpu-1gb',
+                'image' => 'ubuntu-24-04-x64',
+                'name' => 'persistence-fails',
+                'private_key_uuid' => $this->privateKey->uuid,
+            ]);
+        } finally {
+            Server::setEventDispatcher($eventDispatcher);
+        }
+
+        $response->assertServerError();
+        $this->assertDatabaseMissing('servers', [
+            'digitalocean_droplet_id' => 987,
+        ]);
+        Http::assertSent(fn ($request) => $request->method() === 'DELETE'
+            && $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
+    });
+
     test('validates required fields', function () {
         $response = $this->withHeaders([
             'Authorization' => 'Bearer '.$this->bearerToken,
diff --git a/tests/Feature/DigitalOceanServerCreationTest.php b/tests/Feature/DigitalOceanServerCreationTest.php
index b42148dfc..3d81a5dc9 100644
--- a/tests/Feature/DigitalOceanServerCreationTest.php
+++ b/tests/Feature/DigitalOceanServerCreationTest.php
@@ -113,6 +113,54 @@ function submitDigitalOceanServer(): void
     ]);
 });
 
+it('deletes the droplet when local server persistence fails', function () {
+    Http::fake([
+        'https://api.digitalocean.com/v2/account/keys' => Http::response([
+            'ssh_key' => ['id' => 123],
+        ], 201),
+        'https://api.digitalocean.com/v2/account/keys*' => Http::response([
+            'ssh_keys' => [],
+        ], 200),
+        'https://api.digitalocean.com/v2/droplets' => Http::response([
+            'droplet' => [
+                'id' => 555,
+                'status' => 'active',
+                'networks' => [
+                    'v4' => [
+                        ['type' => 'public', 'ip_address' => '203.0.113.40'],
+                    ],
+                ],
+            ],
+        ], 202),
+        'https://api.digitalocean.com/v2/droplets/555' => Http::response(null, 204),
+    ]);
+
+    $eventDispatcher = Server::getEventDispatcher();
+    Server::setEventDispatcher(clone $eventDispatcher);
+    Server::created(function (): void {
+        throw new RuntimeException('local persistence failed');
+    });
+
+    try {
+        Livewire::test(ByDigitalOcean::class, ['selectedTokenUuid' => $this->digitalOceanToken->uuid])
+            ->set('server_name', 'persistence-fails')
+            ->set('selected_region', 'nyc1')
+            ->set('selected_size', 's-1vcpu-1gb')
+            ->set('selected_image', 'ubuntu-24-04-x64')
+            ->set('private_key_id', $this->privateKey->id)
+            ->call('submit')
+            ->assertDispatched('error', 'local persistence failed');
+    } finally {
+        Server::setEventDispatcher($eventDispatcher);
+    }
+
+    $this->assertDatabaseMissing('servers', [
+        'digitalocean_droplet_id' => 555,
+    ]);
+    Http::assertSent(fn ($request) => $request->method() === 'DELETE'
+        && $request->url() === 'https://api.digitalocean.com/v2/droplets/555');
+});
+
 it('renders only the full width buy button at the bottom of the DigitalOcean form', function () {
     Livewire::test(ByDigitalOcean::class)
         ->set('current_step', 2)
diff --git a/tests/Feature/ServerCloudProviderStatusCheckJobTest.php b/tests/Feature/ServerCloudProviderStatusCheckJobTest.php
new file mode 100644
index 000000000..10e1a8c9d
--- /dev/null
+++ b/tests/Feature/ServerCloudProviderStatusCheckJobTest.php
@@ -0,0 +1,149 @@
+<?php
+
+use App\Jobs\ServerCloudProviderStatusCheckJob;
+use App\Jobs\ServerConnectionCheckJob;
+use App\Jobs\ServerManagerJob;
+use App\Models\CloudProviderToken;
+use App\Models\InstanceSettings;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Carbon;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Queue;
+
+uses(RefreshDatabase::class);
+
+function createCloudServerForStatusCheckJobTest(string $provider, array $attributes = []): Server
+{
+    $team = Team::factory()->create();
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $token = CloudProviderToken::create([
+        'team_id' => $team->id,
+        'provider' => $provider,
+        'token' => "test-{$provider}-token",
+        'name' => ucfirst($provider),
+    ]);
+
+    return Server::factory()->create(array_merge([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'cloud_provider_token_id' => $token->id,
+        'ip' => Server::PLACEHOLDER_IP,
+    ], $attributes));
+}
+
+beforeEach(function () {
+    InstanceSettings::forceCreate([
+        'id' => 0,
+        'instance_timezone' => 'UTC',
+    ]);
+    Carbon::setTestNow('2026-07-11 12:00:00');
+});
+
+afterEach(function () {
+    Carbon::setTestNow();
+});
+
+it('syncs provider state for a placeholder server without scheduling SSH', function () {
+    $server = createCloudServerForStatusCheckJobTest('digitalocean', [
+        'digitalocean_droplet_id' => 987,
+        'digitalocean_droplet_status' => 'new',
+    ]);
+
+    Queue::fake();
+
+    (new ServerManagerJob)->handle();
+
+    Queue::assertPushed(ServerCloudProviderStatusCheckJob::class, fn (ServerCloudProviderStatusCheckJob $job) => $job->server->is($server));
+    Queue::assertNotPushed(ServerConnectionCheckJob::class);
+});
+
+it('backfills a DigitalOcean placeholder without changing reachability', function () {
+    $server = createCloudServerForStatusCheckJobTest('digitalocean', [
+        'digitalocean_droplet_id' => 987,
+        'digitalocean_droplet_status' => 'new',
+    ]);
+    $server->settings->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+    ]);
+
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+            'droplet' => [
+                'id' => 987,
+                'status' => 'active',
+                'networks' => [
+                    'v4' => [
+                        ['type' => 'public', 'ip_address' => '203.0.113.10'],
+                    ],
+                ],
+            ],
+        ]),
+    ]);
+
+    (new ServerCloudProviderStatusCheckJob($server))->handle();
+
+    $server->refresh();
+    expect($server->ip)->toBe('203.0.113.10')
+        ->and($server->digitalocean_droplet_status)->toBe('active')
+        ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
+        ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
+});
+
+it('backfills a Hetzner placeholder without changing reachability', function () {
+    $server = createCloudServerForStatusCheckJobTest('hetzner', [
+        'hetzner_server_id' => 123,
+        'hetzner_server_status' => 'starting',
+    ]);
+    $server->settings->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+    ]);
+
+    Http::fake([
+        'https://api.hetzner.cloud/v1/servers/123' => Http::response([
+            'server' => [
+                'id' => 123,
+                'status' => 'running',
+                'public_net' => [
+                    'ipv4' => ['ip' => '198.51.100.20'],
+                ],
+            ],
+        ]),
+    ]);
+
+    (new ServerCloudProviderStatusCheckJob($server))->handle();
+
+    $server->refresh();
+    expect($server->ip)->toBe('198.51.100.20')
+        ->and($server->hetzner_server_status)->toBe('running')
+        ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
+        ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
+});
+
+it('only calls the API matching the cloud provider token', function () {
+    $server = createCloudServerForStatusCheckJobTest('digitalocean', [
+        'vultr_instance_id' => 'unrelated-vultr-instance',
+        'digitalocean_droplet_id' => 987,
+        'digitalocean_droplet_status' => 'new',
+    ]);
+
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+            'droplet' => [
+                'id' => 987,
+                'status' => 'active',
+                'networks' => ['v4' => []],
+            ],
+        ]),
+        'https://api.vultr.com/*' => Http::response(status: 500),
+    ]);
+
+    (new ServerCloudProviderStatusCheckJob($server))->handle();
+
+    Http::assertSentCount(1);
+    Http::assertSent(fn ($request) => $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
+});
diff --git a/tests/Feature/ServerConnectionCheckIsolationTest.php b/tests/Feature/ServerConnectionCheckIsolationTest.php
new file mode 100644
index 000000000..d098c39b5
--- /dev/null
+++ b/tests/Feature/ServerConnectionCheckIsolationTest.php
@@ -0,0 +1,80 @@
+<?php
+
+use App\Jobs\ServerConnectionCheckJob;
+use App\Models\CloudProviderToken;
+use App\Models\PrivateKey;
+use App\Models\Server;
+use App\Models\Team;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Facades\Storage;
+
+uses(RefreshDatabase::class);
+
+function createServerForConnectionIsolationTest(array $attributes = []): Server
+{
+    $team = Team::factory()->create();
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+
+    return Server::factory()->create(array_merge([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+    ], $attributes));
+}
+
+beforeEach(function () {
+    Storage::fake('ssh-keys');
+});
+
+it('never attempts SSH to a placeholder address', function (string $placeholderIp) {
+    $server = createServerForConnectionIsolationTest(['ip' => $placeholderIp]);
+    $server->settings->update([
+        'is_reachable' => true,
+        'is_usable' => true,
+    ]);
+
+    Process::fake();
+
+    (new ServerConnectionCheckJob($server, disableMux: false))->handle();
+
+    Process::assertNothingRan();
+    expect($server->fresh()->unreachable_count)->toBe(0)
+        ->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
+        ->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
+})->with([
+    'reserved provisioning address' => Server::PLACEHOLDER_IP,
+    'unspecified IPv4 address' => '0.0.0.0',
+    'unspecified IPv6 address' => '::',
+]);
+
+it('does not call cloud provider APIs during an SSH connection check', function () {
+    $server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);
+    $token = CloudProviderToken::create([
+        'team_id' => $server->team_id,
+        'provider' => 'vultr',
+        'token' => 'test-vultr-token',
+        'name' => 'Vultr',
+    ]);
+    $server->update([
+        'cloud_provider_token_id' => $token->id,
+        'vultr_instance_id' => 'instance-1',
+        'vultr_instance_status' => 'active',
+    ]);
+
+    Http::fake([
+        'https://api.vultr.com/*' => Http::response([
+            'instance' => ['id' => 'instance-1', 'status' => 'active'],
+        ]),
+    ]);
+    Process::fake([
+        '*' => Process::result(
+            output: '{"Server":{"Version":"27.0.0"}}',
+            exitCode: 0,
+        ),
+    ]);
+
+    (new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
+
+    Http::assertNothingSent();
+});
diff --git a/tests/Feature/VultrApiTest.php b/tests/Feature/VultrApiTest.php
index 8a96ec685..9eb135baf 100644
--- a/tests/Feature/VultrApiTest.php
+++ b/tests/Feature/VultrApiTest.php
@@ -1,11 +1,15 @@
 <?php
 
+use App\Http\Controllers\Api\VultrController;
 use App\Models\CloudProviderToken;
 use App\Models\InstanceSettings;
 use App\Models\PrivateKey;
+use App\Models\Server;
 use App\Models\Team;
 use App\Models\User;
+use Illuminate\Auth\Access\AuthorizationException;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Http;
 
 uses(RefreshDatabase::class);
@@ -165,6 +169,27 @@ function testPublicKey(): string
     });
 });
 
+test('member read token cannot use a stored Vultr token for metadata endpoints', function (string $endpoint) {
+    $member = User::factory()->create();
+    $this->team->members()->attach($member->id, ['role' => 'member']);
+    $memberToken = $member->createToken('member-read', ['read'])->plainTextToken;
+
+    Http::fake();
+
+    $response = $this->withHeaders([
+        'Authorization' => 'Bearer '.$memberToken,
+        'Content-Type' => 'application/json',
+    ])->getJson($endpoint.'?cloud_provider_token_uuid='.$this->vultrToken->uuid);
+
+    $response->assertForbidden();
+    Http::assertNothingSent();
+})->with([
+    '/api/v1/vultr/regions',
+    '/api/v1/vultr/plans',
+    '/api/v1/vultr/os',
+    '/api/v1/vultr/ssh-keys',
+]);
+
 describe('POST /api/v1/servers/vultr', function () {
     test('creates a Vultr server', function () {
         Http::fake([
@@ -223,6 +248,11 @@ function testPublicKey(): string
     });
 
     test('waits for Vultr to assign a real public IP when creation returns placeholder IP', function () {
+        $createdIp = null;
+        Server::created(function (Server $server) use (&$createdIp): void {
+            $createdIp = (string) $server->ip;
+        });
+
         Http::fake([
             'https://api.vultr.com/v2/ssh-keys' => Http::response([
                 'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],
@@ -262,6 +292,8 @@ function testPublicKey(): string
         $response->assertStatus(201);
         $response->assertJsonFragment(['ip' => '1.2.3.4']);
 
+        expect($createdIp)->toBe(Server::PLACEHOLDER_IP);
+
         $this->assertDatabaseHas('servers', [
             'name' => 'test-server',
             'ip' => '1.2.3.4',
@@ -269,6 +301,34 @@ function testPublicKey(): string
         ]);
     });
 
+    test('server creation authorizes access to the stored Vultr token', function () {
+        $member = User::factory()->create();
+        $this->team->members()->attach($member->id, ['role' => 'member']);
+        $this->actingAs($member);
+        $member->withAccessToken($member->createToken('member-all', ['*'])->accessToken);
+
+        $request = Request::create(
+            uri: '/api/v1/servers/vultr',
+            method: 'POST',
+            server: ['CONTENT_TYPE' => 'application/json'],
+            content: json_encode([
+                'cloud_provider_token_uuid' => $this->vultrToken->uuid,
+                'region' => 'ewr',
+                'plan' => 'vc2-1c-1gb',
+                'os_id' => 2284,
+                'name' => 'test-server',
+                'private_key_uuid' => $this->privateKey->uuid,
+            ], JSON_THROW_ON_ERROR),
+        );
+        $request->setUserResolver(fn () => $member);
+
+        Http::fake();
+
+        expect(fn () => app(VultrController::class)->createServer($request))
+            ->toThrow(AuthorizationException::class);
+        Http::assertNothingSent();
+    });
+
     test('reuses existing matching Vultr SSH key', function () {
         Http::fake([
             'https://api.vultr.com/v2/ssh-keys*' => Http::response([
diff --git a/tests/Feature/VultrServerLifecycleTest.php b/tests/Feature/VultrServerLifecycleTest.php
index c89ca47b4..cb3b3108c 100644
--- a/tests/Feature/VultrServerLifecycleTest.php
+++ b/tests/Feature/VultrServerLifecycleTest.php
@@ -123,6 +123,8 @@ function vultrLifecycleTestPrivateKey(): string
 });
 
 it('does not overwrite an established server IP during Vultr status refresh', function () {
+    $this->server->update(['ip' => '198.51.100.20']);
+
     Http::fake([
         'https://api.vultr.com/v2/instances/instance-1' => Http::response([
             'instance' => [
@@ -135,12 +137,12 @@ function vultrLifecycleTestPrivateKey(): string
 
     Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
         ->call('checkVultrInstanceStatus', true)
-        ->assertSet('ip', '1.2.3.4')
+        ->assertSet('ip', '198.51.100.20')
         ->assertSet('vultrInstanceStatus', 'active');
 
     $this->assertDatabaseHas('servers', [
         'id' => $this->server->id,
-        'ip' => '1.2.3.4',
+        'ip' => '198.51.100.20',
         'vultr_instance_status' => 'active',
     ]);
 });
@@ -270,6 +272,26 @@ function vultrLifecycleTestPrivateKey(): string
     Http::assertSent(fn ($request) => $request->url() === 'https://api.vultr.com/v2/instances/instance-1/start');
 });
 
+it('prevents members from starting a Vultr instance', function () {
+    $this->server->update(['vultr_instance_status' => 'stopped']);
+
+    $member = User::factory()->create();
+    $this->team->members()->attach($member->id, ['role' => 'member']);
+    $this->actingAs($member);
+    session(['currentTeam' => $this->team]);
+
+    Http::fake([
+        'https://api.vultr.com/v2/instances/instance-1/start' => Http::response([], 204),
+    ]);
+
+    Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
+        ->call('startVultrInstance')
+        ->assertDispatched('error');
+
+    expect($this->server->fresh()->vultr_instance_status)->toBe('stopped');
+    Http::assertNothingSent();
+});
+
 it('links a server to Vultr by matching IP', function () {
     $unlinkedServer = Server::factory()->create([
         'team_id' => $this->team->id,
diff --git a/tests/Unit/DigitalOceanDeleteServerTest.php b/tests/Unit/DigitalOceanDeleteServerTest.php
index 5b0989607..60cd5ba50 100644
--- a/tests/Unit/DigitalOceanDeleteServerTest.php
+++ b/tests/Unit/DigitalOceanDeleteServerTest.php
@@ -6,6 +6,7 @@
 use App\Models\Server;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;
 
@@ -44,3 +45,59 @@
     Http::assertSent(fn ($request) => $request->method() === 'DELETE'
         && $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
 });
+
+it('retains the server and surfaces a DigitalOcean deletion failure', function () {
+    Http::fake([
+        'https://api.digitalocean.com/v2/droplets/987' => Http::response([
+            'message' => 'deletion failed',
+        ], 500),
+    ]);
+
+    $team = Team::factory()->create();
+    $token = CloudProviderToken::factory()->create([
+        'team_id' => $team->id,
+        'provider' => 'digitalocean',
+        'token' => 'test-digitalocean-token',
+    ]);
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'cloud_provider_token_id' => $token->id,
+        'digitalocean_droplet_id' => 987,
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        deleteFromDigitalOcean: true,
+        digitalOceanDropletId: 987,
+        cloudProviderTokenId: $token->id,
+        teamId: $team->id,
+    ))->toThrow(RequestException::class, 'status code 500');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
+
+it('retains the server when no DigitalOcean token can delete the droplet', function () {
+    Http::preventStrayRequests();
+
+    $team = Team::factory()->create();
+    $privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
+    $server = Server::factory()->create([
+        'team_id' => $team->id,
+        'private_key_id' => $privateKey->id,
+        'cloud_provider_token_id' => null,
+        'digitalocean_droplet_id' => 987,
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        deleteFromDigitalOcean: true,
+        digitalOceanDropletId: 987,
+        teamId: $team->id,
+    ))->toThrow(RuntimeException::class, 'No DigitalOcean token found');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
diff --git a/tests/Unit/DigitalOceanServerStateTest.php b/tests/Unit/DigitalOceanServerStateTest.php
index 93b3379bb..725a38c5b 100644
--- a/tests/Unit/DigitalOceanServerStateTest.php
+++ b/tests/Unit/DigitalOceanServerStateTest.php
@@ -66,8 +66,8 @@ function createDigitalOceanServerForStateTest(string $status = 'active', array $
     expect($server->fresh()->ip)->toBe('203.0.113.10');
 });
 
-it('does not overwrite a real IP from the DigitalOcean droplet state', function () {
-    $server = createDigitalOceanServerForStateTest('active', ['ip' => '198.51.100.20']);
+it('does not overwrite an administrator configured address from the DigitalOcean droplet state', function (string $configuredAddress) {
+    $server = createDigitalOceanServerForStateTest('active', ['ip' => $configuredAddress]);
 
     Http::fake([
         'https://api.digitalocean.com/v2/droplets/987' => Http::response([
@@ -84,7 +84,38 @@ function createDigitalOceanServerForStateTest(string $status = 'active', array $
     ]);
 
     $server->refreshDigitalOceanState();
-    expect($server->fresh()->ip)->toBe('198.51.100.20');
+    expect($server->fresh()->ip)->toBe($configuredAddress);
+})->with([
+    'public IPv4' => '198.51.100.20',
+    'private IPv4' => '10.10.0.12',
+    'IPv6' => '2001:db8::10',
+    'tunnel hostname' => 'server.internal.example.com',
+]);
+
+it('does not overwrite an address configured while DigitalOcean state is loading', function () {
+    $server = createDigitalOceanServerForStateTest('new', ['ip' => Server::PLACEHOLDER_IP]);
+
+    Http::fake(function () use ($server) {
+        Server::query()->findOrFail($server->id)->update([
+            'ip' => 'server.internal.example.com',
+        ]);
+
+        return Http::response([
+            'droplet' => [
+                'id' => 987,
+                'status' => 'active',
+                'networks' => [
+                    'v4' => [
+                        ['type' => 'public', 'ip_address' => '203.0.113.10'],
+                    ],
+                ],
+            ],
+        ], 200);
+    });
+
+    expect($server->refreshDigitalOceanState())->toBe('active')
+        ->and($server->fresh()->ip)->toBe('server.internal.example.com')
+        ->and($server->digitalocean_droplet_status)->toBe('active');
 });
 
 it('does not mark a DigitalOcean droplet as deleted on transient provider errors', function () {
diff --git a/tests/Unit/ServerBackoffTest.php b/tests/Unit/ServerBackoffTest.php
index 7c9910a5c..3bbf41b3a 100644
--- a/tests/Unit/ServerBackoffTest.php
+++ b/tests/Unit/ServerBackoffTest.php
@@ -127,7 +127,7 @@
 });
 
 describe('ServerConnectionCheckJob unreachable_count', function () {
-    it('marks Vultr servers unreachable when provider status is unavailable', function () {
+    it('marks servers unreachable when SSH is unavailable', function () {
         Event::fake([ServerReachabilityChanged::class]);
 
         $settings = Mockery::mock();
@@ -141,14 +141,12 @@
         $server->shouldReceive('getAttribute')->andReturnUsing(fn (string $key) => match ($key) {
             'settings' => $settings,
             'unreachable_notification_sent' => false,
-            'vultr_instance_id' => 'instance-1',
-            'cloudProviderToken' => (object) ['token' => 'test-token'],
+            'ip' => '203.0.113.10',
             'id' => 1,
             'name' => 'test-server',
             'unreachable_count' => 1,
             default => null,
         });
-        $server->shouldReceive('refreshVultrState')->once()->andReturn('stopped');
         $server->shouldReceive('increment')->with('unreachable_count')->once();
         $server->id = 1;
         $server->name = 'test-server';
@@ -158,22 +156,20 @@
         $job->handle();
     });
 
-    it('increments unreachable_count on timeout', function () {
+    it('does not change reachability on a job-level timeout', function () {
         Event::fake([ServerReachabilityChanged::class]);
 
         $settings = Mockery::mock();
         $settings->is_reachable = true;
-        $settings->shouldReceive('update')
-            ->with(['is_reachable' => false, 'is_usable' => false])
-            ->once();
+        $settings->shouldNotReceive('update');
 
         $server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
         $server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);
         $server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn(false);
-        $server->shouldReceive('increment')->with('unreachable_count')->once();
+        $server->shouldNotReceive('increment');
         $server->id = 1;
         $server->name = 'test-server';
-        $server->unreachable_count = 1; // Will become 2 after increment in real code; mock keeps value as-is
+        $server->unreachable_count = 1;
 
         $job = new ServerConnectionCheckJob($server);
         $job->failed(new TimeoutExceededException);
diff --git a/tests/Unit/ServerPlaceholderIpTest.php b/tests/Unit/ServerPlaceholderIpTest.php
index b471ae7e1..94787b4d4 100644
--- a/tests/Unit/ServerPlaceholderIpTest.php
+++ b/tests/Unit/ServerPlaceholderIpTest.php
@@ -1,5 +1,7 @@
 <?php
 
+use App\Jobs\CleanupOrphanedPreviewContainersJob;
+use App\Jobs\ScheduledJobManager;
 use App\Models\PrivateKey;
 use App\Models\Server;
 use App\Models\Team;
@@ -60,8 +62,64 @@ function createServerForPlaceholderIpTest(string $ip): Server
     expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
 });
 
+it('does not replace a placeholder with another placeholder', function (string $replacementIp) {
+    $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+
+    expect($server->backfillPlaceholderIp($replacementIp))->toBeFalse();
+    expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
+})->with([
+    'unspecified IPv4 address' => '0.0.0.0',
+    'unspecified IPv6 address' => '::',
+]);
+
+it('does not overwrite an address configured after the placeholder model was loaded', function () {
+    $staleServer = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
+    $concurrentServer = Server::query()->findOrFail($staleServer->id);
+
+    $concurrentServer->update(['ip' => 'server.internal.example.com']);
+
+    expect($staleServer->backfillPlaceholderIp('203.0.113.10'))->toBeFalse()
+        ->and($staleServer->fresh()->ip)->toBe('server.internal.example.com');
+});
+
 it('skips servers with a placeholder IP in scheduled jobs', function () {
     $server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
 
     expect($server->skipServer())->toBeTrue();
 });
+
+it('excludes every placeholder address from scheduled Docker cleanup', function () {
+    $realServer = createServerForPlaceholderIpTest('203.0.113.10');
+
+    foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
+        Server::factory()->create([
+            'team_id' => $realServer->team_id,
+            'private_key_id' => $realServer->private_key_id,
+            'ip' => $placeholderIp,
+        ]);
+    }
+
+    $method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');
+    $servers = $method->invoke(new ScheduledJobManager)->get();
+
+    expect($servers->modelKeys())->toBe([$realServer->id]);
+});
+
+it('excludes every placeholder address from orphaned preview cleanup', function () {
+    $realServer = createServerForPlaceholderIpTest('203.0.113.10');
+    $realServer->settings->update(['is_reachable' => true, 'is_usable' => true]);
+
+    foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
+        $server = Server::factory()->create([
+            'team_id' => $realServer->team_id,
+            'private_key_id' => $realServer->private_key_id,
+            'ip' => $placeholderIp,
+        ]);
+        $server->settings->update(['is_reachable' => true, 'is_usable' => true]);
+    }
+
+    $method = new ReflectionMethod(CleanupOrphanedPreviewContainersJob::class, 'getServersToCheck');
+    $servers = $method->invoke(new CleanupOrphanedPreviewContainersJob);
+
+    expect($servers->modelKeys())->toBe([$realServer->id]);
+});
diff --git a/tests/Unit/VultrDeleteServerTest.php b/tests/Unit/VultrDeleteServerTest.php
index 05891eed7..82173d40d 100644
--- a/tests/Unit/VultrDeleteServerTest.php
+++ b/tests/Unit/VultrDeleteServerTest.php
@@ -7,6 +7,7 @@
 use App\Models\Server;
 use App\Models\Team;
 use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Http\Client\RequestException;
 use Illuminate\Support\Facades\Http;
 use Tests\TestCase;
 
@@ -116,3 +117,52 @@ function vultrDeleteTestPrivateKey(): string
 
     expect($request->header('Authorization'))->toBe(['Bearer test-vultr-token']);
 });
+
+it('retains the server and surfaces a Vultr deletion failure', function () {
+    Http::fake([
+        'https://api.vultr.com/v2/instances/instance-1' => Http::response([
+            'error' => 'deletion failed',
+        ], 500),
+    ]);
+
+    $server = Server::factory()->create([
+        'team_id' => $this->team->id,
+        'private_key_id' => $this->privateKey->id,
+        'cloud_provider_token_id' => $this->vultrToken->id,
+        'vultr_instance_id' => 'instance-1',
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        cloudProviderTokenId: $this->vultrToken->id,
+        teamId: $this->team->id,
+        deleteFromVultr: true,
+        vultrInstanceId: 'instance-1'
+    ))->toThrow(RequestException::class, 'status code 500');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});
+
+it('retains the server when no Vultr token can delete the instance', function () {
+    Http::preventStrayRequests();
+
+    $this->vultrToken->delete();
+
+    $server = Server::factory()->create([
+        'team_id' => $this->team->id,
+        'private_key_id' => $this->privateKey->id,
+        'cloud_provider_token_id' => null,
+        'vultr_instance_id' => 'instance-1',
+    ]);
+    $server->delete();
+
+    expect(fn () => DeleteServer::run(
+        serverId: $server->id,
+        teamId: $this->team->id,
+        deleteFromVultr: true,
+        vultrInstanceId: 'instance-1'
+    ))->toThrow(RuntimeException::class, 'No Vultr token found');
+
+    expect(Server::withTrashed()->find($server->id))->not->toBeNull();
+});