fix(servers): isolate cloud status checks from SSH checks
Track provider state independently, skip SSH work for placeholder IPs, and clean up failed cloud server provisioning.
This commit is contained in:
parent
e01b8a057e
commit
c8a332a3bc
22 changed files with 954 additions and 180 deletions
|
|
@ -122,12 +122,7 @@ private function deleteFromVultrById(string $vultrInstanceId, ?int $cloudProvide
|
||||||
}
|
}
|
||||||
|
|
||||||
if (! $token) {
|
if (! $token) {
|
||||||
logger()->debug('No Vultr token found for team, skipping Vultr deletion', [
|
throw new \RuntimeException('No Vultr token found for the server team.');
|
||||||
'team_id' => $teamId,
|
|
||||||
'vultr_instance_id' => $vultrInstanceId,
|
|
||||||
]);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$vultrService = new VultrService($token->token);
|
$vultrService = new VultrService($token->token);
|
||||||
|
|
@ -143,6 +138,8 @@ private function deleteFromVultrById(string $vultrInstanceId, ?int $cloudProvide
|
||||||
'vultr_instance_id' => $vultrInstanceId,
|
'vultr_instance_id' => $vultrInstanceId,
|
||||||
'team_id' => $teamId,
|
'team_id' => $teamId,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
throw $e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -165,12 +162,7 @@ private function deleteFromDigitalOceanById(int $digitalOceanDropletId, ?int $cl
|
||||||
}
|
}
|
||||||
|
|
||||||
if (! $token) {
|
if (! $token) {
|
||||||
logger()->debug('No DigitalOcean token found for team, skipping droplet deletion', [
|
throw new \RuntimeException('No DigitalOcean token found for the server team.');
|
||||||
'team_id' => $teamId,
|
|
||||||
'digitalocean_droplet_id' => $digitalOceanDropletId,
|
|
||||||
]);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$digitalOceanService = new DigitalOceanService($token->token);
|
$digitalOceanService = new DigitalOceanService($token->token);
|
||||||
|
|
@ -186,6 +178,8 @@ private function deleteFromDigitalOceanById(int $digitalOceanDropletId, ?int $cl
|
||||||
'digitalocean_droplet_id' => $digitalOceanDropletId,
|
'digitalocean_droplet_id' => $digitalOceanDropletId,
|
||||||
'team_id' => $teamId,
|
'team_id' => $teamId,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
throw $e;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -15,6 +15,7 @@
|
||||||
use App\Services\DigitalOceanService;
|
use App\Services\DigitalOceanService;
|
||||||
use Illuminate\Http\JsonResponse;
|
use Illuminate\Http\JsonResponse;
|
||||||
use Illuminate\Http\Request;
|
use Illuminate\Http\Request;
|
||||||
|
use Illuminate\Support\Facades\DB;
|
||||||
use OpenApi\Attributes as OA;
|
use OpenApi\Attributes as OA;
|
||||||
|
|
||||||
class DigitalOceanController extends Controller
|
class DigitalOceanController extends Controller
|
||||||
|
|
@ -283,6 +284,10 @@ public function createServer(Request $request): JsonResponse
|
||||||
return response()->json(['message' => 'Private key not found.'], 404);
|
return response()->json(['message' => 'Private key not found.'], 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$digitalOceanService = null;
|
||||||
|
$dropletId = null;
|
||||||
|
$server = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$digitalOceanService = new DigitalOceanService($token->token);
|
$digitalOceanService = new DigitalOceanService($token->token);
|
||||||
$sshKeyId = $this->getOrCreateSshKey($digitalOceanService, $privateKey);
|
$sshKeyId = $this->getOrCreateSshKey($digitalOceanService, $privateKey);
|
||||||
|
|
@ -309,16 +314,11 @@ public function createServer(Request $request): JsonResponse
|
||||||
|
|
||||||
$droplet = $digitalOceanService->createDroplet($params);
|
$droplet = $digitalOceanService->createDroplet($params);
|
||||||
$dropletId = (int) $droplet['id'];
|
$dropletId = (int) $droplet['id'];
|
||||||
$droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);
|
|
||||||
$ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);
|
|
||||||
|
|
||||||
if (! $ipAddress) {
|
|
||||||
throw new \Exception('No public IP address available for the new droplet.');
|
|
||||||
}
|
|
||||||
|
|
||||||
|
$server = DB::transaction(function () use ($normalizedServerName, $teamId, $privateKey, $token, $dropletId, $droplet): Server {
|
||||||
$server = Server::create([
|
$server = Server::create([
|
||||||
'name' => $normalizedServerName,
|
'name' => $normalizedServerName,
|
||||||
'ip' => $ipAddress,
|
'ip' => Server::PLACEHOLDER_IP,
|
||||||
'user' => 'root',
|
'user' => 'root',
|
||||||
'port' => 22,
|
'port' => 22,
|
||||||
'team_id' => $teamId,
|
'team_id' => $teamId,
|
||||||
|
|
@ -332,6 +332,23 @@ public function createServer(Request $request): JsonResponse
|
||||||
$server->proxy->set('type', ProxyTypes::TRAEFIK->value);
|
$server->proxy->set('type', ProxyTypes::TRAEFIK->value);
|
||||||
$server->save();
|
$server->save();
|
||||||
|
|
||||||
|
return $server;
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
$droplet = $digitalOceanService->waitForPublicIp($droplet, true, $request->enable_ipv6);
|
||||||
|
$ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $request->enable_ipv6);
|
||||||
|
|
||||||
|
if ($ipAddress) {
|
||||||
|
$server->update([
|
||||||
|
'ip' => $ipAddress,
|
||||||
|
'digitalocean_droplet_status' => $droplet['status'] ?? $server->digitalocean_droplet_status,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
report($e);
|
||||||
|
}
|
||||||
|
|
||||||
if ($request->instant_validate) {
|
if ($request->instant_validate) {
|
||||||
ValidateServer::dispatch($server);
|
ValidateServer::dispatch($server);
|
||||||
}
|
}
|
||||||
|
|
@ -341,15 +358,17 @@ public function createServer(Request $request): JsonResponse
|
||||||
'server_uuid' => $server->uuid,
|
'server_uuid' => $server->uuid,
|
||||||
'server_name' => $server->name,
|
'server_name' => $server->name,
|
||||||
'digitalocean_droplet_id' => $dropletId,
|
'digitalocean_droplet_id' => $dropletId,
|
||||||
'ip' => $ipAddress,
|
'ip' => $server->ip,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
return response()->json([
|
return response()->json([
|
||||||
'uuid' => $server->uuid,
|
'uuid' => $server->uuid,
|
||||||
'digitalocean_droplet_id' => $dropletId,
|
'digitalocean_droplet_id' => $dropletId,
|
||||||
'ip' => $ipAddress,
|
'ip' => $server->ip,
|
||||||
])->setStatusCode(201);
|
])->setStatusCode(201);
|
||||||
} catch (RateLimitException $e) {
|
} catch (RateLimitException $e) {
|
||||||
|
$this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
|
||||||
|
|
||||||
$response = response()->json(['message' => $e->getMessage()], 429);
|
$response = response()->json(['message' => $e->getMessage()], 429);
|
||||||
if ($e->retryAfter !== null) {
|
if ($e->retryAfter !== null) {
|
||||||
$response->header('Retry-After', $e->retryAfter);
|
$response->header('Retry-After', $e->retryAfter);
|
||||||
|
|
@ -357,6 +376,8 @@ public function createServer(Request $request): JsonResponse
|
||||||
|
|
||||||
return $response;
|
return $response;
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
|
$this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
|
||||||
|
|
||||||
logger()->error('Failed to create DigitalOcean server', [
|
logger()->error('Failed to create DigitalOcean server', [
|
||||||
'error' => $e->getMessage(),
|
'error' => $e->getMessage(),
|
||||||
]);
|
]);
|
||||||
|
|
@ -365,6 +386,19 @@ public function createServer(Request $request): JsonResponse
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void
|
||||||
|
{
|
||||||
|
if (! $digitalOceanService || ! $dropletId || $server) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
$digitalOceanService->deleteDroplet($dropletId);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
report($e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private function getOrCreateSshKey(DigitalOceanService $digitalOceanService, PrivateKey $privateKey): int
|
private function getOrCreateSshKey(DigitalOceanService $digitalOceanService, PrivateKey $privateKey): int
|
||||||
{
|
{
|
||||||
$md5Fingerprint = PrivateKey::generateMd5Fingerprint($privateKey->private_key);
|
$md5Fingerprint = PrivateKey::generateMd5Fingerprint($privateKey->private_key);
|
||||||
|
|
|
||||||
|
|
@ -52,6 +52,8 @@ private function getVultrToken(Request $request): CloudProviderToken|JsonRespons
|
||||||
return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
|
return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$this->authorize('view', $token);
|
||||||
|
|
||||||
return $token;
|
return $token;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -277,6 +279,8 @@ public function createServer(Request $request): JsonResponse
|
||||||
return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
|
return response()->json(['message' => 'Vultr cloud provider token not found.'], 404);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$this->authorize('view', $token);
|
||||||
|
|
||||||
$privateKey = PrivateKey::whereTeamId($teamId)->whereUuid($request->private_key_uuid)->first();
|
$privateKey = PrivateKey::whereTeamId($teamId)->whereUuid($request->private_key_uuid)->first();
|
||||||
if (! $privateKey) {
|
if (! $privateKey) {
|
||||||
return response()->json(['message' => 'Private key not found.'], 404);
|
return response()->json(['message' => 'Private key not found.'], 404);
|
||||||
|
|
@ -313,7 +317,7 @@ public function createServer(Request $request): JsonResponse
|
||||||
}
|
}
|
||||||
|
|
||||||
$vultrInstance = $vultrService->createInstance($params);
|
$vultrInstance = $vultrService->createInstance($params);
|
||||||
$ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? '0.0.0.0';
|
$ipAddress = $vultrService->getPublicIp($vultrInstance, $request->disable_public_ipv4, $request->enable_ipv6) ?? Server::PLACEHOLDER_IP;
|
||||||
|
|
||||||
$server = Server::create([
|
$server = Server::create([
|
||||||
'name' => $normalizedServerName,
|
'name' => $normalizedServerName,
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@
|
||||||
use Illuminate\Queue\InteractsWithQueue;
|
use Illuminate\Queue\InteractsWithQueue;
|
||||||
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||||
use Illuminate\Queue\SerializesModels;
|
use Illuminate\Queue\SerializesModels;
|
||||||
|
use Illuminate\Support\Collection;
|
||||||
use Illuminate\Support\Facades\Log;
|
use Illuminate\Support\Facades\Log;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -53,11 +54,13 @@ public function handle(): void
|
||||||
/**
|
/**
|
||||||
* Get all functional servers to check for orphaned containers.
|
* Get all functional servers to check for orphaned containers.
|
||||||
*/
|
*/
|
||||||
private function getServersToCheck(): \Illuminate\Support\Collection
|
private function getServersToCheck(): Collection
|
||||||
{
|
{
|
||||||
$query = Server::whereRelation('settings', 'is_usable', true)
|
$query = Server::whereRelation('settings', 'is_usable', true)
|
||||||
->whereRelation('settings', 'is_reachable', true)
|
->whereRelation('settings', 'is_reachable', true)
|
||||||
->where('ip', '!=', '1.2.3.4');
|
->whereNotNull('ip')
|
||||||
|
->where('ip', '!=', '')
|
||||||
|
->whereNotIn('ip', Server::PLACEHOLDER_IPS);
|
||||||
|
|
||||||
if (isCloud()) {
|
if (isCloud()) {
|
||||||
$query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true);
|
$query = $query->whereRelation('team.subscription', 'stripe_invoice_paid', true);
|
||||||
|
|
@ -99,7 +102,7 @@ private function cleanupOrphanedContainersOnServer(Server $server): void
|
||||||
/**
|
/**
|
||||||
* Get all PR containers on a server (containers with pullRequestId > 0).
|
* Get all PR containers on a server (containers with pullRequestId > 0).
|
||||||
*/
|
*/
|
||||||
private function getPRContainersOnServer(Server $server): \Illuminate\Support\Collection
|
private function getPRContainersOnServer(Server $server): Collection
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
$output = instant_remote_process([
|
$output = instant_remote_process([
|
||||||
|
|
|
||||||
|
|
@ -457,7 +457,9 @@ private function processDockerCleanup(Server $server): void
|
||||||
private function getServersForCleanupQuery(): Builder
|
private function getServersForCleanupQuery(): Builder
|
||||||
{
|
{
|
||||||
$query = Server::with('settings')
|
$query = Server::with('settings')
|
||||||
->where('ip', '!=', '1.2.3.4');
|
->whereNotNull('ip')
|
||||||
|
->where('ip', '!=', '')
|
||||||
|
->whereNotIn('ip', Server::PLACEHOLDER_IPS);
|
||||||
|
|
||||||
if (isCloud()) {
|
if (isCloud()) {
|
||||||
$query
|
$query
|
||||||
|
|
|
||||||
59
app/Jobs/ServerCloudProviderStatusCheckJob.php
Normal file
59
app/Jobs/ServerCloudProviderStatusCheckJob.php
Normal file
|
|
@ -0,0 +1,59 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
namespace App\Jobs;
|
||||||
|
|
||||||
|
use App\Models\Server;
|
||||||
|
use Illuminate\Bus\Queueable;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||||
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
|
use Illuminate\Foundation\Bus\Dispatchable;
|
||||||
|
use Illuminate\Queue\InteractsWithQueue;
|
||||||
|
use Illuminate\Queue\Middleware\WithoutOverlapping;
|
||||||
|
use Illuminate\Queue\SerializesModels;
|
||||||
|
use Illuminate\Support\Facades\Log;
|
||||||
|
|
||||||
|
class ServerCloudProviderStatusCheckJob implements ShouldBeEncrypted, ShouldQueue
|
||||||
|
{
|
||||||
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
||||||
|
|
||||||
|
public $tries = 1;
|
||||||
|
|
||||||
|
public $timeout = 120;
|
||||||
|
|
||||||
|
public function __construct(public Server $server)
|
||||||
|
{
|
||||||
|
$this->onQueue('high');
|
||||||
|
}
|
||||||
|
|
||||||
|
public function middleware(): array
|
||||||
|
{
|
||||||
|
return [(new WithoutOverlapping('server-cloud-provider-status-'.$this->server->uuid))->expireAfter(130)->dontRelease()];
|
||||||
|
}
|
||||||
|
|
||||||
|
public function handle(): void
|
||||||
|
{
|
||||||
|
try {
|
||||||
|
if (! $this->server->cloudProviderToken) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
match ($this->server->cloudProviderToken->provider) {
|
||||||
|
'hetzner' => $this->server->hetzner_server_id
|
||||||
|
? $this->server->refreshHetznerState()
|
||||||
|
: null,
|
||||||
|
'vultr' => $this->server->vultr_instance_id
|
||||||
|
? $this->server->refreshVultrState()
|
||||||
|
: null,
|
||||||
|
'digitalocean' => $this->server->digitalocean_droplet_id
|
||||||
|
? $this->server->refreshDigitalOceanState()
|
||||||
|
: null,
|
||||||
|
default => null,
|
||||||
|
};
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
Log::debug('Cloud provider status check failed', [
|
||||||
|
'server_id' => $this->server->id,
|
||||||
|
'error' => $e->getMessage(),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -6,7 +6,6 @@
|
||||||
use App\Helpers\SshMultiplexingHelper;
|
use App\Helpers\SshMultiplexingHelper;
|
||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
use App\Services\ConfigurationRepository;
|
use App\Services\ConfigurationRepository;
|
||||||
use App\Services\HetznerService;
|
|
||||||
use Illuminate\Bus\Queueable;
|
use Illuminate\Bus\Queueable;
|
||||||
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
use Illuminate\Contracts\Queue\ShouldBeEncrypted;
|
||||||
use Illuminate\Contracts\Queue\ShouldQueue;
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
||||||
|
|
@ -42,8 +41,12 @@ private function disableSshMux(): void
|
||||||
$configRepository->disableSshMux();
|
$configRepository->disableSshMux();
|
||||||
}
|
}
|
||||||
|
|
||||||
public function handle()
|
public function handle(): void
|
||||||
{
|
{
|
||||||
|
if ($this->server->hasPlaceholderIp()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
$wasReachable = (bool) $this->server->settings->is_reachable;
|
$wasReachable = (bool) $this->server->settings->is_reachable;
|
||||||
$wasNotified = (bool) $this->server->unreachable_notification_sent;
|
$wasNotified = (bool) $this->server->unreachable_notification_sent;
|
||||||
|
|
||||||
|
|
@ -62,19 +65,6 @@ public function handle()
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check Hetzner server status if applicable
|
|
||||||
if ($this->server->hetzner_server_id && $this->server->cloudProviderToken) {
|
|
||||||
$this->checkHetznerStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($this->server->vultr_instance_id && $this->server->cloudProviderToken) {
|
|
||||||
$this->checkVultrStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
if ($this->server->digitalocean_droplet_id && $this->server->cloudProviderToken) {
|
|
||||||
$this->checkDigitalOceanStatus();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Temporarily disable mux if requested
|
// Temporarily disable mux if requested
|
||||||
if ($this->disableMux) {
|
if ($this->disableMux) {
|
||||||
$this->disableSshMux();
|
$this->disableSshMux();
|
||||||
|
|
@ -136,17 +126,6 @@ public function handle()
|
||||||
public function failed(?\Throwable $exception): void
|
public function failed(?\Throwable $exception): void
|
||||||
{
|
{
|
||||||
if ($exception instanceof TimeoutExceededException) {
|
if ($exception instanceof TimeoutExceededException) {
|
||||||
$wasReachable = (bool) $this->server->settings->is_reachable;
|
|
||||||
$wasNotified = (bool) $this->server->unreachable_notification_sent;
|
|
||||||
|
|
||||||
$this->server->settings->update([
|
|
||||||
'is_reachable' => false,
|
|
||||||
'is_usable' => false,
|
|
||||||
]);
|
|
||||||
$this->server->increment('unreachable_count');
|
|
||||||
|
|
||||||
$this->dispatchReachabilityChangedIfNeeded($wasReachable, $wasNotified, false);
|
|
||||||
|
|
||||||
// Delete the queue job so it doesn't appear in Horizon's failed list.
|
// Delete the queue job so it doesn't appear in Horizon's failed list.
|
||||||
$this->job?->delete();
|
$this->job?->delete();
|
||||||
}
|
}
|
||||||
|
|
@ -171,63 +150,6 @@ private function dispatchReachabilityChangedIfNeeded(bool $wasReachable, bool $w
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private function checkHetznerStatus(): void
|
|
||||||
{
|
|
||||||
$status = null;
|
|
||||||
|
|
||||||
try {
|
|
||||||
$hetznerService = new HetznerService($this->server->cloudProviderToken->token);
|
|
||||||
$serverData = $hetznerService->getServer($this->server->hetzner_server_id);
|
|
||||||
$status = $serverData['status'] ?? null;
|
|
||||||
|
|
||||||
} catch (\Throwable) {
|
|
||||||
// Silently ignore — server may have been deleted from Hetzner.
|
|
||||||
}
|
|
||||||
if ($this->server->hetzner_server_status !== $status) {
|
|
||||||
$this->server->update(['hetzner_server_status' => $status]);
|
|
||||||
$this->server->hetzner_server_status = $status;
|
|
||||||
if ($status === 'off') {
|
|
||||||
throw new \Exception('Server is powered off');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
private function checkVultrStatus(): void
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
$status = $this->server->refreshVultrState();
|
|
||||||
} catch (\Throwable) {
|
|
||||||
// Silently ignore transient Vultr API errors.
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (in_array($status, ['stopped', 'suspended', 'deleted'], true)) {
|
|
||||||
throw new \Exception('Vultr instance is not running');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function checkDigitalOceanStatus(): void
|
|
||||||
{
|
|
||||||
try {
|
|
||||||
$status = $this->server->refreshDigitalOceanState();
|
|
||||||
} catch (\Throwable $e) {
|
|
||||||
Log::debug('ServerConnectionCheck: DigitalOcean status check failed', [
|
|
||||||
'server_id' => $this->server->id,
|
|
||||||
'error' => $e->getMessage(),
|
|
||||||
]);
|
|
||||||
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->server->digitalocean_droplet_status = $status;
|
|
||||||
|
|
||||||
if (in_array($status, ['off', 'archive', 'deleted'], true)) {
|
|
||||||
throw new \Exception('DigitalOcean droplet is not running');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private function checkConnection(): bool
|
private function checkConnection(): bool
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,13 @@ public function handle(): void
|
||||||
// Get all servers to process
|
// Get all servers to process
|
||||||
$servers = $this->getServers();
|
$servers = $this->getServers();
|
||||||
|
|
||||||
|
// Provider state checks run independently so slow APIs cannot block SSH checks.
|
||||||
|
$this->dispatchCloudProviderStatusChecks($servers);
|
||||||
|
|
||||||
|
$servers = $servers
|
||||||
|
->reject(fn (Server $server) => $server->hasPlaceholderIp())
|
||||||
|
->values();
|
||||||
|
|
||||||
// Dispatch ServerConnectionCheck for all servers efficiently
|
// Dispatch ServerConnectionCheck for all servers efficiently
|
||||||
$this->dispatchConnectionChecks($servers);
|
$this->dispatchConnectionChecks($servers);
|
||||||
|
|
||||||
|
|
@ -64,24 +71,45 @@ public function handle(): void
|
||||||
|
|
||||||
private function getServers(): Collection
|
private function getServers(): Collection
|
||||||
{
|
{
|
||||||
$allServers = Server::with('settings')->where('ip', '!=', '1.2.3.4');
|
$allServers = Server::with(['settings', 'cloudProviderToken']);
|
||||||
|
|
||||||
if (isCloud()) {
|
if (isCloud()) {
|
||||||
$servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
|
$servers = $allServers->whereRelation('team.subscription', 'stripe_invoice_paid', true)->get();
|
||||||
$own = Team::find(0)->servers()->with('settings')->get();
|
$own = Team::find(0)->servers()->with(['settings', 'cloudProviderToken'])->get();
|
||||||
|
|
||||||
return $servers->merge($own);
|
return $servers->merge($own)->unique('id')->values();
|
||||||
} else {
|
} else {
|
||||||
return $allServers->get();
|
return $allServers->get();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function dispatchCloudProviderStatusChecks(Collection $servers): void
|
||||||
|
{
|
||||||
|
if (! shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-cloud-provider-status-checks', $this->executionTime)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
$servers->each(function (Server $server) {
|
||||||
|
$hasCloudResource = $server->hetzner_server_id
|
||||||
|
|| $server->vultr_instance_id
|
||||||
|
|| $server->digitalocean_droplet_id;
|
||||||
|
|
||||||
|
if ($hasCloudResource && $server->cloudProviderToken) {
|
||||||
|
ServerCloudProviderStatusCheckJob::dispatch($server);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private function dispatchConnectionChecks(Collection $servers): void
|
private function dispatchConnectionChecks(Collection $servers): void
|
||||||
{
|
{
|
||||||
|
|
||||||
if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {
|
if (shouldRunCronNow($this->checkFrequency, $this->instanceTimezone, 'server-connection-checks', $this->executionTime)) {
|
||||||
$servers->each(function (Server $server) {
|
$servers->each(function (Server $server) {
|
||||||
try {
|
try {
|
||||||
|
if ($server->hasPlaceholderIp()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
|
// Skip SSH connection check if Sentinel is healthy — its heartbeat already proves connectivity
|
||||||
if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
|
if ($server->isSentinelEnabled() && $server->isSentinelLive()) {
|
||||||
return;
|
return;
|
||||||
|
|
|
||||||
|
|
@ -14,6 +14,7 @@
|
||||||
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
use Illuminate\Foundation\Auth\Access\AuthorizesRequests;
|
||||||
use Illuminate\Http\Client\RequestException;
|
use Illuminate\Http\Client\RequestException;
|
||||||
use Illuminate\Support\Collection;
|
use Illuminate\Support\Collection;
|
||||||
|
use Illuminate\Support\Facades\DB;
|
||||||
use Livewire\Attributes\Locked;
|
use Livewire\Attributes\Locked;
|
||||||
use Livewire\Component;
|
use Livewire\Component;
|
||||||
|
|
||||||
|
|
@ -449,6 +450,10 @@ public function submit()
|
||||||
{
|
{
|
||||||
$this->validate();
|
$this->validate();
|
||||||
|
|
||||||
|
$digitalOceanService = null;
|
||||||
|
$dropletId = null;
|
||||||
|
$server = null;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$this->authorize('create', Server::class);
|
$this->authorize('create', Server::class);
|
||||||
|
|
||||||
|
|
@ -468,9 +473,11 @@ public function submit()
|
||||||
|
|
||||||
$digitalOceanService = new DigitalOceanService($this->getDigitalOceanToken());
|
$digitalOceanService = new DigitalOceanService($this->getDigitalOceanToken());
|
||||||
$droplet = $this->createDigitalOceanDroplet($digitalOceanService);
|
$droplet = $this->createDigitalOceanDroplet($digitalOceanService);
|
||||||
|
$dropletId = (int) $droplet['id'];
|
||||||
|
|
||||||
// Persist the server immediately so the droplet is always tracked
|
// Persist the server immediately so the droplet is always tracked
|
||||||
// in Coolify, even if waiting for the public IP fails below.
|
// in Coolify, even if waiting for the public IP fails below.
|
||||||
|
$server = DB::transaction(function () use ($dropletId, $droplet): Server {
|
||||||
$server = Server::create([
|
$server = Server::create([
|
||||||
'name' => strtolower(trim($this->server_name)),
|
'name' => strtolower(trim($this->server_name)),
|
||||||
'ip' => Server::PLACEHOLDER_IP,
|
'ip' => Server::PLACEHOLDER_IP,
|
||||||
|
|
@ -479,7 +486,7 @@ public function submit()
|
||||||
'team_id' => currentTeam()->id,
|
'team_id' => currentTeam()->id,
|
||||||
'private_key_id' => $this->private_key_id,
|
'private_key_id' => $this->private_key_id,
|
||||||
'cloud_provider_token_id' => $this->selected_token_id,
|
'cloud_provider_token_id' => $this->selected_token_id,
|
||||||
'digitalocean_droplet_id' => $droplet['id'],
|
'digitalocean_droplet_id' => $dropletId,
|
||||||
'digitalocean_droplet_status' => $droplet['status'] ?? null,
|
'digitalocean_droplet_status' => $droplet['status'] ?? null,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
|
@ -487,6 +494,9 @@ public function submit()
|
||||||
$server->proxy->set('type', ProxyTypes::TRAEFIK->value);
|
$server->proxy->set('type', ProxyTypes::TRAEFIK->value);
|
||||||
$server->save();
|
$server->save();
|
||||||
|
|
||||||
|
return $server;
|
||||||
|
});
|
||||||
|
|
||||||
try {
|
try {
|
||||||
$droplet = $digitalOceanService->waitForPublicIp($droplet, true, $this->enable_ipv6);
|
$droplet = $digitalOceanService->waitForPublicIp($droplet, true, $this->enable_ipv6);
|
||||||
$ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $this->enable_ipv6);
|
$ipAddress = $digitalOceanService->getPublicIpAddress($droplet, true, $this->enable_ipv6);
|
||||||
|
|
@ -510,10 +520,25 @@ public function submit()
|
||||||
|
|
||||||
return redirectRoute($this, 'server.show', [$server->uuid]);
|
return redirectRoute($this, 'server.show', [$server->uuid]);
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
|
$this->deleteUntrackedDroplet($digitalOceanService, $dropletId, $server);
|
||||||
|
|
||||||
return handleError($e, $this);
|
return handleError($e, $this);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function deleteUntrackedDroplet(?DigitalOceanService $digitalOceanService, ?int $dropletId, ?Server $server): void
|
||||||
|
{
|
||||||
|
if (! $digitalOceanService || ! $dropletId || $server) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
$digitalOceanService->deleteDroplet($dropletId);
|
||||||
|
} catch (\Throwable $e) {
|
||||||
|
report($e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public function render()
|
public function render()
|
||||||
{
|
{
|
||||||
return view('livewire.server.new.by-digital-ocean');
|
return view('livewire.server.new.by-digital-ocean');
|
||||||
|
|
|
||||||
|
|
@ -611,6 +611,7 @@ public function startHetznerServer()
|
||||||
public function startVultrInstance()
|
public function startVultrInstance()
|
||||||
{
|
{
|
||||||
try {
|
try {
|
||||||
|
$this->authorize('update', $this->server);
|
||||||
if (! $this->server->vultr_instance_id || ! $this->server->cloudProviderToken) {
|
if (! $this->server->vultr_instance_id || ! $this->server->cloudProviderToken) {
|
||||||
$this->dispatch('error', 'This server is not associated with a Vultr instance or token.');
|
$this->dispatch('error', 'This server is not associated with a Vultr instance or token.');
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@
|
||||||
use App\Notifications\Server\Unreachable;
|
use App\Notifications\Server\Unreachable;
|
||||||
use App\Services\ConfigurationRepository;
|
use App\Services\ConfigurationRepository;
|
||||||
use App\Services\DigitalOceanService;
|
use App\Services\DigitalOceanService;
|
||||||
|
use App\Services\HetznerService;
|
||||||
use App\Services\VultrService;
|
use App\Services\VultrService;
|
||||||
use App\Support\ValidationPatterns;
|
use App\Support\ValidationPatterns;
|
||||||
use App\Traits\ClearsGlobalSearchCache;
|
use App\Traits\ClearsGlobalSearchCache;
|
||||||
|
|
@ -119,6 +120,8 @@ class Server extends BaseModel
|
||||||
*/
|
*/
|
||||||
public const PLACEHOLDER_IP = '1.2.3.4';
|
public const PLACEHOLDER_IP = '1.2.3.4';
|
||||||
|
|
||||||
|
public const PLACEHOLDER_IPS = [self::PLACEHOLDER_IP, '0.0.0.0', '::'];
|
||||||
|
|
||||||
public static $batch_counter = 0;
|
public static $batch_counter = 0;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -317,9 +320,12 @@ public function type()
|
||||||
public function hasPlaceholderIp(): bool
|
public function hasPlaceholderIp(): bool
|
||||||
{
|
{
|
||||||
// Cast: the saving hook stores the ip as a Stringable in memory.
|
// Cast: the saving hook stores the ip as a Stringable in memory.
|
||||||
$ip = (string) $this->ip;
|
return self::isPlaceholderIp((string) $this->ip);
|
||||||
|
}
|
||||||
|
|
||||||
return blank($ip) || in_array($ip, [self::PLACEHOLDER_IP, '0.0.0.0', '::'], true);
|
public static function isPlaceholderIp(?string $ip): bool
|
||||||
|
{
|
||||||
|
return blank($ip) || in_array($ip, self::PLACEHOLDER_IPS, true);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
@ -328,13 +334,66 @@ public function hasPlaceholderIp(): bool
|
||||||
*/
|
*/
|
||||||
public function backfillPlaceholderIp(?string $ip): bool
|
public function backfillPlaceholderIp(?string $ip): bool
|
||||||
{
|
{
|
||||||
if ($ip && $this->hasPlaceholderIp()) {
|
if (self::isPlaceholderIp($ip)) {
|
||||||
$this->update(['ip' => $ip]);
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$updated = static::query()
|
||||||
|
->whereKey($this->getKey())
|
||||||
|
->where(function (Builder $query): void {
|
||||||
|
$query->whereNull('ip')
|
||||||
|
->orWhere('ip', '')
|
||||||
|
->orWhereIn('ip', self::PLACEHOLDER_IPS);
|
||||||
|
})
|
||||||
|
->update(['ip' => $ip]);
|
||||||
|
|
||||||
|
if ($updated === 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->forceFill(['ip' => $ip]);
|
||||||
|
$this->syncOriginalAttribute('ip');
|
||||||
|
static::flushIdentityMap();
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
/**
|
||||||
|
* Persist provider status without saving a stale in-memory IP value.
|
||||||
|
*
|
||||||
|
* @param array<string, mixed> $updates
|
||||||
|
*/
|
||||||
|
private function persistProviderState(array $updates): void
|
||||||
|
{
|
||||||
|
if (empty($updates)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
static::query()->whereKey($this->getKey())->update($updates);
|
||||||
|
$this->forceFill($updates);
|
||||||
|
$this->syncOriginalAttributes(array_keys($updates));
|
||||||
|
static::flushIdentityMap();
|
||||||
|
}
|
||||||
|
|
||||||
|
public function refreshHetznerState(): ?string
|
||||||
|
{
|
||||||
|
if (! $this->hetzner_server_id || ! $this->cloudProviderToken || $this->cloudProviderToken->provider !== 'hetzner') {
|
||||||
|
return $this->hetzner_server_status;
|
||||||
|
}
|
||||||
|
|
||||||
|
$hetznerService = new HetznerService($this->cloudProviderToken->token);
|
||||||
|
$server = $hetznerService->getServer($this->hetzner_server_id);
|
||||||
|
$status = $server['status'] ?? null;
|
||||||
|
$assignedIp = data_get($server, 'public_net.ipv4.ip') ?? data_get($server, 'public_net.ipv6.ip');
|
||||||
|
|
||||||
|
$updates = [];
|
||||||
|
if ($this->hetzner_server_status !== $status) {
|
||||||
|
$updates['hetzner_server_status'] = $status;
|
||||||
|
}
|
||||||
|
$this->persistProviderState($updates);
|
||||||
|
$this->backfillPlaceholderIp($assignedIp);
|
||||||
|
|
||||||
|
return $status;
|
||||||
}
|
}
|
||||||
|
|
||||||
public function refreshVultrState(): ?string
|
public function refreshVultrState(): ?string
|
||||||
|
|
@ -352,8 +411,7 @@ public function refreshVultrState(): ?string
|
||||||
}
|
}
|
||||||
|
|
||||||
if ($this->vultr_instance_status !== 'deleted') {
|
if ($this->vultr_instance_status !== 'deleted') {
|
||||||
$this->update(['vultr_instance_status' => 'deleted']);
|
$this->persistProviderState(['vultr_instance_status' => 'deleted']);
|
||||||
$this->forceFill(['vultr_instance_status' => 'deleted']);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return 'deleted';
|
return 'deleted';
|
||||||
|
|
@ -368,15 +426,8 @@ public function refreshVultrState(): ?string
|
||||||
if ($this->vultr_instance_status !== $status) {
|
if ($this->vultr_instance_status !== $status) {
|
||||||
$updates['vultr_instance_status'] = $status;
|
$updates['vultr_instance_status'] = $status;
|
||||||
}
|
}
|
||||||
|
$this->persistProviderState($updates);
|
||||||
if ($this->hasPlaceholderIp() && $publicIp) {
|
$this->backfillPlaceholderIp($publicIp);
|
||||||
$updates['ip'] = $publicIp;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (! empty($updates)) {
|
|
||||||
$this->update($updates);
|
|
||||||
$this->forceFill($updates);
|
|
||||||
}
|
|
||||||
|
|
||||||
return $status;
|
return $status;
|
||||||
}
|
}
|
||||||
|
|
@ -393,7 +444,7 @@ public function refreshDigitalOceanState(): ?string
|
||||||
$droplet = $digitalOceanService->getDroplet((int) $this->digitalocean_droplet_id);
|
$droplet = $digitalOceanService->getDroplet((int) $this->digitalocean_droplet_id);
|
||||||
} catch (RequestException $e) {
|
} catch (RequestException $e) {
|
||||||
if ($e->response?->status() === 404) {
|
if ($e->response?->status() === 404) {
|
||||||
$this->update(['digitalocean_droplet_status' => 'deleted']);
|
$this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
|
||||||
|
|
||||||
return 'deleted';
|
return 'deleted';
|
||||||
}
|
}
|
||||||
|
|
@ -401,7 +452,7 @@ public function refreshDigitalOceanState(): ?string
|
||||||
throw $e;
|
throw $e;
|
||||||
} catch (\Throwable $e) {
|
} catch (\Throwable $e) {
|
||||||
if ((int) $e->getCode() === 404) {
|
if ((int) $e->getCode() === 404) {
|
||||||
$this->update(['digitalocean_droplet_status' => 'deleted']);
|
$this->persistProviderState(['digitalocean_droplet_status' => 'deleted']);
|
||||||
|
|
||||||
return 'deleted';
|
return 'deleted';
|
||||||
}
|
}
|
||||||
|
|
@ -416,12 +467,8 @@ public function refreshDigitalOceanState(): ?string
|
||||||
$status = $droplet['status'] ?? null;
|
$status = $droplet['status'] ?? null;
|
||||||
$ip = $digitalOceanService->getPublicIpAddress($droplet);
|
$ip = $digitalOceanService->getPublicIpAddress($droplet);
|
||||||
|
|
||||||
$updates = ['digitalocean_droplet_status' => $status];
|
$this->persistProviderState(['digitalocean_droplet_status' => $status]);
|
||||||
if ($ip && $this->hasPlaceholderIp()) {
|
$this->backfillPlaceholderIp($ip);
|
||||||
$updates['ip'] = $ip;
|
|
||||||
}
|
|
||||||
|
|
||||||
$this->update($updates);
|
|
||||||
|
|
||||||
return $status;
|
return $status;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@
|
||||||
use App\Models\CloudProviderToken;
|
use App\Models\CloudProviderToken;
|
||||||
use App\Models\InstanceSettings;
|
use App\Models\InstanceSettings;
|
||||||
use App\Models\PrivateKey;
|
use App\Models\PrivateKey;
|
||||||
|
use App\Models\Server;
|
||||||
use App\Models\Team;
|
use App\Models\Team;
|
||||||
use App\Models\User;
|
use App\Models\User;
|
||||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||||
|
|
@ -133,6 +134,109 @@
|
||||||
&& $request['user_data'] === '#cloud-config');
|
&& $request['user_data'] === '#cloud-config');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('tracks the droplet with a placeholder IP when waiting for its IP fails', function () {
|
||||||
|
Http::fake([
|
||||||
|
'https://api.digitalocean.com/v2/account/keys' => Http::response([
|
||||||
|
'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
|
||||||
|
], 201),
|
||||||
|
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
|
||||||
|
'ssh_keys' => [],
|
||||||
|
'links' => ['pages' => []],
|
||||||
|
], 200),
|
||||||
|
'https://api.digitalocean.com/v2/droplets' => Http::response([
|
||||||
|
'droplet' => [
|
||||||
|
'id' => 987,
|
||||||
|
'name' => 'waiting-for-ip',
|
||||||
|
'status' => 'new',
|
||||||
|
'networks' => ['v4' => [], 'v6' => []],
|
||||||
|
],
|
||||||
|
], 202),
|
||||||
|
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
|
||||||
|
'message' => 'temporary provider failure',
|
||||||
|
], 500),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$response = $this->withHeaders([
|
||||||
|
'Authorization' => 'Bearer '.$this->bearerToken,
|
||||||
|
'Content-Type' => 'application/json',
|
||||||
|
])->postJson('/api/v1/servers/digitalocean', [
|
||||||
|
'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
|
||||||
|
'region' => 'nyc1',
|
||||||
|
'size' => 's-1vcpu-1gb',
|
||||||
|
'image' => 'ubuntu-24-04-x64',
|
||||||
|
'name' => 'waiting-for-ip',
|
||||||
|
'private_key_uuid' => $this->privateKey->uuid,
|
||||||
|
]);
|
||||||
|
|
||||||
|
$response->assertCreated();
|
||||||
|
$response->assertJsonFragment([
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
'ip' => Server::PLACEHOLDER_IP,
|
||||||
|
]);
|
||||||
|
|
||||||
|
$this->assertDatabaseHas('servers', [
|
||||||
|
'name' => 'waiting-for-ip',
|
||||||
|
'ip' => Server::PLACEHOLDER_IP,
|
||||||
|
'team_id' => $this->team->id,
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
'digitalocean_droplet_status' => 'new',
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('deletes the droplet when local server persistence fails', function () {
|
||||||
|
Http::fake([
|
||||||
|
'https://api.digitalocean.com/v2/account/keys' => Http::response([
|
||||||
|
'ssh_key' => ['id' => 123, 'fingerprint' => 'aa:bb:cc:dd'],
|
||||||
|
], 201),
|
||||||
|
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
|
||||||
|
'ssh_keys' => [],
|
||||||
|
'links' => ['pages' => []],
|
||||||
|
], 200),
|
||||||
|
'https://api.digitalocean.com/v2/droplets' => Http::response([
|
||||||
|
'droplet' => [
|
||||||
|
'id' => 987,
|
||||||
|
'name' => 'persistence-fails',
|
||||||
|
'status' => 'active',
|
||||||
|
'networks' => [
|
||||||
|
'v4' => [
|
||||||
|
['ip_address' => '203.0.113.10', 'type' => 'public'],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
], 202),
|
||||||
|
'https://api.digitalocean.com/v2/droplets/987' => Http::response(null, 204),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$eventDispatcher = Server::getEventDispatcher();
|
||||||
|
Server::setEventDispatcher(clone $eventDispatcher);
|
||||||
|
Server::created(function (): void {
|
||||||
|
throw new RuntimeException('local persistence failed');
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
$response = $this->withHeaders([
|
||||||
|
'Authorization' => 'Bearer '.$this->bearerToken,
|
||||||
|
'Content-Type' => 'application/json',
|
||||||
|
])->postJson('/api/v1/servers/digitalocean', [
|
||||||
|
'cloud_provider_token_id' => $this->digitalOceanToken->uuid,
|
||||||
|
'region' => 'nyc1',
|
||||||
|
'size' => 's-1vcpu-1gb',
|
||||||
|
'image' => 'ubuntu-24-04-x64',
|
||||||
|
'name' => 'persistence-fails',
|
||||||
|
'private_key_uuid' => $this->privateKey->uuid,
|
||||||
|
]);
|
||||||
|
} finally {
|
||||||
|
Server::setEventDispatcher($eventDispatcher);
|
||||||
|
}
|
||||||
|
|
||||||
|
$response->assertServerError();
|
||||||
|
$this->assertDatabaseMissing('servers', [
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
]);
|
||||||
|
Http::assertSent(fn ($request) => $request->method() === 'DELETE'
|
||||||
|
&& $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
|
||||||
|
});
|
||||||
|
|
||||||
test('validates required fields', function () {
|
test('validates required fields', function () {
|
||||||
$response = $this->withHeaders([
|
$response = $this->withHeaders([
|
||||||
'Authorization' => 'Bearer '.$this->bearerToken,
|
'Authorization' => 'Bearer '.$this->bearerToken,
|
||||||
|
|
|
||||||
|
|
@ -113,6 +113,54 @@ function submitDigitalOceanServer(): void
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('deletes the droplet when local server persistence fails', function () {
|
||||||
|
Http::fake([
|
||||||
|
'https://api.digitalocean.com/v2/account/keys' => Http::response([
|
||||||
|
'ssh_key' => ['id' => 123],
|
||||||
|
], 201),
|
||||||
|
'https://api.digitalocean.com/v2/account/keys*' => Http::response([
|
||||||
|
'ssh_keys' => [],
|
||||||
|
], 200),
|
||||||
|
'https://api.digitalocean.com/v2/droplets' => Http::response([
|
||||||
|
'droplet' => [
|
||||||
|
'id' => 555,
|
||||||
|
'status' => 'active',
|
||||||
|
'networks' => [
|
||||||
|
'v4' => [
|
||||||
|
['type' => 'public', 'ip_address' => '203.0.113.40'],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
], 202),
|
||||||
|
'https://api.digitalocean.com/v2/droplets/555' => Http::response(null, 204),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$eventDispatcher = Server::getEventDispatcher();
|
||||||
|
Server::setEventDispatcher(clone $eventDispatcher);
|
||||||
|
Server::created(function (): void {
|
||||||
|
throw new RuntimeException('local persistence failed');
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
Livewire::test(ByDigitalOcean::class, ['selectedTokenUuid' => $this->digitalOceanToken->uuid])
|
||||||
|
->set('server_name', 'persistence-fails')
|
||||||
|
->set('selected_region', 'nyc1')
|
||||||
|
->set('selected_size', 's-1vcpu-1gb')
|
||||||
|
->set('selected_image', 'ubuntu-24-04-x64')
|
||||||
|
->set('private_key_id', $this->privateKey->id)
|
||||||
|
->call('submit')
|
||||||
|
->assertDispatched('error', 'local persistence failed');
|
||||||
|
} finally {
|
||||||
|
Server::setEventDispatcher($eventDispatcher);
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->assertDatabaseMissing('servers', [
|
||||||
|
'digitalocean_droplet_id' => 555,
|
||||||
|
]);
|
||||||
|
Http::assertSent(fn ($request) => $request->method() === 'DELETE'
|
||||||
|
&& $request->url() === 'https://api.digitalocean.com/v2/droplets/555');
|
||||||
|
});
|
||||||
|
|
||||||
it('renders only the full width buy button at the bottom of the DigitalOcean form', function () {
|
it('renders only the full width buy button at the bottom of the DigitalOcean form', function () {
|
||||||
Livewire::test(ByDigitalOcean::class)
|
Livewire::test(ByDigitalOcean::class)
|
||||||
->set('current_step', 2)
|
->set('current_step', 2)
|
||||||
|
|
|
||||||
149
tests/Feature/ServerCloudProviderStatusCheckJobTest.php
Normal file
149
tests/Feature/ServerCloudProviderStatusCheckJobTest.php
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
use App\Jobs\ServerCloudProviderStatusCheckJob;
|
||||||
|
use App\Jobs\ServerConnectionCheckJob;
|
||||||
|
use App\Jobs\ServerManagerJob;
|
||||||
|
use App\Models\CloudProviderToken;
|
||||||
|
use App\Models\InstanceSettings;
|
||||||
|
use App\Models\PrivateKey;
|
||||||
|
use App\Models\Server;
|
||||||
|
use App\Models\Team;
|
||||||
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||||
|
use Illuminate\Support\Carbon;
|
||||||
|
use Illuminate\Support\Facades\Http;
|
||||||
|
use Illuminate\Support\Facades\Queue;
|
||||||
|
|
||||||
|
uses(RefreshDatabase::class);
|
||||||
|
|
||||||
|
function createCloudServerForStatusCheckJobTest(string $provider, array $attributes = []): Server
|
||||||
|
{
|
||||||
|
$team = Team::factory()->create();
|
||||||
|
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
|
||||||
|
$token = CloudProviderToken::create([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'provider' => $provider,
|
||||||
|
'token' => "test-{$provider}-token",
|
||||||
|
'name' => ucfirst($provider),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return Server::factory()->create(array_merge([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'private_key_id' => $privateKey->id,
|
||||||
|
'cloud_provider_token_id' => $token->id,
|
||||||
|
'ip' => Server::PLACEHOLDER_IP,
|
||||||
|
], $attributes));
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(function () {
|
||||||
|
InstanceSettings::forceCreate([
|
||||||
|
'id' => 0,
|
||||||
|
'instance_timezone' => 'UTC',
|
||||||
|
]);
|
||||||
|
Carbon::setTestNow('2026-07-11 12:00:00');
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(function () {
|
||||||
|
Carbon::setTestNow();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('syncs provider state for a placeholder server without scheduling SSH', function () {
|
||||||
|
$server = createCloudServerForStatusCheckJobTest('digitalocean', [
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
'digitalocean_droplet_status' => 'new',
|
||||||
|
]);
|
||||||
|
|
||||||
|
Queue::fake();
|
||||||
|
|
||||||
|
(new ServerManagerJob)->handle();
|
||||||
|
|
||||||
|
Queue::assertPushed(ServerCloudProviderStatusCheckJob::class, fn (ServerCloudProviderStatusCheckJob $job) => $job->server->is($server));
|
||||||
|
Queue::assertNotPushed(ServerConnectionCheckJob::class);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('backfills a DigitalOcean placeholder without changing reachability', function () {
|
||||||
|
$server = createCloudServerForStatusCheckJobTest('digitalocean', [
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
'digitalocean_droplet_status' => 'new',
|
||||||
|
]);
|
||||||
|
$server->settings->update([
|
||||||
|
'is_reachable' => true,
|
||||||
|
'is_usable' => true,
|
||||||
|
]);
|
||||||
|
|
||||||
|
Http::fake([
|
||||||
|
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
|
||||||
|
'droplet' => [
|
||||||
|
'id' => 987,
|
||||||
|
'status' => 'active',
|
||||||
|
'networks' => [
|
||||||
|
'v4' => [
|
||||||
|
['type' => 'public', 'ip_address' => '203.0.113.10'],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
(new ServerCloudProviderStatusCheckJob($server))->handle();
|
||||||
|
|
||||||
|
$server->refresh();
|
||||||
|
expect($server->ip)->toBe('203.0.113.10')
|
||||||
|
->and($server->digitalocean_droplet_status)->toBe('active')
|
||||||
|
->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
|
||||||
|
->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('backfills a Hetzner placeholder without changing reachability', function () {
|
||||||
|
$server = createCloudServerForStatusCheckJobTest('hetzner', [
|
||||||
|
'hetzner_server_id' => 123,
|
||||||
|
'hetzner_server_status' => 'starting',
|
||||||
|
]);
|
||||||
|
$server->settings->update([
|
||||||
|
'is_reachable' => true,
|
||||||
|
'is_usable' => true,
|
||||||
|
]);
|
||||||
|
|
||||||
|
Http::fake([
|
||||||
|
'https://api.hetzner.cloud/v1/servers/123' => Http::response([
|
||||||
|
'server' => [
|
||||||
|
'id' => 123,
|
||||||
|
'status' => 'running',
|
||||||
|
'public_net' => [
|
||||||
|
'ipv4' => ['ip' => '198.51.100.20'],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
(new ServerCloudProviderStatusCheckJob($server))->handle();
|
||||||
|
|
||||||
|
$server->refresh();
|
||||||
|
expect($server->ip)->toBe('198.51.100.20')
|
||||||
|
->and($server->hetzner_server_status)->toBe('running')
|
||||||
|
->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
|
||||||
|
->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('only calls the API matching the cloud provider token', function () {
|
||||||
|
$server = createCloudServerForStatusCheckJobTest('digitalocean', [
|
||||||
|
'vultr_instance_id' => 'unrelated-vultr-instance',
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
'digitalocean_droplet_status' => 'new',
|
||||||
|
]);
|
||||||
|
|
||||||
|
Http::fake([
|
||||||
|
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
|
||||||
|
'droplet' => [
|
||||||
|
'id' => 987,
|
||||||
|
'status' => 'active',
|
||||||
|
'networks' => ['v4' => []],
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
'https://api.vultr.com/*' => Http::response(status: 500),
|
||||||
|
]);
|
||||||
|
|
||||||
|
(new ServerCloudProviderStatusCheckJob($server))->handle();
|
||||||
|
|
||||||
|
Http::assertSentCount(1);
|
||||||
|
Http::assertSent(fn ($request) => $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
|
||||||
|
});
|
||||||
80
tests/Feature/ServerConnectionCheckIsolationTest.php
Normal file
80
tests/Feature/ServerConnectionCheckIsolationTest.php
Normal file
|
|
@ -0,0 +1,80 @@
|
||||||
|
<?php
|
||||||
|
|
||||||
|
use App\Jobs\ServerConnectionCheckJob;
|
||||||
|
use App\Models\CloudProviderToken;
|
||||||
|
use App\Models\PrivateKey;
|
||||||
|
use App\Models\Server;
|
||||||
|
use App\Models\Team;
|
||||||
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||||
|
use Illuminate\Support\Facades\Http;
|
||||||
|
use Illuminate\Support\Facades\Process;
|
||||||
|
use Illuminate\Support\Facades\Storage;
|
||||||
|
|
||||||
|
uses(RefreshDatabase::class);
|
||||||
|
|
||||||
|
function createServerForConnectionIsolationTest(array $attributes = []): Server
|
||||||
|
{
|
||||||
|
$team = Team::factory()->create();
|
||||||
|
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
|
||||||
|
|
||||||
|
return Server::factory()->create(array_merge([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'private_key_id' => $privateKey->id,
|
||||||
|
], $attributes));
|
||||||
|
}
|
||||||
|
|
||||||
|
beforeEach(function () {
|
||||||
|
Storage::fake('ssh-keys');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('never attempts SSH to a placeholder address', function (string $placeholderIp) {
|
||||||
|
$server = createServerForConnectionIsolationTest(['ip' => $placeholderIp]);
|
||||||
|
$server->settings->update([
|
||||||
|
'is_reachable' => true,
|
||||||
|
'is_usable' => true,
|
||||||
|
]);
|
||||||
|
|
||||||
|
Process::fake();
|
||||||
|
|
||||||
|
(new ServerConnectionCheckJob($server, disableMux: false))->handle();
|
||||||
|
|
||||||
|
Process::assertNothingRan();
|
||||||
|
expect($server->fresh()->unreachable_count)->toBe(0)
|
||||||
|
->and((bool) $server->settings->fresh()->is_reachable)->toBeTrue()
|
||||||
|
->and((bool) $server->settings->fresh()->is_usable)->toBeTrue();
|
||||||
|
})->with([
|
||||||
|
'reserved provisioning address' => Server::PLACEHOLDER_IP,
|
||||||
|
'unspecified IPv4 address' => '0.0.0.0',
|
||||||
|
'unspecified IPv6 address' => '::',
|
||||||
|
]);
|
||||||
|
|
||||||
|
it('does not call cloud provider APIs during an SSH connection check', function () {
|
||||||
|
$server = createServerForConnectionIsolationTest(['ip' => '203.0.113.10']);
|
||||||
|
$token = CloudProviderToken::create([
|
||||||
|
'team_id' => $server->team_id,
|
||||||
|
'provider' => 'vultr',
|
||||||
|
'token' => 'test-vultr-token',
|
||||||
|
'name' => 'Vultr',
|
||||||
|
]);
|
||||||
|
$server->update([
|
||||||
|
'cloud_provider_token_id' => $token->id,
|
||||||
|
'vultr_instance_id' => 'instance-1',
|
||||||
|
'vultr_instance_status' => 'active',
|
||||||
|
]);
|
||||||
|
|
||||||
|
Http::fake([
|
||||||
|
'https://api.vultr.com/*' => Http::response([
|
||||||
|
'instance' => ['id' => 'instance-1', 'status' => 'active'],
|
||||||
|
]),
|
||||||
|
]);
|
||||||
|
Process::fake([
|
||||||
|
'*' => Process::result(
|
||||||
|
output: '{"Server":{"Version":"27.0.0"}}',
|
||||||
|
exitCode: 0,
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
|
(new ServerConnectionCheckJob($server->fresh(), disableMux: false))->handle();
|
||||||
|
|
||||||
|
Http::assertNothingSent();
|
||||||
|
});
|
||||||
|
|
@ -1,11 +1,15 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
|
use App\Http\Controllers\Api\VultrController;
|
||||||
use App\Models\CloudProviderToken;
|
use App\Models\CloudProviderToken;
|
||||||
use App\Models\InstanceSettings;
|
use App\Models\InstanceSettings;
|
||||||
use App\Models\PrivateKey;
|
use App\Models\PrivateKey;
|
||||||
|
use App\Models\Server;
|
||||||
use App\Models\Team;
|
use App\Models\Team;
|
||||||
use App\Models\User;
|
use App\Models\User;
|
||||||
|
use Illuminate\Auth\Access\AuthorizationException;
|
||||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||||
|
use Illuminate\Http\Request;
|
||||||
use Illuminate\Support\Facades\Http;
|
use Illuminate\Support\Facades\Http;
|
||||||
|
|
||||||
uses(RefreshDatabase::class);
|
uses(RefreshDatabase::class);
|
||||||
|
|
@ -165,6 +169,27 @@ function testPublicKey(): string
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('member read token cannot use a stored Vultr token for metadata endpoints', function (string $endpoint) {
|
||||||
|
$member = User::factory()->create();
|
||||||
|
$this->team->members()->attach($member->id, ['role' => 'member']);
|
||||||
|
$memberToken = $member->createToken('member-read', ['read'])->plainTextToken;
|
||||||
|
|
||||||
|
Http::fake();
|
||||||
|
|
||||||
|
$response = $this->withHeaders([
|
||||||
|
'Authorization' => 'Bearer '.$memberToken,
|
||||||
|
'Content-Type' => 'application/json',
|
||||||
|
])->getJson($endpoint.'?cloud_provider_token_uuid='.$this->vultrToken->uuid);
|
||||||
|
|
||||||
|
$response->assertForbidden();
|
||||||
|
Http::assertNothingSent();
|
||||||
|
})->with([
|
||||||
|
'/api/v1/vultr/regions',
|
||||||
|
'/api/v1/vultr/plans',
|
||||||
|
'/api/v1/vultr/os',
|
||||||
|
'/api/v1/vultr/ssh-keys',
|
||||||
|
]);
|
||||||
|
|
||||||
describe('POST /api/v1/servers/vultr', function () {
|
describe('POST /api/v1/servers/vultr', function () {
|
||||||
test('creates a Vultr server', function () {
|
test('creates a Vultr server', function () {
|
||||||
Http::fake([
|
Http::fake([
|
||||||
|
|
@ -223,6 +248,11 @@ function testPublicKey(): string
|
||||||
});
|
});
|
||||||
|
|
||||||
test('waits for Vultr to assign a real public IP when creation returns placeholder IP', function () {
|
test('waits for Vultr to assign a real public IP when creation returns placeholder IP', function () {
|
||||||
|
$createdIp = null;
|
||||||
|
Server::created(function (Server $server) use (&$createdIp): void {
|
||||||
|
$createdIp = (string) $server->ip;
|
||||||
|
});
|
||||||
|
|
||||||
Http::fake([
|
Http::fake([
|
||||||
'https://api.vultr.com/v2/ssh-keys' => Http::response([
|
'https://api.vultr.com/v2/ssh-keys' => Http::response([
|
||||||
'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],
|
'ssh_key' => ['id' => 'key-1', 'ssh_key' => testPublicKey()],
|
||||||
|
|
@ -262,6 +292,8 @@ function testPublicKey(): string
|
||||||
$response->assertStatus(201);
|
$response->assertStatus(201);
|
||||||
$response->assertJsonFragment(['ip' => '1.2.3.4']);
|
$response->assertJsonFragment(['ip' => '1.2.3.4']);
|
||||||
|
|
||||||
|
expect($createdIp)->toBe(Server::PLACEHOLDER_IP);
|
||||||
|
|
||||||
$this->assertDatabaseHas('servers', [
|
$this->assertDatabaseHas('servers', [
|
||||||
'name' => 'test-server',
|
'name' => 'test-server',
|
||||||
'ip' => '1.2.3.4',
|
'ip' => '1.2.3.4',
|
||||||
|
|
@ -269,6 +301,34 @@ function testPublicKey(): string
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('server creation authorizes access to the stored Vultr token', function () {
|
||||||
|
$member = User::factory()->create();
|
||||||
|
$this->team->members()->attach($member->id, ['role' => 'member']);
|
||||||
|
$this->actingAs($member);
|
||||||
|
$member->withAccessToken($member->createToken('member-all', ['*'])->accessToken);
|
||||||
|
|
||||||
|
$request = Request::create(
|
||||||
|
uri: '/api/v1/servers/vultr',
|
||||||
|
method: 'POST',
|
||||||
|
server: ['CONTENT_TYPE' => 'application/json'],
|
||||||
|
content: json_encode([
|
||||||
|
'cloud_provider_token_uuid' => $this->vultrToken->uuid,
|
||||||
|
'region' => 'ewr',
|
||||||
|
'plan' => 'vc2-1c-1gb',
|
||||||
|
'os_id' => 2284,
|
||||||
|
'name' => 'test-server',
|
||||||
|
'private_key_uuid' => $this->privateKey->uuid,
|
||||||
|
], JSON_THROW_ON_ERROR),
|
||||||
|
);
|
||||||
|
$request->setUserResolver(fn () => $member);
|
||||||
|
|
||||||
|
Http::fake();
|
||||||
|
|
||||||
|
expect(fn () => app(VultrController::class)->createServer($request))
|
||||||
|
->toThrow(AuthorizationException::class);
|
||||||
|
Http::assertNothingSent();
|
||||||
|
});
|
||||||
|
|
||||||
test('reuses existing matching Vultr SSH key', function () {
|
test('reuses existing matching Vultr SSH key', function () {
|
||||||
Http::fake([
|
Http::fake([
|
||||||
'https://api.vultr.com/v2/ssh-keys*' => Http::response([
|
'https://api.vultr.com/v2/ssh-keys*' => Http::response([
|
||||||
|
|
|
||||||
|
|
@ -123,6 +123,8 @@ function vultrLifecycleTestPrivateKey(): string
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not overwrite an established server IP during Vultr status refresh', function () {
|
it('does not overwrite an established server IP during Vultr status refresh', function () {
|
||||||
|
$this->server->update(['ip' => '198.51.100.20']);
|
||||||
|
|
||||||
Http::fake([
|
Http::fake([
|
||||||
'https://api.vultr.com/v2/instances/instance-1' => Http::response([
|
'https://api.vultr.com/v2/instances/instance-1' => Http::response([
|
||||||
'instance' => [
|
'instance' => [
|
||||||
|
|
@ -135,12 +137,12 @@ function vultrLifecycleTestPrivateKey(): string
|
||||||
|
|
||||||
Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
|
Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
|
||||||
->call('checkVultrInstanceStatus', true)
|
->call('checkVultrInstanceStatus', true)
|
||||||
->assertSet('ip', '1.2.3.4')
|
->assertSet('ip', '198.51.100.20')
|
||||||
->assertSet('vultrInstanceStatus', 'active');
|
->assertSet('vultrInstanceStatus', 'active');
|
||||||
|
|
||||||
$this->assertDatabaseHas('servers', [
|
$this->assertDatabaseHas('servers', [
|
||||||
'id' => $this->server->id,
|
'id' => $this->server->id,
|
||||||
'ip' => '1.2.3.4',
|
'ip' => '198.51.100.20',
|
||||||
'vultr_instance_status' => 'active',
|
'vultr_instance_status' => 'active',
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
@ -270,6 +272,26 @@ function vultrLifecycleTestPrivateKey(): string
|
||||||
Http::assertSent(fn ($request) => $request->url() === 'https://api.vultr.com/v2/instances/instance-1/start');
|
Http::assertSent(fn ($request) => $request->url() === 'https://api.vultr.com/v2/instances/instance-1/start');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('prevents members from starting a Vultr instance', function () {
|
||||||
|
$this->server->update(['vultr_instance_status' => 'stopped']);
|
||||||
|
|
||||||
|
$member = User::factory()->create();
|
||||||
|
$this->team->members()->attach($member->id, ['role' => 'member']);
|
||||||
|
$this->actingAs($member);
|
||||||
|
session(['currentTeam' => $this->team]);
|
||||||
|
|
||||||
|
Http::fake([
|
||||||
|
'https://api.vultr.com/v2/instances/instance-1/start' => Http::response([], 204),
|
||||||
|
]);
|
||||||
|
|
||||||
|
Livewire::test(Show::class, ['server_uuid' => $this->server->uuid])
|
||||||
|
->call('startVultrInstance')
|
||||||
|
->assertDispatched('error');
|
||||||
|
|
||||||
|
expect($this->server->fresh()->vultr_instance_status)->toBe('stopped');
|
||||||
|
Http::assertNothingSent();
|
||||||
|
});
|
||||||
|
|
||||||
it('links a server to Vultr by matching IP', function () {
|
it('links a server to Vultr by matching IP', function () {
|
||||||
$unlinkedServer = Server::factory()->create([
|
$unlinkedServer = Server::factory()->create([
|
||||||
'team_id' => $this->team->id,
|
'team_id' => $this->team->id,
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@
|
||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
use App\Models\Team;
|
use App\Models\Team;
|
||||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||||
|
use Illuminate\Http\Client\RequestException;
|
||||||
use Illuminate\Support\Facades\Http;
|
use Illuminate\Support\Facades\Http;
|
||||||
use Tests\TestCase;
|
use Tests\TestCase;
|
||||||
|
|
||||||
|
|
@ -44,3 +45,59 @@
|
||||||
Http::assertSent(fn ($request) => $request->method() === 'DELETE'
|
Http::assertSent(fn ($request) => $request->method() === 'DELETE'
|
||||||
&& $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
|
&& $request->url() === 'https://api.digitalocean.com/v2/droplets/987');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('retains the server and surfaces a DigitalOcean deletion failure', function () {
|
||||||
|
Http::fake([
|
||||||
|
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
|
||||||
|
'message' => 'deletion failed',
|
||||||
|
], 500),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$team = Team::factory()->create();
|
||||||
|
$token = CloudProviderToken::factory()->create([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'provider' => 'digitalocean',
|
||||||
|
'token' => 'test-digitalocean-token',
|
||||||
|
]);
|
||||||
|
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
|
||||||
|
$server = Server::factory()->create([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'private_key_id' => $privateKey->id,
|
||||||
|
'cloud_provider_token_id' => $token->id,
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
]);
|
||||||
|
$server->delete();
|
||||||
|
|
||||||
|
expect(fn () => DeleteServer::run(
|
||||||
|
serverId: $server->id,
|
||||||
|
deleteFromDigitalOcean: true,
|
||||||
|
digitalOceanDropletId: 987,
|
||||||
|
cloudProviderTokenId: $token->id,
|
||||||
|
teamId: $team->id,
|
||||||
|
))->toThrow(RequestException::class, 'status code 500');
|
||||||
|
|
||||||
|
expect(Server::withTrashed()->find($server->id))->not->toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('retains the server when no DigitalOcean token can delete the droplet', function () {
|
||||||
|
Http::preventStrayRequests();
|
||||||
|
|
||||||
|
$team = Team::factory()->create();
|
||||||
|
$privateKey = PrivateKey::factory()->create(['team_id' => $team->id]);
|
||||||
|
$server = Server::factory()->create([
|
||||||
|
'team_id' => $team->id,
|
||||||
|
'private_key_id' => $privateKey->id,
|
||||||
|
'cloud_provider_token_id' => null,
|
||||||
|
'digitalocean_droplet_id' => 987,
|
||||||
|
]);
|
||||||
|
$server->delete();
|
||||||
|
|
||||||
|
expect(fn () => DeleteServer::run(
|
||||||
|
serverId: $server->id,
|
||||||
|
deleteFromDigitalOcean: true,
|
||||||
|
digitalOceanDropletId: 987,
|
||||||
|
teamId: $team->id,
|
||||||
|
))->toThrow(RuntimeException::class, 'No DigitalOcean token found');
|
||||||
|
|
||||||
|
expect(Server::withTrashed()->find($server->id))->not->toBeNull();
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -66,8 +66,8 @@ function createDigitalOceanServerForStateTest(string $status = 'active', array $
|
||||||
expect($server->fresh()->ip)->toBe('203.0.113.10');
|
expect($server->fresh()->ip)->toBe('203.0.113.10');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not overwrite a real IP from the DigitalOcean droplet state', function () {
|
it('does not overwrite an administrator configured address from the DigitalOcean droplet state', function (string $configuredAddress) {
|
||||||
$server = createDigitalOceanServerForStateTest('active', ['ip' => '198.51.100.20']);
|
$server = createDigitalOceanServerForStateTest('active', ['ip' => $configuredAddress]);
|
||||||
|
|
||||||
Http::fake([
|
Http::fake([
|
||||||
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
|
'https://api.digitalocean.com/v2/droplets/987' => Http::response([
|
||||||
|
|
@ -84,7 +84,38 @@ function createDigitalOceanServerForStateTest(string $status = 'active', array $
|
||||||
]);
|
]);
|
||||||
|
|
||||||
$server->refreshDigitalOceanState();
|
$server->refreshDigitalOceanState();
|
||||||
expect($server->fresh()->ip)->toBe('198.51.100.20');
|
expect($server->fresh()->ip)->toBe($configuredAddress);
|
||||||
|
})->with([
|
||||||
|
'public IPv4' => '198.51.100.20',
|
||||||
|
'private IPv4' => '10.10.0.12',
|
||||||
|
'IPv6' => '2001:db8::10',
|
||||||
|
'tunnel hostname' => 'server.internal.example.com',
|
||||||
|
]);
|
||||||
|
|
||||||
|
it('does not overwrite an address configured while DigitalOcean state is loading', function () {
|
||||||
|
$server = createDigitalOceanServerForStateTest('new', ['ip' => Server::PLACEHOLDER_IP]);
|
||||||
|
|
||||||
|
Http::fake(function () use ($server) {
|
||||||
|
Server::query()->findOrFail($server->id)->update([
|
||||||
|
'ip' => 'server.internal.example.com',
|
||||||
|
]);
|
||||||
|
|
||||||
|
return Http::response([
|
||||||
|
'droplet' => [
|
||||||
|
'id' => 987,
|
||||||
|
'status' => 'active',
|
||||||
|
'networks' => [
|
||||||
|
'v4' => [
|
||||||
|
['type' => 'public', 'ip_address' => '203.0.113.10'],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
],
|
||||||
|
], 200);
|
||||||
|
});
|
||||||
|
|
||||||
|
expect($server->refreshDigitalOceanState())->toBe('active')
|
||||||
|
->and($server->fresh()->ip)->toBe('server.internal.example.com')
|
||||||
|
->and($server->digitalocean_droplet_status)->toBe('active');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('does not mark a DigitalOcean droplet as deleted on transient provider errors', function () {
|
it('does not mark a DigitalOcean droplet as deleted on transient provider errors', function () {
|
||||||
|
|
|
||||||
|
|
@ -127,7 +127,7 @@
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('ServerConnectionCheckJob unreachable_count', function () {
|
describe('ServerConnectionCheckJob unreachable_count', function () {
|
||||||
it('marks Vultr servers unreachable when provider status is unavailable', function () {
|
it('marks servers unreachable when SSH is unavailable', function () {
|
||||||
Event::fake([ServerReachabilityChanged::class]);
|
Event::fake([ServerReachabilityChanged::class]);
|
||||||
|
|
||||||
$settings = Mockery::mock();
|
$settings = Mockery::mock();
|
||||||
|
|
@ -141,14 +141,12 @@
|
||||||
$server->shouldReceive('getAttribute')->andReturnUsing(fn (string $key) => match ($key) {
|
$server->shouldReceive('getAttribute')->andReturnUsing(fn (string $key) => match ($key) {
|
||||||
'settings' => $settings,
|
'settings' => $settings,
|
||||||
'unreachable_notification_sent' => false,
|
'unreachable_notification_sent' => false,
|
||||||
'vultr_instance_id' => 'instance-1',
|
'ip' => '203.0.113.10',
|
||||||
'cloudProviderToken' => (object) ['token' => 'test-token'],
|
|
||||||
'id' => 1,
|
'id' => 1,
|
||||||
'name' => 'test-server',
|
'name' => 'test-server',
|
||||||
'unreachable_count' => 1,
|
'unreachable_count' => 1,
|
||||||
default => null,
|
default => null,
|
||||||
});
|
});
|
||||||
$server->shouldReceive('refreshVultrState')->once()->andReturn('stopped');
|
|
||||||
$server->shouldReceive('increment')->with('unreachable_count')->once();
|
$server->shouldReceive('increment')->with('unreachable_count')->once();
|
||||||
$server->id = 1;
|
$server->id = 1;
|
||||||
$server->name = 'test-server';
|
$server->name = 'test-server';
|
||||||
|
|
@ -158,22 +156,20 @@
|
||||||
$job->handle();
|
$job->handle();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('increments unreachable_count on timeout', function () {
|
it('does not change reachability on a job-level timeout', function () {
|
||||||
Event::fake([ServerReachabilityChanged::class]);
|
Event::fake([ServerReachabilityChanged::class]);
|
||||||
|
|
||||||
$settings = Mockery::mock();
|
$settings = Mockery::mock();
|
||||||
$settings->is_reachable = true;
|
$settings->is_reachable = true;
|
||||||
$settings->shouldReceive('update')
|
$settings->shouldNotReceive('update');
|
||||||
->with(['is_reachable' => false, 'is_usable' => false])
|
|
||||||
->once();
|
|
||||||
|
|
||||||
$server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
|
$server = Mockery::mock(Server::class)->makePartial()->shouldAllowMockingProtectedMethods();
|
||||||
$server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);
|
$server->shouldReceive('getAttribute')->with('settings')->andReturn($settings);
|
||||||
$server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn(false);
|
$server->shouldReceive('getAttribute')->with('unreachable_notification_sent')->andReturn(false);
|
||||||
$server->shouldReceive('increment')->with('unreachable_count')->once();
|
$server->shouldNotReceive('increment');
|
||||||
$server->id = 1;
|
$server->id = 1;
|
||||||
$server->name = 'test-server';
|
$server->name = 'test-server';
|
||||||
$server->unreachable_count = 1; // Will become 2 after increment in real code; mock keeps value as-is
|
$server->unreachable_count = 1;
|
||||||
|
|
||||||
$job = new ServerConnectionCheckJob($server);
|
$job = new ServerConnectionCheckJob($server);
|
||||||
$job->failed(new TimeoutExceededException);
|
$job->failed(new TimeoutExceededException);
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,7 @@
|
||||||
<?php
|
<?php
|
||||||
|
|
||||||
|
use App\Jobs\CleanupOrphanedPreviewContainersJob;
|
||||||
|
use App\Jobs\ScheduledJobManager;
|
||||||
use App\Models\PrivateKey;
|
use App\Models\PrivateKey;
|
||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
use App\Models\Team;
|
use App\Models\Team;
|
||||||
|
|
@ -60,8 +62,64 @@ function createServerForPlaceholderIpTest(string $ip): Server
|
||||||
expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
|
expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('does not replace a placeholder with another placeholder', function (string $replacementIp) {
|
||||||
|
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
|
||||||
|
|
||||||
|
expect($server->backfillPlaceholderIp($replacementIp))->toBeFalse();
|
||||||
|
expect($server->fresh()->ip)->toBe(Server::PLACEHOLDER_IP);
|
||||||
|
})->with([
|
||||||
|
'unspecified IPv4 address' => '0.0.0.0',
|
||||||
|
'unspecified IPv6 address' => '::',
|
||||||
|
]);
|
||||||
|
|
||||||
|
it('does not overwrite an address configured after the placeholder model was loaded', function () {
|
||||||
|
$staleServer = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
|
||||||
|
$concurrentServer = Server::query()->findOrFail($staleServer->id);
|
||||||
|
|
||||||
|
$concurrentServer->update(['ip' => 'server.internal.example.com']);
|
||||||
|
|
||||||
|
expect($staleServer->backfillPlaceholderIp('203.0.113.10'))->toBeFalse()
|
||||||
|
->and($staleServer->fresh()->ip)->toBe('server.internal.example.com');
|
||||||
|
});
|
||||||
|
|
||||||
it('skips servers with a placeholder IP in scheduled jobs', function () {
|
it('skips servers with a placeholder IP in scheduled jobs', function () {
|
||||||
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
|
$server = createServerForPlaceholderIpTest(Server::PLACEHOLDER_IP);
|
||||||
|
|
||||||
expect($server->skipServer())->toBeTrue();
|
expect($server->skipServer())->toBeTrue();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('excludes every placeholder address from scheduled Docker cleanup', function () {
|
||||||
|
$realServer = createServerForPlaceholderIpTest('203.0.113.10');
|
||||||
|
|
||||||
|
foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
|
||||||
|
Server::factory()->create([
|
||||||
|
'team_id' => $realServer->team_id,
|
||||||
|
'private_key_id' => $realServer->private_key_id,
|
||||||
|
'ip' => $placeholderIp,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
$method = new ReflectionMethod(ScheduledJobManager::class, 'getServersForCleanupQuery');
|
||||||
|
$servers = $method->invoke(new ScheduledJobManager)->get();
|
||||||
|
|
||||||
|
expect($servers->modelKeys())->toBe([$realServer->id]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('excludes every placeholder address from orphaned preview cleanup', function () {
|
||||||
|
$realServer = createServerForPlaceholderIpTest('203.0.113.10');
|
||||||
|
$realServer->settings->update(['is_reachable' => true, 'is_usable' => true]);
|
||||||
|
|
||||||
|
foreach (Server::PLACEHOLDER_IPS as $placeholderIp) {
|
||||||
|
$server = Server::factory()->create([
|
||||||
|
'team_id' => $realServer->team_id,
|
||||||
|
'private_key_id' => $realServer->private_key_id,
|
||||||
|
'ip' => $placeholderIp,
|
||||||
|
]);
|
||||||
|
$server->settings->update(['is_reachable' => true, 'is_usable' => true]);
|
||||||
|
}
|
||||||
|
|
||||||
|
$method = new ReflectionMethod(CleanupOrphanedPreviewContainersJob::class, 'getServersToCheck');
|
||||||
|
$servers = $method->invoke(new CleanupOrphanedPreviewContainersJob);
|
||||||
|
|
||||||
|
expect($servers->modelKeys())->toBe([$realServer->id]);
|
||||||
|
});
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@
|
||||||
use App\Models\Server;
|
use App\Models\Server;
|
||||||
use App\Models\Team;
|
use App\Models\Team;
|
||||||
use Illuminate\Foundation\Testing\RefreshDatabase;
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
||||||
|
use Illuminate\Http\Client\RequestException;
|
||||||
use Illuminate\Support\Facades\Http;
|
use Illuminate\Support\Facades\Http;
|
||||||
use Tests\TestCase;
|
use Tests\TestCase;
|
||||||
|
|
||||||
|
|
@ -116,3 +117,52 @@ function vultrDeleteTestPrivateKey(): string
|
||||||
|
|
||||||
expect($request->header('Authorization'))->toBe(['Bearer test-vultr-token']);
|
expect($request->header('Authorization'))->toBe(['Bearer test-vultr-token']);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('retains the server and surfaces a Vultr deletion failure', function () {
|
||||||
|
Http::fake([
|
||||||
|
'https://api.vultr.com/v2/instances/instance-1' => Http::response([
|
||||||
|
'error' => 'deletion failed',
|
||||||
|
], 500),
|
||||||
|
]);
|
||||||
|
|
||||||
|
$server = Server::factory()->create([
|
||||||
|
'team_id' => $this->team->id,
|
||||||
|
'private_key_id' => $this->privateKey->id,
|
||||||
|
'cloud_provider_token_id' => $this->vultrToken->id,
|
||||||
|
'vultr_instance_id' => 'instance-1',
|
||||||
|
]);
|
||||||
|
$server->delete();
|
||||||
|
|
||||||
|
expect(fn () => DeleteServer::run(
|
||||||
|
serverId: $server->id,
|
||||||
|
cloudProviderTokenId: $this->vultrToken->id,
|
||||||
|
teamId: $this->team->id,
|
||||||
|
deleteFromVultr: true,
|
||||||
|
vultrInstanceId: 'instance-1'
|
||||||
|
))->toThrow(RequestException::class, 'status code 500');
|
||||||
|
|
||||||
|
expect(Server::withTrashed()->find($server->id))->not->toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('retains the server when no Vultr token can delete the instance', function () {
|
||||||
|
Http::preventStrayRequests();
|
||||||
|
|
||||||
|
$this->vultrToken->delete();
|
||||||
|
|
||||||
|
$server = Server::factory()->create([
|
||||||
|
'team_id' => $this->team->id,
|
||||||
|
'private_key_id' => $this->privateKey->id,
|
||||||
|
'cloud_provider_token_id' => null,
|
||||||
|
'vultr_instance_id' => 'instance-1',
|
||||||
|
]);
|
||||||
|
$server->delete();
|
||||||
|
|
||||||
|
expect(fn () => DeleteServer::run(
|
||||||
|
serverId: $server->id,
|
||||||
|
teamId: $this->team->id,
|
||||||
|
deleteFromVultr: true,
|
||||||
|
vultrInstanceId: 'instance-1'
|
||||||
|
))->toThrow(RuntimeException::class, 'No Vultr token found');
|
||||||
|
|
||||||
|
expect(Server::withTrashed()->find($server->id))->not->toBeNull();
|
||||||
|
});
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue