> $servers Self-contained per-server teardown payload captured before the cascade. */ public function __construct( public int $teamId, public array $servers, ) {} /** * Collect the team's v5 servers (with their applications and SSH key * material) into a self-contained payload and dispatch the teardown job. * * Must be called from the Team `deleting` hook, while the rows still exist. * Returns without dispatching when the team owns no v5 servers. */ public static function dispatchForTeam(Team $team): void { // Guard against contexts where the v5 tables do not exist (e.g. v4-only // schemas) so team deletion is never broken by this teardown. if (! Schema::hasTable('v5_servers')) { return; } $servers = V5Server::query() ->where('team_id', $team->id) ->with('privateKey') ->get(); if ($servers->isEmpty()) { return; } $applicationsByServer = V5Application::query() ->where('team_id', $team->id) ->whereNotNull('server_id') ->get() ->groupBy('server_id'); $payload = $servers->map(function (V5Server $server) use ($applicationsByServer): array { return [ 'id' => $server->id, 'uuid' => $server->uuid, 'name' => $server->name, 'host' => $server->host, 'ssh_user' => $server->ssh_user, 'ssh_port' => (int) $server->ssh_port, 'node_address' => $server->node_address, 'wireguard_management_ip' => $server->wireguard_management_ip, 'is_ingress' => (bool) $server->is_ingress, 'ingress_type' => $server->ingress_type, 'status' => $server->status, 'last_bootstrapped_at' => $server->last_bootstrapped_at?->toISOString(), // Captured before the cascade removes the row so the job can // revoke the host token after the DB rows are gone. 'agent_token_jti' => $server->agent_token_jti, 'agent_token_expires_at' => $server->agent_token_expires_at?->toISOString(), // Encrypted at rest on the model; needed to SSH into the host. 'private_key' => $server->privateKey instanceof PrivateKey ? $server->privateKey->private_key : null, 'applications' => ($applicationsByServer[$server->id] ?? collect()) ->map(fn (V5Application $application): array => [ 'id' => $application->id, 'container_name' => $application->container_name, 'runtime_container_id' => $application->runtime_container_id, ]) ->values() ->all(), ]; })->all(); self::dispatch($team->id, $payload); } public function handle(): void { $incompleteHosts = []; foreach ($this->servers as $serverPayload) { if (! $this->teardownServer($serverPayload)) { $incompleteHosts[] = [ 'server_id' => $serverPayload['id'] ?? null, 'host' => $serverPayload['host'] ?? null, ]; } } // Teardown is best-effort and never fails the job (an unreachable host // must not abort the others), so this is the single operator-facing // signal that some hosts could not be reached and may now hold orphaned // containers/mesh with no DB row left to reconcile them. if ($incompleteHosts !== []) { Log::error('v5 team teardown incomplete — '.count($incompleteHosts).' host(s) may have orphaned containers/mesh', [ 'team_id' => $this->teamId, 'hosts' => $incompleteHosts, ]); } } /** * Tear down a single host. Returns false when any on-host teardown step * (container removal, ingress stop, bootstrap-marker removal) failed, so the * caller can surface the host as potentially orphaned. Never throws: a * single unreachable host must not abort teardown of the other hosts. * * @param array $serverPayload */ private function teardownServer(array $serverPayload): bool { $server = $this->reconstructServer($serverPayload); $serverId = $serverPayload['id'] ?? null; $host = $serverPayload['host'] ?? null; $succeeded = true; foreach ($serverPayload['applications'] ?? [] as $applicationPayload) { try { $application = $this->reconstructApplication($applicationPayload, $server); DestroyNginxApplication::run($application); } catch (\Throwable $exception) { $succeeded = false; Log::warning('V5 team teardown: failed to remove application container', [ 'team_id' => $this->teamId, 'server_id' => $serverId, 'host' => $host, 'container_name' => $applicationPayload['container_name'] ?? null, 'error' => $exception->getMessage(), ]); } } if ($server->isIngress() && $server->status === ServerStatus::Installed->value) { try { StopCaddyIngress::run($server); } catch (\Throwable $exception) { $succeeded = false; Log::warning('V5 team teardown: failed to stop Caddy ingress', [ 'team_id' => $this->teamId, 'server_id' => $serverId, 'host' => $host, 'error' => $exception->getMessage(), ]); } } if (($serverPayload['last_bootstrapped_at'] ?? null) !== null) { try { if (! RemoveBootstrapMarker::run($server)) { $succeeded = false; Log::warning('V5 team teardown: could not remove on-host bootstrap identity over SSH', [ 'team_id' => $this->teamId, 'server_id' => $serverId, 'host' => $host, ]); } } catch (\Throwable $exception) { $succeeded = false; Log::warning('V5 team teardown: bootstrap marker removal threw', [ 'team_id' => $this->teamId, 'server_id' => $serverId, 'host' => $host, 'error' => $exception->getMessage(), ]); } } // Revocation is best-effort and independent of the on-host cleanup: a // failed flux push does not mean the host is orphaned, so it never flips // $succeeded (it is logged separately inside AgentTokenIssuer::revoke). $this->revokeAgentTokenIfSupported($server, $serverId, $host); return $succeeded; } /** * Reconstruct a non-persisted V5Server (with its private key relation * pre-set) so the teardown actions never hit the deleted DB rows. * * @param array $serverPayload */ private function reconstructServer(array $serverPayload): V5Server { $server = new V5Server; $server->forceFill([ 'id' => $serverPayload['id'] ?? null, 'uuid' => $serverPayload['uuid'] ?? null, 'name' => $serverPayload['name'] ?? null, 'host' => $serverPayload['host'] ?? null, 'ssh_user' => $serverPayload['ssh_user'] ?? null, 'ssh_port' => $serverPayload['ssh_port'] ?? 22, 'node_address' => $serverPayload['node_address'] ?? null, 'wireguard_management_ip' => $serverPayload['wireguard_management_ip'] ?? null, 'is_ingress' => (bool) ($serverPayload['is_ingress'] ?? false), 'ingress_type' => $serverPayload['ingress_type'] ?? null, 'status' => $serverPayload['status'] ?? null, 'agent_token_jti' => $serverPayload['agent_token_jti'] ?? null, 'agent_token_expires_at' => $serverPayload['agent_token_expires_at'] ?? null, ]); // Non-persisted: StopCaddyIngress / the actions must not try to update a // row that the cascade already removed. $server->exists = false; $privateKeyMaterial = $serverPayload['private_key'] ?? null; if (is_string($privateKeyMaterial) && $privateKeyMaterial !== '') { $privateKey = new PrivateKey; $privateKey->forceFill(['private_key' => $privateKeyMaterial]); $server->setRelation('privateKey', $privateKey); } else { $server->setRelation('privateKey', null); } return $server; } /** * @param array $applicationPayload */ private function reconstructApplication(array $applicationPayload, V5Server $server): V5Application { $application = new V5Application; $application->forceFill([ 'id' => $applicationPayload['id'] ?? null, 'container_name' => $applicationPayload['container_name'] ?? null, 'runtime_container_id' => $applicationPayload['runtime_container_id'] ?? null, 'server_id' => $server->id, ]); $application->exists = false; $application->setRelation('server', $server); return $application; } /** * If a coold-side agent-token revocation ever lands on AgentTokenIssuer, * call it best-effort. Guarded so this job never hard-depends on a method * that may not exist yet. */ private function revokeAgentTokenIfSupported(V5Server $server, mixed $serverId, mixed $host): void { if (! method_exists(AgentTokenIssuer::class, 'revokeForServer')) { return; } try { app(AgentTokenIssuer::class)->revokeForServer($server); } catch (\Throwable $exception) { Log::warning('V5 team teardown: agent token revocation failed', [ 'team_id' => $this->teamId, 'server_id' => $serverId, 'host' => $host, 'error' => $exception->getMessage(), ]); } } }