refactor(v5): defer builder bootstrap support
Remove active builder flags from dev Lima sync, bootstrap commands, and server capabilities while documenting the v5 control-plane and host-agent boundaries.
This commit is contained in:
Andras Bacsai
parent
068a2979fd
commit
4938b65f20
22 changed files with 361 additions and 219 deletions
|
|
@ -29,7 +29,6 @@ public function handle(
|
|||
User $user,
|
||||
?PrivateKey $privateKey,
|
||||
string $clusterName,
|
||||
int $builderCapacity,
|
||||
array $servers,
|
||||
): Cluster {
|
||||
$cluster = Cluster::query()->updateOrCreate([
|
||||
|
|
@ -40,9 +39,7 @@ public function handle(
|
|||
'description' => 'Local Lima development cluster managed by scripts/dev.sh.',
|
||||
]);
|
||||
|
||||
$builderCapacity = max(0, $builderCapacity);
|
||||
$builderEnabled = $builderCapacity > 0;
|
||||
$capabilities = $builderEnabled ? ['coold', 'builder'] : ['coold'];
|
||||
$capabilities = ['coold'];
|
||||
|
||||
foreach ($servers as $server) {
|
||||
$wireguardManagementIp = $server['wireguard_management_ip'] ?? null;
|
||||
|
|
@ -54,8 +51,8 @@ public function handle(
|
|||
'ssh_port' => $server['ssh_port'],
|
||||
'status' => 'installed',
|
||||
'capabilities' => $capabilities,
|
||||
'builder_enabled' => $builderEnabled,
|
||||
'builder_capacity' => $builderCapacity,
|
||||
'builder_enabled' => false,
|
||||
'builder_capacity' => 0,
|
||||
'node_address' => $wireguardManagementIp ?: $server['host'],
|
||||
'wireguard_management_ip' => $wireguardManagementIp,
|
||||
'last_bootstrapped_at' => now(),
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ class V5SyncDevLimaServers extends Command
|
|||
{--user-id=0 : User recorded as creator}
|
||||
{--private-key-id= : Optional private key used by the dev servers}
|
||||
{--cluster=Development-Lima : Cluster name for the dev Lima servers}
|
||||
{--builder-capacity=2 : Builder capacity to record for each dev server}
|
||||
{--server=* : Server as name|host|ssh_user|ssh_port|wireguard_management_ip}
|
||||
{--force : Allow running outside local/development environments}';
|
||||
|
||||
|
|
@ -82,7 +81,6 @@ public function handle(): int
|
|||
user: $user,
|
||||
privateKey: $privateKey,
|
||||
clusterName: (string) $this->option('cluster'),
|
||||
builderCapacity: (int) $this->option('builder-capacity'),
|
||||
servers: $parsedServers,
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1057,26 +1057,6 @@ private function bootstrapCommand(V5Cluster $cluster, Collection $servers, V5Ser
|
|||
$command[] = '--skip-default-deny';
|
||||
}
|
||||
|
||||
$builderServers = $servers->filter(fn (V5Server $server) => $server->builder_enabled);
|
||||
if ($cluster->builder_enabled && $builderServers->isNotEmpty()) {
|
||||
array_push(
|
||||
$command,
|
||||
'--enable-builder',
|
||||
'--builder-hosts',
|
||||
$builderServers
|
||||
->map(fn (V5Server $server) => $this->bootstrapNode($server))
|
||||
->implode(','),
|
||||
'--builder-capacity',
|
||||
(string) $cluster->builder_capacity,
|
||||
'--builder-cpu-quota',
|
||||
$cluster->builder_cpu_quota,
|
||||
'--builder-memory-max',
|
||||
$cluster->builder_memory_max,
|
||||
'--builder-timeout-secs',
|
||||
(string) $cluster->builder_timeout_secs,
|
||||
);
|
||||
}
|
||||
|
||||
$command[] = '--yes';
|
||||
|
||||
return $command;
|
||||
|
|
@ -1519,7 +1499,6 @@ private function privateKeys(mixed $currentTeam): array
|
|||
private function serverCapabilities(bool $builderEnabled, bool $ingressEnabled): array
|
||||
{
|
||||
return collect(['coold'])
|
||||
->when($builderEnabled, fn ($capabilities) => $capabilities->push('builder'))
|
||||
->when($ingressEnabled, fn ($capabilities) => $capabilities->push('ingress'))
|
||||
->unique()
|
||||
->values()
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ public function handle(): void
|
|||
return;
|
||||
}
|
||||
|
||||
$result = Process::timeout(max(60, (int) $cluster->builder_timeout_secs + 120))
|
||||
$result = Process::timeout(300)
|
||||
->run($this->bootstrapCommand($cluster, $servers, $server, $sshConfigLocation, $action));
|
||||
$output = trim($result->output()."\n".$result->errorOutput());
|
||||
$successful = $result->successful();
|
||||
|
|
@ -119,7 +119,6 @@ public function handle(): void
|
|||
|
||||
$capabilities = collect($server->capabilities ?? [])
|
||||
->push('coold')
|
||||
->when($server->builder_enabled, fn ($capabilities) => $capabilities->push('builder'))
|
||||
->when($server->isIngress(), fn ($capabilities) => $capabilities->push('ingress'))
|
||||
->unique()
|
||||
->values()
|
||||
|
|
@ -229,26 +228,6 @@ private function bootstrapCommand(V5Cluster $cluster, Collection $servers, V5Ser
|
|||
$command[] = '--skip-default-deny';
|
||||
}
|
||||
|
||||
$builderServers = $servers->filter(fn (V5Server $server) => $server->builder_enabled);
|
||||
if ($cluster->builder_enabled && $builderServers->isNotEmpty()) {
|
||||
array_push(
|
||||
$command,
|
||||
'--enable-builder',
|
||||
'--builder-hosts',
|
||||
$builderServers
|
||||
->map(fn (V5Server $server) => $this->bootstrapNode($server))
|
||||
->implode(','),
|
||||
'--builder-capacity',
|
||||
(string) $cluster->builder_capacity,
|
||||
'--builder-cpu-quota',
|
||||
$cluster->builder_cpu_quota,
|
||||
'--builder-memory-max',
|
||||
$cluster->builder_memory_max,
|
||||
'--builder-timeout-secs',
|
||||
(string) $cluster->builder_timeout_secs,
|
||||
);
|
||||
}
|
||||
|
||||
$command[] = '--yes';
|
||||
|
||||
return $command;
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ class FluxClient
|
|||
public function listContainers(string $hostId): array
|
||||
{
|
||||
$payload = $this->dispatch($hostId, [
|
||||
'type' => 'list_containers',
|
||||
'type' => 'containers.list',
|
||||
]);
|
||||
|
||||
$data = $payload['data'] ?? [];
|
||||
|
|
|
|||
|
|
@ -4,7 +4,5 @@
|
|||
'coolify_cli_bin' => env('COOLIFY_CLI_BIN', '/usr/local/bin/coolify'),
|
||||
'coold_version' => env('COOLIFY_COOLD_VERSION', 'nightly'),
|
||||
'corrosion_version' => env('COOLIFY_CORROSION_VERSION', 'v1.0.0'),
|
||||
'dev_builder_capacity' => (int) env('COOLIFY_COOLD_VM_BUILDER_CAPACITY', 2),
|
||||
'dev_builder_enabled' => (int) env('COOLIFY_COOLD_VM_BUILDER_CAPACITY', 2) > 0,
|
||||
'dev_ssh_user' => env('COOLIFY_CLI_SSH_USER', 'coolify'),
|
||||
];
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ public function run(): void
|
|||
->orderBy('id')
|
||||
->first();
|
||||
|
||||
$builderCapacity = max(0, (int) config('coold.dev_builder_capacity', 2));
|
||||
$sshUser = (string) config('coold.dev_ssh_user', 'coolify');
|
||||
$servers = collect($this->servers())
|
||||
->map(fn (array $server): array => [
|
||||
|
|
@ -44,7 +43,6 @@ public function run(): void
|
|||
user: $user,
|
||||
privateKey: $privateKey,
|
||||
clusterName: self::CLUSTER_NAME,
|
||||
builderCapacity: $builderCapacity,
|
||||
servers: $servers,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ ## Documents
|
|||
| [Responsibility split](responsibility-split.md) | What belongs in Coolify, Flux, and coold. |
|
||||
| [Primitives](primitives.md) | Canonical host primitive surface Coolify may dispatch. |
|
||||
| [Deploy flows](deploy-flows.md) | User-functionality flows, starting with `nginx:alpine`. |
|
||||
| [ADRs](adr/README.md) | Architecture decision records for Coolify, Flux, coold, and their contracts. |
|
||||
| [ADR 0001](adr/0001-control-plane-flux-coold-split.md) | Decision record for the v5 split. |
|
||||
|
||||
## Core rule
|
||||
|
|
|
|||
|
|
@ -12,8 +12,10 @@ ## Context
|
|||
long-lived agent streams or directly expose host runtime sockets.
|
||||
|
||||
Host operations also need a narrow privileged boundary. Podman, firewall, DNS,
|
||||
Corrosion, and build supervision require local host privileges that should not
|
||||
be spread across the Laravel app or arbitrary scripts.
|
||||
and Corrosion require local host privileges that should not be spread across the
|
||||
Laravel app or arbitrary scripts. Future builder supervision belongs behind the
|
||||
same boundary, but its active primitive/API shape is deferred to a separate
|
||||
decision.
|
||||
|
||||
## Decision
|
||||
|
||||
|
|
@ -27,7 +29,7 @@ ## Decision
|
|||
primitive requests to connected hosts and resolves pending responses.
|
||||
3. **coold** runs once per host. It exposes a closed set of host primitives and
|
||||
owns privileged local execution through Podman, firewall, DNS, Corrosion,
|
||||
host facts, and builder supervision.
|
||||
and host facts.
|
||||
|
||||
coold must not expose raw Podman passthrough. Every supported operation must be
|
||||
an explicit primitive with validation and a stable protocol shape.
|
||||
|
|
|
|||
66
docs/v5/architecture/adr/0002-coold-host-agent-boundary.md
Normal file
66
docs/v5/architecture/adr/0002-coold-host-agent-boundary.md
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
# ADR 0002: Keep coold as a narrow host agent
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
Coolify v5 needs a process on each managed host that can reach local runtime
|
||||
surfaces such as Podman, Corrosion, DNS bind addresses, and firewall state. That
|
||||
process must run close to privileged host APIs, but Coolify's product model,
|
||||
RBAC, deployment history, billing, and audit state belong in the Laravel control
|
||||
plane.
|
||||
|
||||
If coold grows app-aware behavior, it becomes a second control plane with local
|
||||
copies of product rules. If Coolify reaches around coold with raw host access,
|
||||
the privileged boundary disappears and host behavior becomes harder to validate.
|
||||
|
||||
## Decision
|
||||
|
||||
coold is a per-host agent with a narrow, explicit primitive surface. It executes
|
||||
host-local operations requested through Flux, reports typed results, and owns
|
||||
host safety checks for those operations.
|
||||
|
||||
coold owns local runtime integration: Podman access, service-discovery sync,
|
||||
embedded DNS, Corrosion writes for this host's endpoints, host facts, and the
|
||||
firewall mutation/reconciliation surface when those primitives are active.
|
||||
|
||||
coold must not own Coolify product concepts. It does not decide what an
|
||||
application, project, team, deployment, domain, rollback, billing event, or audit
|
||||
record means. Those concepts remain in Coolify Laravel. coold also must not
|
||||
expose raw Podman passthrough; every operation needs an explicit primitive with
|
||||
validation and a stable protocol shape.
|
||||
|
||||
Builder supervision is intentionally deferred. When it returns, it should be
|
||||
recorded in a separate ADR/API because scheduling, capacity, logs, artifacts,
|
||||
cancellation, restart adoption, and registry flow need their own trade-off.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- coold can be reasoned about as reusable host infrastructure, not a Coolify app
|
||||
runtime.
|
||||
- Privileged host access has one narrow boundary with explicit validation.
|
||||
- The primitive API can evolve through reviewed protocol additions instead of
|
||||
ad hoc Podman command exposure.
|
||||
- Coolify Laravel remains the only owner of durable product state and business
|
||||
audit.
|
||||
|
||||
### Negative
|
||||
|
||||
- New deploy features may require new coold primitives before they can ship.
|
||||
- Some operations are more verbose than raw Podman passthrough.
|
||||
- Cross-repo changes must keep Coolify, Flux, and coold protocol expectations in
|
||||
sync.
|
||||
|
||||
## Boundary rules
|
||||
|
||||
- If the question is "is this user/team allowed to do this?", Coolify answers.
|
||||
- If the question is "is this host operation safe to execute here?", coold may
|
||||
reject it.
|
||||
- If the operation only makes sense because of Coolify's product model, it stays
|
||||
out of coold.
|
||||
- If another orchestrator could safely reuse the same host operation, it is a
|
||||
candidate coold primitive.
|
||||
|
|
@ -0,0 +1,65 @@
|
|||
# ADR 0003: Keep Flux as the connection broker
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
Coolify v5 needs to send host primitive requests to many managed hosts, including
|
||||
hosts behind NAT, firewalls, or corporate networks. coold can solve the host
|
||||
inbound problem by dialing out, but Laravel request workers should not own
|
||||
thousands of long-lived HTTP/2 agent streams or in-memory pending response maps.
|
||||
|
||||
The system also needs a place to translate between Coolify's short-lived local
|
||||
request/response lane and coold's long-lived outbound stream without turning that
|
||||
place into another product control plane.
|
||||
|
||||
## Decision
|
||||
|
||||
Flux is the central connection broker between Coolify Laravel and coold agents.
|
||||
Laravel talks to Flux over a local Unix socket. coold agents dial Flux over an
|
||||
outbound authenticated gRPC stream. Flux keeps the connected-host stream
|
||||
registry, routes requests to the selected host stream, tracks pending request IDs,
|
||||
and resolves typed responses back to Laravel.
|
||||
|
||||
Flux owns transport concerns: stream lifecycle, request correlation, timeouts,
|
||||
disconnected-host responses, pending-request caps, late-result handling, and
|
||||
host-agent authentication for inbound coold streams.
|
||||
|
||||
Flux must not own Coolify product concepts. It does not decide which user may
|
||||
deploy, which host should run an application, what a domain means, how rollback
|
||||
works, or how deployment state advances. Flux treats `containers.start` or
|
||||
`images.pull` as protocol frames routed to a host, not as product actions.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Hosts can remain behind NAT because coold dials out to Flux.
|
||||
- Laravel stays focused on durable product state and short-lived request work.
|
||||
- Long-lived stream management can scale and fail independently from PHP-FPM
|
||||
workers.
|
||||
- Backpressure, timeouts, and disconnected-host behavior have one transport
|
||||
boundary.
|
||||
- Flux can be tested as a protocol router without needing Coolify's app model.
|
||||
|
||||
### Negative
|
||||
|
||||
- The architecture has one more runtime component to deploy and monitor.
|
||||
- Protocol changes must stay coordinated across Coolify, Flux, and coold.
|
||||
- Flux outage blocks dispatch to connected hosts even when Laravel and hosts are
|
||||
otherwise healthy.
|
||||
- Flux must stay intentionally narrow; adding product decisions would create a
|
||||
second control plane.
|
||||
|
||||
## Boundary rules
|
||||
|
||||
- Coolify chooses the target host; Flux only routes to that host's connected
|
||||
stream.
|
||||
- coold authenticates to Flux as a host agent; Laravel reaches Flux through the
|
||||
local Unix-socket lane.
|
||||
- Flux may answer transport failures such as disconnected host, timeout, or
|
||||
pending-cap overflow.
|
||||
- Flux may not inspect product ownership, RBAC, billing, deployment state,
|
||||
domains, secrets, or audit meaning.
|
||||
|
|
@ -0,0 +1,70 @@
|
|||
# ADR 0004: Keep Coolify Laravel as the product control plane
|
||||
|
||||
## Status
|
||||
|
||||
Accepted.
|
||||
|
||||
## Context
|
||||
|
||||
Coolify v5 manages user intent across teams, projects, environments,
|
||||
applications, services, databases, servers, domains, deployments, secrets,
|
||||
notifications, billing, and audit history. Those concepts require durable state,
|
||||
RBAC, validation, UI/API workflows, and user-facing status.
|
||||
|
||||
The v5 runtime also needs Flux and coold so hosts behind NAT can receive work and
|
||||
privileged host operations stay close to the host. If product decisions move into
|
||||
Flux or coold, the system gains multiple control planes with duplicated rules and
|
||||
unclear ownership. If Coolify directly mutates hosts, the Flux/coold boundary is
|
||||
bypassed and host safety checks become optional.
|
||||
|
||||
## Decision
|
||||
|
||||
Coolify Laravel is the product control plane for v5. It owns user intent,
|
||||
durable product state, RBAC, API tokens, sessions, SSO/OAuth, projects,
|
||||
environments, resources, deployment state machines, placement decisions, proxy
|
||||
and ingress intent, secret resolution, notifications, billing/subscriptions,
|
||||
business audit, deployment logs, and user-facing status.
|
||||
|
||||
Coolify chooses what should happen and which host should receive the work. It
|
||||
then turns product intent into ordered host primitives and dispatches them
|
||||
through Flux to coold. Coolify records primitive results and advances the durable
|
||||
resource or deployment state.
|
||||
|
||||
Coolify must not hold long-lived coold streams or directly expose privileged host
|
||||
runtime sockets as part of normal v5 operation. It also must not rely on Flux or
|
||||
coold to understand product concepts such as teams, applications, domains,
|
||||
rollbacks, billing, or audit meaning.
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
|
||||
- Product behavior has one durable source of truth.
|
||||
- RBAC, validation, audit, and user-facing status stay near the UI/API and
|
||||
database model.
|
||||
- Flux and coold remain reusable infrastructure boundaries instead of becoming
|
||||
hidden product layers.
|
||||
- Deployment state machines can be tested against typed primitive results rather
|
||||
than raw host side effects.
|
||||
|
||||
### Negative
|
||||
|
||||
- Coolify must model enough deployment state to recover from partial host
|
||||
failures and retries.
|
||||
- New user-facing features may require both product changes in Coolify and new
|
||||
primitives in coold.
|
||||
- Coolify cannot shortcut missing primitives by using raw host access without
|
||||
weakening the architecture boundary.
|
||||
|
||||
## Boundary rules
|
||||
|
||||
- If the decision involves a user, team, project, environment, resource,
|
||||
deployment, domain, secret, notification, billing event, or audit record,
|
||||
Coolify owns it.
|
||||
- If the decision is which connected host should receive work, Coolify chooses
|
||||
and Flux routes.
|
||||
- If the work is a concrete host operation, Coolify dispatches an explicit
|
||||
primitive instead of mutating the host directly.
|
||||
- If Coolify cannot express a required host operation as an existing primitive,
|
||||
the protocol needs a reviewed primitive addition instead of a product-layer
|
||||
bypass.
|
||||
52
docs/v5/architecture/adr/README.md
Normal file
52
docs/v5/architecture/adr/README.md
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
# Architecture Decision Records
|
||||
|
||||
This directory records architecture decisions for the Coolify v5 ecosystem:
|
||||
Coolify Laravel, Flux, coold, and the contracts between them.
|
||||
|
||||
ADRs are for decisions that are hard to reverse, surprising without context, and
|
||||
made after a real trade-off. If a note is just reference material, put it in the
|
||||
main architecture docs instead.
|
||||
|
||||
## Index
|
||||
|
||||
| ADR | Decision |
|
||||
| --- | --- |
|
||||
| [0001](0001-control-plane-flux-coold-split.md) | Split v5 into Coolify control plane, Flux broker, and coold host agent. |
|
||||
| [0002](0002-coold-host-agent-boundary.md) | Keep coold as a narrow host agent with explicit primitives. |
|
||||
| [0003](0003-flux-connection-broker-boundary.md) | Keep Flux as the connection broker between Coolify and coold. |
|
||||
| [0004](0004-coolify-control-plane-boundary.md) | Keep Coolify Laravel as the product control plane. |
|
||||
|
||||
## Format
|
||||
|
||||
Use the next sequential number and a short slug:
|
||||
|
||||
```text
|
||||
0005-short-decision-slug.md
|
||||
```
|
||||
|
||||
Start small. A useful ADR can be one paragraph:
|
||||
|
||||
```md
|
||||
# ADR 0005: Short title of the decision
|
||||
|
||||
Coolify v5 will ... because ... This trades ... for ...
|
||||
```
|
||||
|
||||
Add optional sections only when they clarify the decision:
|
||||
|
||||
- **Status**: `Proposed`, `Accepted`, `Deprecated`, or `Superseded by ADR NNNN`.
|
||||
- **Context**: what forced the decision.
|
||||
- **Decision**: what we chose.
|
||||
- **Consequences**: non-obvious benefits, costs, or follow-up constraints.
|
||||
- **Considered options**: rejected alternatives worth remembering.
|
||||
|
||||
## Scope guide
|
||||
|
||||
- Coolify ADRs cover product/control-plane state, RBAC, deployments, billing,
|
||||
APIs, UI-visible behavior, and durable audit/history.
|
||||
- Flux ADRs cover connection brokerage, host stream routing, request/response
|
||||
correlation, backpressure, and transport-level authentication.
|
||||
- coold ADRs cover host primitives, privileged execution, Podman/firewall/DNS,
|
||||
local safety rules, and agent lifecycle.
|
||||
- Cross-cutting ADRs belong here when the decision changes contracts between
|
||||
Coolify, Flux, and coold.
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
# v5 Deploy Flows
|
||||
|
||||
These flows are written from the user-functionality view. Coolify owns the
|
||||
state machine; Flux routes primitive requests; coold executes host operations.
|
||||
|
||||
## Flow: deploy Docker image app `nginx:alpine`
|
||||
|
||||
User intent:
|
||||
|
||||
```text
|
||||
Run nginx:alpine on host H1, expose port 80, and route nginx.example.com to it.
|
||||
```
|
||||
|
||||
### Steps
|
||||
|
||||
| Step | Coolify | Flux | coold |
|
||||
| --- | --- | --- | --- |
|
||||
| Create app | Stores app, source type `docker_image`, image `nginx:alpine`, target host, port, domain. | No action. | No action. |
|
||||
| Start deploy | Creates deployment record and enters deploy state machine. | No action until dispatch. | No action. |
|
||||
| Pull image | Sends `images.pull` to H1. | Routes request to H1's stream. | Pulls image through Podman. |
|
||||
| Prepare network/volume | Sends required `networks.create` / `volumes.create` operations. | Routes requests. | Creates idempotent host resources. |
|
||||
| Create container | Sends `containers.create` with image, name, env, labels, network, port, health check, mounts, DNS. | Routes request. | Applies deny filter and creates container through Podman. |
|
||||
| Start container | Sends `containers.start`. | Routes request. | Starts container. |
|
||||
| Check status | Sends `containers.inspect` or reads status stream. | Routes request. | Reads local Podman state. |
|
||||
| Register endpoint | Sends `services.register`. | Routes request. | Writes this host's endpoint row to Corrosion. |
|
||||
| Allow traffic | Sends `firewall.allow` for proxy-to-container traffic when required. | Routes request. | Writes iptables and nft allow rules. |
|
||||
| Configure ingress | Renders proxy config from Coolify domain/resource state. If reload is needed, sends `containers.exec` or equivalent primitive for the proxy container. | Routes runtime reload request. | Executes the reload primitive only. |
|
||||
| Mark running | Stores final status, deployment result, and user-facing logs. | Resolves final responses. | Keeps container running and reports future state. |
|
||||
|
||||
### Expected data locations
|
||||
|
||||
| Data | Owner |
|
||||
| --- | --- |
|
||||
| App name, team, project, env, domain, image ref, desired port | Coolify database |
|
||||
| Deployment state and history | Coolify database |
|
||||
| Open host stream and pending request IDs | Flux memory |
|
||||
| Pulled image and running container | Host Podman storage |
|
||||
| Endpoint rows for this host | Corrosion via coold |
|
||||
| Firewall tuples and snapshots | Host kernel + `/etc/coolify` via coold |
|
||||
|
||||
### No build path
|
||||
|
||||
`nginx:alpine` is a prebuilt image. This flow does not use Git clone,
|
||||
Dockerfile, buildpacks, Railpack, or the builder subprocess.
|
||||
|
||||
## Flow: delete the nginx app
|
||||
|
||||
| Step | Coolify | Flux | coold |
|
||||
| --- | --- | --- | --- |
|
||||
| User deletes resource | Validates permission and starts cleanup state machine. | No action until dispatch. | No action. |
|
||||
| Stop container | Sends `containers.stop`. | Routes request. | Stops container. |
|
||||
| Remove container | Sends `containers.delete`. | Routes request. | Deletes container. |
|
||||
| Remove endpoint | Sends `services.unregister`. | Routes request. | Removes service endpoint row. |
|
||||
| Remove firewall rule | Sends `firewall.revoke`. | Routes request. | Removes iptables and nft allow rules. |
|
||||
| Update proxy | Removes rendered route and reloads proxy if needed. | Routes reload primitive. | Executes proxy reload primitive. |
|
||||
| Finish cleanup | Marks resource deleted and records outcome. | Resolves responses. | No product state retained. |
|
||||
|
||||
## Flow: Git app with build
|
||||
|
||||
A Git app adds a build phase before container creation.
|
||||
|
||||
```text
|
||||
Coolify resolves source + secrets + build config
|
||||
↓
|
||||
Flux routes build request to a host with builder capability
|
||||
↓
|
||||
coold supervises builder subprocess
|
||||
↓
|
||||
builder writes image/result
|
||||
↓
|
||||
Coolify deploy state machine continues with image/container primitives
|
||||
```
|
||||
|
||||
coold supervises the builder process. The builder owns build implementation.
|
||||
Coolify owns the decision to build, the build configuration, and the deployment
|
||||
state transitions around the build.
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
# v5 Architecture Overview
|
||||
|
||||
Coolify v5 separates product intent from host execution.
|
||||
|
||||
- **Coolify Laravel** is the control plane. It owns users, teams, projects,
|
||||
environments, resources, deployments, domains, secrets, RBAC, audit-worthy
|
||||
state, and the deployment state machines.
|
||||
- **Flux** is the connection broker. Laravel talks to Flux over a Unix socket.
|
||||
coold agents dial Flux over outbound gRPC streams. Flux maps host IDs to
|
||||
connected streams and maps request IDs to pending responses.
|
||||
- **coold** is the per-host executor. It owns local runtime access: Podman,
|
||||
host networks, DNS, firewall mutations, service-discovery sync, host facts,
|
||||
and builder subprocess supervision.
|
||||
|
||||
## Data flow
|
||||
|
||||
```text
|
||||
1. A user/API/webhook asks Coolify to do something.
|
||||
2. Coolify validates permissions and stores desired state in its database.
|
||||
3. Coolify's state machine turns that intent into ordered host primitives.
|
||||
4. Coolify sends each primitive to Flux over /run/coolify/flux.sock.
|
||||
5. Flux forwards the primitive to the target host's open coold stream.
|
||||
6. coold executes the primitive locally and returns a typed result.
|
||||
7. Flux resolves the pending request.
|
||||
8. Coolify records the result and moves the deployment/resource state forward.
|
||||
```
|
||||
|
||||
## Non-goals for Flux and coold
|
||||
|
||||
Flux and coold are not product layers. They do not decide which app to deploy,
|
||||
which user is allowed to deploy, how to roll back, what domains mean, or where
|
||||
business audit belongs. They only handle routing and host execution.
|
||||
|
||||
## Current implementation note
|
||||
|
||||
This directory describes the target v5 architecture. Some primitives are not
|
||||
implemented in coold yet. Until implemented, docs should label them as target
|
||||
primitives and code should not pretend they exist.
|
||||
|
|
@ -82,16 +82,11 @@ ## Host facts
|
|||
| `host.stats` | Return CPU, memory, disk, and container stats snapshot. |
|
||||
| `host.containers` | Return host container summaries. |
|
||||
|
||||
## Builder
|
||||
## Builder (deferred)
|
||||
|
||||
Builds are routed through Flux to a host with builder capability. coold should
|
||||
supervise the builder process but should not implement build logic inline.
|
||||
|
||||
| Primitive | Purpose |
|
||||
| --- | --- |
|
||||
| `build.dispatch` | Start a builder request on a capable host. |
|
||||
| `build.cancel` | Cancel a running build request. |
|
||||
| `build.result` | Return or long-poll build result. |
|
||||
Builder is intentionally not part of the active v5 Flux/coold primitive
|
||||
surface. Before reintroducing it, add an ADR/API covering scheduling, capacity,
|
||||
logs, artifacts, cancellation, restart adoption, and registry flow.
|
||||
|
||||
## Not primitives
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ INSTANCE="$(read_coolify_env COOLIFY_COOLD_LIMA_INSTANCE coold-dev)"
|
|||
VERSION="$(read_coolify_env COOLIFY_COOLD_VERSION nightly)"
|
||||
CORROSION_VERSION="$(read_coolify_env COOLIFY_CORROSION_VERSION v1.0.0)"
|
||||
FLUX_URL="$(read_coolify_env COOLIFY_COOLD_VM_FLUX_URL http://host.lima.internal:6443)"
|
||||
BUILDER_CAPACITY="$(read_coolify_env COOLIFY_COOLD_VM_BUILDER_CAPACITY 2)"
|
||||
START_TIMEOUT="$(read_coolify_env COOLIFY_COOLD_VM_START_TIMEOUT 300)"
|
||||
WG_IP="$(read_coolify_env COOLIFY_COOLD_VM_WG_IP "")"
|
||||
WG_PEER_IP="$(read_coolify_env COOLIFY_COOLD_VM_WG_PEER_IP "")"
|
||||
|
|
@ -36,10 +35,6 @@ WG_PEER_ENDPOINT="$(read_coolify_env COOLIFY_COOLD_VM_WG_PEER_ENDPOINT "")"
|
|||
WG_PEER_PUBLIC_KEY="$(read_coolify_env COOLIFY_COOLD_VM_WG_PEER_PUBLIC_KEY "")"
|
||||
CONTAINER_SUBNET="$(read_coolify_env COOLIFY_COOLD_VM_CONTAINER_SUBNET 10.210.0.0/24)"
|
||||
CONTAINER_GATEWAY="$(read_coolify_env COOLIFY_COOLD_VM_CONTAINER_GATEWAY 10.210.0.1)"
|
||||
BUILDER_ENABLED="true"
|
||||
if [ "$BUILDER_CAPACITY" = "0" ]; then
|
||||
BUILDER_ENABLED="false"
|
||||
fi
|
||||
TEMPLATE="$ROOT/dev/lima/coold.yaml"
|
||||
GUEST_COOLIFY_ROOT="/workspace/coolify"
|
||||
|
||||
|
|
@ -68,7 +63,6 @@ Environment:
|
|||
COOLIFY_COOLD_VERSION coold release tag to install (default: nightly)
|
||||
COOLIFY_CORROSION_VERSION corrosion release tag to install (default: v1.0.0)
|
||||
COOLIFY_COOLD_VM_FLUX_URL Flux gRPC URL visible from the VM (default: http://host.lima.internal:6443)
|
||||
COOLIFY_COOLD_VM_BUILDER_CAPACITY VM builder capacity to advertise (default: 2; set 0 to disable)
|
||||
COOLIFY_COOLD_VM_START_TIMEOUT Seconds to wait for Lima SSH/provisioning (default: 300)
|
||||
COOLIFY_COOLD_VM_WG_IP Optional WireGuard mgmt IP for this host
|
||||
COOLIFY_COOLD_VM_CONTAINER_SUBNET Podman mesh subnet for this host
|
||||
|
|
@ -270,7 +264,7 @@ write_runtime_config() {
|
|||
bootstrap="\"$WG_PEER_IP:8787\""
|
||||
fi
|
||||
|
||||
lima_shell sudo install -d -m 0755 /etc/corrosion/schemas /etc/coolify /run/coolify /var/lib/corrosion /var/run/corrosion /var/lib/coolify-dev /var/lib/coolify-builder/work
|
||||
lima_shell sudo install -d -m 0755 /etc/corrosion/schemas /etc/coolify /run/coolify /var/lib/corrosion /var/run/corrosion /var/lib/coolify-dev
|
||||
|
||||
lima_shell sudo tee /etc/corrosion/schemas/coolify.sql >/dev/null <<'SQL'
|
||||
CREATE TABLE service_endpoints (
|
||||
|
|
@ -315,14 +309,11 @@ run_foreground() {
|
|||
(cd /tmp && limactl shell "$INSTANCE" -- sudo \
|
||||
env COOLIFY_COOLD_HOST_MGMT_IP="${WG_IP:-127.0.0.1}" \
|
||||
COOLIFY_COOLD_FLUX_URL="$FLUX_URL" \
|
||||
COOLIFY_COOLD_BUILDER_ENABLED="$BUILDER_ENABLED" \
|
||||
COOLIFY_COOLD_BUILDER_CAPACITY="$BUILDER_CAPACITY" \
|
||||
CONTAINER_GATEWAY="$CONTAINER_GATEWAY" \
|
||||
bash -s) <<'RUNNER'
|
||||
set -euo pipefail
|
||||
|
||||
echo "coold: $(/usr/local/bin/coold --version)"
|
||||
echo "builder: $(/usr/local/bin/builder --version)"
|
||||
echo "corrosion: $(/usr/local/bin/corrosion --version 2>/dev/null || cat /usr/local/bin/corrosion.version)"
|
||||
echo "starting packaged coold endpoint with corrosion in local mode"
|
||||
|
||||
|
|
@ -340,10 +331,6 @@ COOLIFY_COOLD_NAMESPACES="${COOLIFY_COOLD_NAMESPACES:-default:coolify-default-me
|
|||
COOLIFY_COOLD_DNS_ZONE="${COOLIFY_COOLD_DNS_ZONE:-coolify.internal}" \
|
||||
COOLIFY_COOLD_FLUX_URL="${COOLIFY_COOLD_FLUX_URL:-http://host.lima.internal:6443}" \
|
||||
COOLIFY_COOLD_HOST_JWT_PATH="${COOLIFY_COOLD_HOST_JWT_PATH:-/etc/coolify/host-jwt}" \
|
||||
COOLIFY_COOLD_BUILDER_ENABLED="${COOLIFY_COOLD_BUILDER_ENABLED:-true}" \
|
||||
COOLIFY_COOLD_BUILDER_CAPACITY="${COOLIFY_COOLD_BUILDER_CAPACITY:-2}" \
|
||||
COOLIFY_COOLD_BUILDER_BIN="${COOLIFY_COOLD_BUILDER_BIN:-/usr/local/bin/builder}" \
|
||||
COOLIFY_COOLD_BUILDER_WORK_DIR="${COOLIFY_COOLD_BUILDER_WORK_DIR:-/var/lib/coolify-builder/work}" \
|
||||
/usr/local/bin/coold &
|
||||
|
||||
wait
|
||||
|
|
@ -442,10 +429,6 @@ Environment=COOLIFY_COOLD_NAMESPACES=default:coolify-default-mesh:$CONTAINER_GAT
|
|||
Environment=COOLIFY_COOLD_DNS_ZONE=coolify.internal
|
||||
Environment=COOLIFY_COOLD_FLUX_URL=$FLUX_URL
|
||||
Environment=COOLIFY_COOLD_HOST_JWT_PATH=/etc/coolify/host-jwt
|
||||
Environment=COOLIFY_COOLD_BUILDER_ENABLED=$BUILDER_ENABLED
|
||||
Environment=COOLIFY_COOLD_BUILDER_CAPACITY=$BUILDER_CAPACITY
|
||||
Environment=COOLIFY_COOLD_BUILDER_BIN=/usr/local/bin/builder
|
||||
Environment=COOLIFY_COOLD_BUILDER_WORK_DIR=/var/lib/coolify-builder/work
|
||||
ExecStart=/usr/local/bin/coold
|
||||
AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_ADMIN CAP_NET_RAW
|
||||
Restart=on-failure
|
||||
|
|
|
|||
|
|
@ -413,13 +413,8 @@ mint_host_jwt_for_host() {
|
|||
local attempts=60
|
||||
local output
|
||||
local caps
|
||||
local builder_capacity
|
||||
|
||||
builder_capacity="$(read_coolify_env COOLIFY_COOLD_VM_BUILDER_CAPACITY 2)"
|
||||
caps="coold"
|
||||
if [ "$builder_capacity" != "0" ]; then
|
||||
caps="coold,builder"
|
||||
fi
|
||||
|
||||
for attempt in $(seq 1 "$attempts"); do
|
||||
if output="$(spin exec -T coolify php artisan flux:dev "$host_id" --caps="$caps" 2>&1)"; then
|
||||
|
|
@ -496,7 +491,6 @@ sync_v5_dev_lima_servers() {
|
|||
spin exec -T \
|
||||
-e COOLIFY_CLI_SSH_USER="$ssh_user" \
|
||||
coolify php artisan v5:sync-dev-lima-servers \
|
||||
--builder-capacity="$(read_coolify_env COOLIFY_COOLD_VM_BUILDER_CAPACITY 2)" \
|
||||
"${server_args[@]}"
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
'coold VM WireGuard IP 2 default' => 'COOLIFY_COOLD_VM_WG_IP_2',
|
||||
'coold VM WireGuard port 1 default' => 'COOLIFY_COOLD_VM_WG_PORT_1',
|
||||
'coold VM WireGuard port 2 default' => 'COOLIFY_COOLD_VM_WG_PORT_2',
|
||||
'coold VM builder capacity default' => 'COOLIFY_COOLD_VM_BUILDER_CAPACITY',
|
||||
'coold VM enabled default' => 'COOLIFY_COOLD_VM_ENABLED',
|
||||
'coold VM stop on down default' => 'COOLIFY_COOLD_VM_STOP_ON_DOWN',
|
||||
'dev follow logs default' => 'COOLIFY_DEV_FOLLOW_LOGS',
|
||||
|
|
@ -25,8 +24,7 @@
|
|||
->and(config('coold.coold_version'))->toBe('nightly')
|
||||
->and(config('coold.corrosion_version'))->toBe('v1.0.0')
|
||||
->and(config('coold.dev_ssh_user'))->toBe('coolify')
|
||||
->and(config('coold.dev_builder_capacity'))->toBe(2)
|
||||
->and(config('coold.dev_builder_enabled'))->toBeTrue();
|
||||
->and(config('coold.dev_ssh_user'))->toBe('coolify');
|
||||
});
|
||||
|
||||
it('runs the v5 dev Lima seeder with the normal development database seeder', function () {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
|
||||
$exitCode = Artisan::call('flux:dev', [
|
||||
'host_id' => 'coold-dev',
|
||||
'--caps' => 'coold,builder',
|
||||
'--caps' => 'coold',
|
||||
'--ttl' => '600',
|
||||
]);
|
||||
|
||||
|
|
@ -23,7 +23,7 @@
|
|||
|
||||
expect($claims->sub)->toBe('coold-dev')
|
||||
->and($claims->aud)->toBe('coold')
|
||||
->and($claims->caps)->toBe(['coold', 'builder'])
|
||||
->and($claims->caps)->toBe(['coold'])
|
||||
->and($claims->exp)->toBeGreaterThan(time());
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -2291,7 +2291,7 @@
|
|||
->assertJsonPath('cluster.servers.0.builderCpuQuota', '200%')
|
||||
->assertJsonPath('cluster.servers.0.ingressEnabled', false)
|
||||
->assertJsonPath('cluster.servers.0.ingressType', null)
|
||||
->assertJsonPath('cluster.servers.0.capabilities', ['coold', 'builder'])
|
||||
->assertJsonPath('cluster.servers.0.capabilities', ['coold'])
|
||||
->assertJsonPath('cluster.servers.0.wireguardListenPortOverride', 51821)
|
||||
->assertJsonPath('cluster.servers.0.wireguardEndpointOverride', 'prod-01.example.com:51821')
|
||||
->assertJsonPath('cluster.servers.0.wireguardManagementIp', null)
|
||||
|
|
@ -2310,7 +2310,7 @@
|
|||
->exists())->toBeTrue();
|
||||
|
||||
expect(V5Server::query()->where('name', 'prod-01')->first()->capabilities)
|
||||
->toBe(['coold', 'builder']);
|
||||
->toBe(['coold']);
|
||||
});
|
||||
|
||||
it('adds a v5 server with caddy ingress enabled', function () {
|
||||
|
|
@ -2720,7 +2720,7 @@
|
|||
&& cliFlagValue($command, '--corrosion-version') === 'v1.1.0'
|
||||
&& cliFlagValue($command, '--wg-listen-port-overrides') === $node.'=51831'
|
||||
&& cliFlagValue($command, '--wg-endpoint-overrides') === $node.'=prod-01.example.com:51831'
|
||||
&& in_array('--enable-builder', $command, true)
|
||||
&& ! in_array('--enable-builder', $command, true)
|
||||
&& in_array('--yes', $command, true)
|
||||
&& ! in_array('--new-nodes', $command, true);
|
||||
});
|
||||
|
|
@ -3143,7 +3143,7 @@
|
|||
expect($server->builder_enabled)->toBeTrue()
|
||||
->and($server->builder_capacity)->toBe(5)
|
||||
->and($server->builder_cpu_quota)->toBe('350%')
|
||||
->and($server->capabilities)->toBe(['coold', 'builder'])
|
||||
->and($server->capabilities)->toBe(['coold'])
|
||||
->and($server->host)->toBe('203.0.113.10')
|
||||
->and($server->ssh_user)->toBe('root')
|
||||
->and($server->ssh_port)->toBe(22)
|
||||
|
|
@ -4640,7 +4640,6 @@
|
|||
'--team-id' => $team->id,
|
||||
'--user-id' => $user->id,
|
||||
'--cluster' => 'Development-Lima',
|
||||
'--builder-capacity' => 2,
|
||||
'--server' => [
|
||||
'coold-dev|host.docker.internal|developer|61332|100.64.0.1',
|
||||
'coold-dev-2|host.docker.internal|developer|61379|100.64.0.2',
|
||||
|
|
@ -4683,7 +4682,6 @@
|
|||
'--team-id' => $team->id,
|
||||
'--user-id' => $user->id,
|
||||
'--cluster' => 'Development-Lima',
|
||||
'--builder-capacity' => 2,
|
||||
'--server' => [
|
||||
'coold-dev|host.docker.internal|developer|61332',
|
||||
],
|
||||
|
|
@ -4699,7 +4697,6 @@
|
|||
createSharedUserAndTeamTables();
|
||||
[$user, $team] = createV5UserWithTeam();
|
||||
createV5PrivateKey($team, 'Dev Lima Key');
|
||||
config()->set('coold.dev_builder_capacity', 2);
|
||||
|
||||
(new V5DevLimaSeeder)->run();
|
||||
(new V5DevLimaSeeder)->run();
|
||||
|
|
@ -4715,7 +4712,7 @@
|
|||
->and(V5Server::query()->where('name', 'coold-dev')->where('node_address', '100.64.0.1')->where('wireguard_management_ip', '100.64.0.1')->exists())->toBeTrue()
|
||||
->and(V5Server::query()->where('name', 'coold-dev-2')->where('node_address', '100.64.0.2')->where('wireguard_management_ip', '100.64.0.2')->exists())->toBeTrue()
|
||||
->and(V5Server::query()->where('status', 'installed')->count())->toBe(2)
|
||||
->and(V5Server::query()->where('builder_enabled', true)->where('builder_capacity', 2)->count())->toBe(2)
|
||||
->and(V5Server::query()->where('builder_enabled', false)->where('builder_capacity', 0)->count())->toBe(2)
|
||||
->and(V5Server::query()->where('cluster_id', $cluster->id)->count())->toBe(2);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -240,3 +240,87 @@ function withFakeFluxSocket(string $response, Closure $callback): void
|
|||
@unlink($socketPath);
|
||||
}
|
||||
}
|
||||
|
||||
it('dispatches container inventory through the containers list primitive', function () {
|
||||
if (! function_exists('pcntl_fork')) {
|
||||
$this->markTestSkipped('pcntl is required to fake a Flux Unix socket.');
|
||||
}
|
||||
|
||||
$body = json_encode([
|
||||
'request_id' => 'test-request',
|
||||
'status' => 'ok',
|
||||
'data' => [],
|
||||
], JSON_THROW_ON_ERROR);
|
||||
$requestPath = storage_path('framework/testing/flux-request-'.bin2hex(random_bytes(8)).'.txt');
|
||||
|
||||
withFakeFluxSocketCapturingRequest(
|
||||
"HTTP/1.1 200 OK\r\n".
|
||||
"Content-Type: application/json\r\n".
|
||||
'Content-Length: '.strlen($body)."\r\n".
|
||||
"\r\n".
|
||||
$body,
|
||||
$requestPath,
|
||||
fn () => (new FluxClient)->listContainers('100.64.0.10')
|
||||
);
|
||||
|
||||
$request = file_get_contents($requestPath) ?: '';
|
||||
@unlink($requestPath);
|
||||
|
||||
expect($request)->toContain('"type":"containers.list"')
|
||||
->not->toContain('list_containers');
|
||||
});
|
||||
|
||||
function withFakeFluxSocketCapturingRequest(string $response, string $requestPath, Closure $callback): void
|
||||
{
|
||||
$directory = storage_path('framework/testing');
|
||||
|
||||
if (! is_dir($directory)) {
|
||||
mkdir($directory, 0777, true);
|
||||
}
|
||||
|
||||
$socketPath = $directory.'/flux-'.bin2hex(random_bytes(8)).'.sock';
|
||||
$server = stream_socket_server("unix://{$socketPath}", $errorCode, $errorMessage);
|
||||
|
||||
expect($server)->not->toBeFalse("Could not create fake Flux socket: {$errorMessage} ({$errorCode})");
|
||||
|
||||
$pid = pcntl_fork();
|
||||
|
||||
if ($pid === 0) {
|
||||
$connection = stream_socket_accept($server, 5);
|
||||
|
||||
if ($connection !== false) {
|
||||
$request = '';
|
||||
|
||||
while (! str_contains($request, "\r\n\r\n") && ! feof($connection)) {
|
||||
$request .= fread($connection, 8192);
|
||||
}
|
||||
|
||||
if (preg_match('/Content-Length: (\d+)/i', $request, $matches) === 1) {
|
||||
$remaining = (int) $matches[1] - strlen(substr($request, strpos($request, "\r\n\r\n") + 4));
|
||||
|
||||
while ($remaining > 0 && ! feof($connection)) {
|
||||
$chunk = fread($connection, $remaining);
|
||||
$request .= $chunk;
|
||||
$remaining -= strlen($chunk);
|
||||
}
|
||||
}
|
||||
|
||||
file_put_contents($requestPath, $request);
|
||||
fwrite($connection, $response);
|
||||
fclose($connection);
|
||||
}
|
||||
|
||||
fclose($server);
|
||||
exit(0);
|
||||
}
|
||||
|
||||
fclose($server);
|
||||
|
||||
try {
|
||||
config(['flux.unix_socket_path' => $socketPath]);
|
||||
$callback();
|
||||
pcntl_waitpid($pid, $status);
|
||||
} finally {
|
||||
@unlink($socketPath);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue