From 8dc9e40163dbc173a424c36bd2b533380f320de6 Mon Sep 17 00:00:00 2001 From: rosslh Date: Sat, 12 Sep 2026 16:44:22 -0400 Subject: [PATCH] feat(security): sign CDN artifacts, verify before execution (H5) --- .forgejo/workflows/build.yml | 119 ++++++++++++++ .gitattributes | 6 +- app/Actions/Server/UpdateCoolify.php | 51 +++++- scripts/artifact-signing-pubkey.pem | 3 + scripts/artifacts.manifest | 7 + scripts/artifacts.manifest.sig | 2 + scripts/sign-artifacts.sh | 83 ++++++++++ scripts/tests/upgrade-verification.bats | 198 ++++++++++++++++++++++++ scripts/upgrade.sh | 142 ++++++++++++++--- tests/Unit/UpdateCoolifyTest.php | 42 ++++- 10 files changed, 621 insertions(+), 32 deletions(-) create mode 100644 scripts/artifact-signing-pubkey.pem create mode 100644 scripts/artifacts.manifest create mode 100644 scripts/artifacts.manifest.sig create mode 100755 scripts/sign-artifacts.sh create mode 100644 scripts/tests/upgrade-verification.bats diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 4bdb21722..e02c59a07 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -100,6 +100,82 @@ jobs: - name: Install curl run: apk add --no-cache curl + # H5 (cdn-artifact-signing): refuse to publish artifacts whose signed + # manifest is stale, unsigned, or signed with a different key than the + # one pinned in the verifying scripts. Verification uses only the + # committed PUBLIC key — the signing key never reaches CI. A failure + # here means someone edited a published artifact without running + # scripts/sign-artifacts.sh; nothing is uploaded, the fleet keeps the + # previous consistent artifact set. + - name: Run artifact verification tests + run: | + apk add --no-cache bats bash openssl coreutils + bats scripts/tests/ + + - name: Verify artifact manifest signature and hashes + run: | + apk add --no-cache openssl + + openssl base64 -d -in scripts/artifacts.manifest.sig -out /tmp/sig.bin + openssl pkeyutl -verify -pubin -inkey scripts/artifact-signing-pubkey.pem \ + -rawin -in scripts/artifacts.manifest -sigfile /tmp/sig.bin + + head -1 scripts/artifacts.manifest | grep -qx 'mapledeploy-coolify-artifacts-v1' + + # Rollback protection: the signed serial must exist and must be + # strictly greater than the serial currently served from the CDN + # (fleet VMs refuse lower serials, so publishing an equal-or-lower + # one would brick their updates or reintroduce a replayable set). + NEW_SERIAL=$(awk '$1 == "serial" { print $2; exit }' scripts/artifacts.manifest) + echo "$NEW_SERIAL" | grep -qE '^[0-9]+$' || { + echo "ERROR: manifest has no valid serial line — re-run scripts/sign-artifacts.sh" >&2 + exit 1 + } + SERVED_SERIAL=$(curl -fsSL "${{ env.CDN_BASE_URL }}/coolify/artifacts.manifest" 2>/dev/null | awk '$1 == "serial" { print $2; exit }' || true) + if echo "$SERVED_SERIAL" | grep -qE '^[0-9]+$'; then + if [ "$NEW_SERIAL" -le "$SERVED_SERIAL" ]; then + echo "ERROR: manifest serial ${NEW_SERIAL} is not greater than the served serial ${SERVED_SERIAL} — re-run scripts/sign-artifacts.sh" >&2 + exit 1 + fi + else + echo "No served manifest with a serial found (first publish, or CDN unreachable) — skipping monotonicity check." + fi + + check() { + local file="$1" + local name="$2" + local expected actual + expected=$(awk -v n="$name" '$2 == n { print $1 }' scripts/artifacts.manifest) + if [ -z "$expected" ]; then + echo "ERROR: no manifest entry for ${name}" >&2 + exit 1 + fi + actual=$(sha256sum "$file" | cut -d' ' -f1) + if [ "$expected" != "$actual" ]; then + echo "ERROR: manifest hash for ${name} is stale — run scripts/sign-artifacts.sh and commit" >&2 + exit 1 + fi + } + check scripts/upgrade.sh upgrade.sh + check scripts/upgrade-postgres.sh upgrade-postgres.sh + check docker-compose.yml docker-compose.yml + check docker-compose.prod.yml docker-compose.prod.yml + check .env.production .env.production + + # The verifying scripts pin the key inline (they run standalone on + # VM hosts). Assert the pinned copies match the committed pem so + # they cannot drift apart silently. + KEY_B64=$(sed -n '2p' scripts/artifact-signing-pubkey.pem) + grep -qF "$KEY_B64" scripts/upgrade.sh || { + echo "ERROR: upgrade.sh pins a different public key than scripts/artifact-signing-pubkey.pem" >&2 + exit 1 + } + grep -qF "$KEY_B64" app/Actions/Server/UpdateCoolify.php || { + echo "ERROR: UpdateCoolify.php pins a different public key than scripts/artifact-signing-pubkey.pem" >&2 + exit 1 + } + echo "Artifact manifest verified." + - name: Upload artifacts to Bunny CDN run: | STORAGE_URL="https://storage.bunnycdn.com/${{ env.CDN_STORAGE_ZONE }}/coolify" @@ -120,6 +196,12 @@ jobs: upload docker-compose.yml docker-compose.yml upload docker-compose.prod.yml docker-compose.prod.yml upload .env.production .env.production + # H5: manifest and signature go up last, after every artifact they + # attest to. A VM fetching mid-publish sees either a consistent set + # or a verification failure that aborts its upgrade before any file + # is moved into place; the next attempt succeeds. + upload scripts/artifacts.manifest artifacts.manifest + upload scripts/artifacts.manifest.sig artifacts.manifest.sig echo "All artifacts uploaded." @@ -129,3 +211,40 @@ jobs: -H "AccessKey: ${{ secrets.BUNNY_API_KEY }}" \ -H "Content-Type: application/json" echo "CDN cache purged." + + # H5: verify what the CDN actually SERVES after the purge, not just + # what was uploaded to the storage origin. A failed purge or a stale + # edge would otherwise be discovered by the next customer provision + # instead of by this run. Retries cover purge propagation delay. + - name: Verify served artifact set + run: | + set -e + verify_served() { + local base="${{ env.CDN_BASE_URL }}/coolify" + local dir; dir=$(mktemp -d) + for f in artifacts.manifest artifacts.manifest.sig upgrade.sh upgrade-postgres.sh docker-compose.yml docker-compose.prod.yml .env.production; do + curl -fsSL -H 'Cache-Control: no-cache' "${base}/${f}" -o "${dir}/${f}" || return 1 + done + openssl base64 -d -in "${dir}/artifacts.manifest.sig" -out "${dir}/sig.bin" || return 1 + openssl pkeyutl -verify -pubin -inkey scripts/artifact-signing-pubkey.pem \ + -rawin -in "${dir}/artifacts.manifest" -sigfile "${dir}/sig.bin" >/dev/null || return 1 + cmp -s "${dir}/artifacts.manifest" scripts/artifacts.manifest || return 1 + local f expected actual + for f in upgrade.sh upgrade-postgres.sh docker-compose.yml docker-compose.prod.yml .env.production; do + expected=$(awk -v n="$f" '$2 == n { print $1 }' "${dir}/artifacts.manifest") + actual=$(sha256sum "${dir}/${f}" | cut -d' ' -f1) + [ -n "$expected" ] && [ "$expected" = "$actual" ] || return 1 + done + return 0 + } + for attempt in 1 2 3 4 5; do + if verify_served; then + echo "Served artifact set verified (attempt ${attempt})." + exit 0 + fi + echo "Served set not yet consistent (attempt ${attempt}/5), waiting 30s..." + sleep 30 + done + echo "ERROR: CDN is not serving the artifact set this run published." >&2 + echo " Do NOT deploy anything that depends on this publish; investigate the Bunny purge/edge state." >&2 + exit 1 diff --git a/.gitattributes b/.gitattributes index c48a5898b..d7926a912 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,4 +8,8 @@ /.github export-ignore CHANGELOG.md export-ignore -.styleci.yml export-ignore \ No newline at end of file +.styleci.yml export-ignore +# MapleDeploy (H5): the manifest and signature are cryptographically signed +# bytes — line-ending normalization must never touch them. +scripts/artifacts.manifest -text +scripts/artifacts.manifest.sig -text diff --git a/app/Actions/Server/UpdateCoolify.php b/app/Actions/Server/UpdateCoolify.php index 1614f1039..6e4c46648 100644 --- a/app/Actions/Server/UpdateCoolify.php +++ b/app/Actions/Server/UpdateCoolify.php @@ -121,12 +121,49 @@ private function update() // MapleDeploy branding: always use the fork registry default, ignoring per-instance overrides $registryUrl = config('constants.coolify.registry_url'); - remote_process([ - "curl -fsSL {$upgradeScriptUrl} -o /data/coolify/source/upgrade.sh", - 'bash /data/coolify/source/upgrade.sh '. - escapeshellarg($this->latestVersion).' '. - escapeshellarg($latestHelperImageVersion).' '. - escapeshellarg($registryUrl), - ], $this->server); + // MapleDeploy: artifact verification (H5, cdn-artifact-signing). + // upgrade.sh is fetched into a temp dir and executed only after its + // sha256 matches the Ed25519-signed manifest published beside it. + // upgrade.sh then verifies its own downloads the same way. The + // pinned public key must match scripts/artifact-signing-pubkey.pem + // (CI asserts this). A verification failure aborts before anything + // is written to /data/coolify/source. + // NOTE: this path assumes Server 0 is root (MapleDeploy provisions it + // that way). remote_process's non-root sudo parser rewrites per array + // ELEMENT, which would mangle this multi-line script; if the fork ever + // supports non-root instance servers, this must be revisited. + $cdnBase = dirname($upgradeScriptUrl); + $manifestUrl = escapeshellarg($cdnBase.'/artifacts.manifest'); + $manifestSigUrl = escapeshellarg($cdnBase.'/artifacts.manifest.sig'); + $upgradeUrl = escapeshellarg($upgradeScriptUrl); + $upgradeArgs = escapeshellarg($this->latestVersion).' '. + escapeshellarg($latestHelperImageVersion).' '. + escapeshellarg($registryUrl); + + // upgrade.sh is executed from the root-owned staging dir, not from + // /data/coolify/source (bind-mounted into the container, so uid 9999 + // could swap the file between verify and execute); the copy installed + // there is for the emergency SSH fallback and the next run's serial + // state only. + $script = << "\$STAGING/pubkey.pem" + openssl base64 -d -in "\$STAGING/artifacts.manifest.sig" -out "\$STAGING/sig.bin" + openssl pkeyutl -verify -pubin -inkey "\$STAGING/pubkey.pem" -rawin -in "\$STAGING/artifacts.manifest" -sigfile "\$STAGING/sig.bin" >/dev/null + head -1 "\$STAGING/artifacts.manifest" | grep -q '^mapledeploy-coolify-artifacts-v1\$' + EXPECTED=\$(awk '\$2 == "upgrade.sh" { print \$1 }' "\$STAGING/artifacts.manifest") + test -n "\$EXPECTED" + ACTUAL=\$(sha256sum "\$STAGING/upgrade.sh" | cut -d' ' -f1) + test "\$EXPECTED" = "\$ACTUAL" + cp "\$STAGING/upgrade.sh" /data/coolify/source/upgrade.sh + bash "\$STAGING/upgrade.sh" {$upgradeArgs} + BASH; + + remote_process([$script], $this->server); } } diff --git a/scripts/artifact-signing-pubkey.pem b/scripts/artifact-signing-pubkey.pem new file mode 100644 index 000000000..d8a0f68bb --- /dev/null +++ b/scripts/artifact-signing-pubkey.pem @@ -0,0 +1,3 @@ +-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAS2KmuRRjkdub0vjbO7wfmIdo60xYSvxx2hJ7oRYU+/k= +-----END PUBLIC KEY----- diff --git a/scripts/artifacts.manifest b/scripts/artifacts.manifest new file mode 100644 index 000000000..596021930 --- /dev/null +++ b/scripts/artifacts.manifest @@ -0,0 +1,7 @@ +mapledeploy-coolify-artifacts-v1 +serial 1789245703 +bad9a54828cf1dccff693bee2a77182f5bc4bd702505a1fc7672c6753d01b167 .env.production +632742d114a2fd4562ea23f214b2f5326f6aff8c15d8b8c653bf9886713151bd docker-compose.prod.yml +0223699dfef8a421116872b050830b21cfedfc58911576a0129c2082aeaadc59 docker-compose.yml +aa65c277fc13ab5b27c349fbfef12defc59c084b244e4b258329e1f6c570ae8d upgrade-postgres.sh +0a859777c812272ffcf7bb9bd87d21d25be9f108b1b2c289bc95d3d412ab69f5 upgrade.sh diff --git a/scripts/artifacts.manifest.sig b/scripts/artifacts.manifest.sig new file mode 100644 index 000000000..48ed746cf --- /dev/null +++ b/scripts/artifacts.manifest.sig @@ -0,0 +1,2 @@ +h/PqvVa1mzB2xDmyCKW/Gc3Ff4z7XeK9KJ5g+RTqcrAF8v2VoE2DHe5oREpM6MVv +LDxOIDXk0d91AOE0yVdLBg== diff --git a/scripts/sign-artifacts.sh b/scripts/sign-artifacts.sh new file mode 100755 index 000000000..bf31e52c8 --- /dev/null +++ b/scripts/sign-artifacts.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# MapleDeploy: artifact manifest signing (H5, cdn-artifact-signing). +# +# Regenerates scripts/artifacts.manifest (sha256 of every CDN-published +# artifact that customer VMs execute or load) and signs it with the +# Ed25519 key stored in 1Password. Run this after ANY change to the +# artifacts listed below, then commit the manifest and signature — +# CI verifies them against the working tree and refuses to publish +# artifacts whose manifest is stale or unsigned. +# +# The private key lives ONLY in 1Password ("Coolify Artifact Signing +# Key" in the Lawrence Digital vault). It is read via `op` for the +# few milliseconds signing takes and never touches CI. +# +# Requires: op (signed in), OpenSSL 3.x (`brew install openssl` on +# macOS — LibreSSL lacks Ed25519 pkeyutl support). + +set -euo pipefail + +cd "$(dirname "$0")/.." + +# The artifact set published to updates.mapledeploy.ca/coolify/ by +# .forgejo/workflows/build.yml. versions.json is deliberately absent: +# it is generated in CI (timestamped version), is never executed, and +# tampering with it is contained by UpdateCoolify's downgrade guard +# plus the authenticated registry pull — see SECURITY_BASELINE.md. +ARTIFACTS=( + ".env.production" + "docker-compose.prod.yml" + "docker-compose.yml" + "scripts/upgrade-postgres.sh" + "scripts/upgrade.sh" +) + +MANIFEST="scripts/artifacts.manifest" +SIGNATURE="scripts/artifacts.manifest.sig" +KEY_REF="op://Lawrence Digital/Coolify Artifact Signing Key/private key" + +OPENSSL_BIN="${OPENSSL_BIN:-openssl}" +if ! "$OPENSSL_BIN" version | grep -q "^OpenSSL 3"; then + if [ -x /opt/homebrew/bin/openssl ]; then + OPENSSL_BIN=/opt/homebrew/bin/openssl + else + echo "ERROR: OpenSSL 3.x required (found: $("$OPENSSL_BIN" version)). brew install openssl" >&2 + exit 1 + fi +fi + +# Manifest: a fixed header line for domain separation, a monotonic +# "serial " line (rollback protection: verifiers refuse a +# manifest whose serial is lower than the one they installed last, and CI +# refuses to publish a serial not greater than the currently served one), +# then one "sha256 cdn-filename" line per artifact. CDN filenames are +# basenames (CI uploads scripts/upgrade.sh as upgrade.sh). +{ + echo "mapledeploy-coolify-artifacts-v1" + printf 'serial %s\n' "$(date +%s)" + for artifact in "${ARTIFACTS[@]}"; do + hash=$(sha256sum "$artifact" | cut -d' ' -f1) + printf '%s %s\n' "$hash" "$(basename "$artifact")" + done +} > "$MANIFEST" + +# Sign with the key from 1Password. The key exists on disk only inside +# a 700 temp dir for the duration of the signing call. +KEYDIR=$(mktemp -d) +trap 'rm -rf "$KEYDIR"' EXIT +op read "$KEY_REF" > "${KEYDIR}/sec.pem" +"$OPENSSL_BIN" pkeyutl -sign -inkey "${KEYDIR}/sec.pem" -rawin \ + -in "$MANIFEST" -out "${KEYDIR}/sig.bin" +# Base64 with default line wrapping: decoding with plain +# `openssl base64 -d` then works on every verifier (the -A variant +# chokes on trailing newlines in some OpenSSL builds). +"$OPENSSL_BIN" base64 -in "${KEYDIR}/sig.bin" -out "$SIGNATURE" + +# Self-check against the committed public key before declaring success. +"$OPENSSL_BIN" base64 -d -in "$SIGNATURE" -out "${KEYDIR}/sig.check" +"$OPENSSL_BIN" pkeyutl -verify -pubin -inkey scripts/artifact-signing-pubkey.pem \ + -rawin -in "$MANIFEST" -sigfile "${KEYDIR}/sig.check" + +echo "Signed manifest:" +cat "$MANIFEST" +echo "Commit ${MANIFEST} and ${SIGNATURE}." diff --git a/scripts/tests/upgrade-verification.bats b/scripts/tests/upgrade-verification.bats new file mode 100644 index 000000000..9d455cee1 --- /dev/null +++ b/scripts/tests/upgrade-verification.bats @@ -0,0 +1,198 @@ +#!/usr/bin/env bats +# MapleDeploy: artifact verification tests for scripts/upgrade.sh (H5, +# cdn-artifact-signing). Prove that the upgrade refuses tampered, missing, +# or wrongly-signed CDN artifact sets BEFORE overwriting anything in the +# live source tree, and that a correctly signed set is installed. +# +# Uses a real Ed25519 keypair (generated per test) — the crypto is not +# mocked. Docker is mocked to produce no compose images, so a run that +# gets PAST verification exits at the image-extraction step with +# "Failed to parse docker-compose configuration"; the tests use that +# distinct later error as proof verification succeeded. +# +# Run locally with bats-core; CI runs this via the build workflow. + +SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" + +setup() { + MOCK_BIN="${BATS_TEST_TMPDIR}/bin" + mkdir -p "$MOCK_BIN" + + # Instant sleep, inert docker (no compose images -> extraction fails + # right after verification), inert chown/chmod. NOTE: once the chmod + # mock is on PATH, a bare `chmod` in this setup would be a no-op and + # silently leave later mocks non-executable — always /bin/chmod here. + for cmd in sleep chown chmod docker; do + printf '#!/bin/bash\nexit 0\n' > "${MOCK_BIN}/${cmd}" + /bin/chmod +x "${MOCK_BIN}/${cmd}" + done + export PATH="${MOCK_BIN}:${PATH}" + + # Live source tree with pre-existing content that must survive any + # failed verification. + export COOLIFY_SOURCE_DIR="${BATS_TEST_TMPDIR}/source" + mkdir -p "$COOLIFY_SOURCE_DIR" + echo "REGISTRY_URL=forgejo.mapledeploy.ca" > "${COOLIFY_SOURCE_DIR}/.env" + echo "old-compose" > "${COOLIFY_SOURCE_DIR}/docker-compose.yml" + echo "old-prod-compose" > "${COOLIFY_SOURCE_DIR}/docker-compose.prod.yml" + echo "OLD_ENV=1" > "${COOLIFY_SOURCE_DIR}/.env.production" + echo "old-postgres" > "${COOLIFY_SOURCE_DIR}/upgrade-postgres.sh" + + # Signing keypair; upgrade.sh reads the public key through the test + # override MAPLEDEPLOY_ARTIFACT_PUBKEY_PEM_FILE. + KEY_DIR="${BATS_TEST_TMPDIR}/keys" + mkdir -p "$KEY_DIR" + openssl genpkey -algorithm ED25519 -out "${KEY_DIR}/sec.pem" 2>/dev/null + openssl pkey -in "${KEY_DIR}/sec.pem" -pubout -out "${KEY_DIR}/pub.pem" 2>/dev/null + export MAPLEDEPLOY_ARTIFACT_PUBKEY_PEM_FILE="${KEY_DIR}/pub.pem" + + # Mock CDN content: the artifact set upgrade.sh downloads. + CDN_DIR="${BATS_TEST_TMPDIR}/cdn" + mkdir -p "$CDN_DIR" + echo "new-compose" > "${CDN_DIR}/docker-compose.yml" + echo "new-prod-compose" > "${CDN_DIR}/docker-compose.prod.yml" + echo "NEW_ENV=1" > "${CDN_DIR}/.env.production" + echo "new-postgres" > "${CDN_DIR}/upgrade-postgres.sh" + sign_manifest + + cat > "${MOCK_BIN}/curl" << MOCK +#!/bin/bash +CDN_DIR="${CDN_DIR}" +url=""; outpath=""; prev="" +for arg in "\$@"; do + if [[ "\$prev" == "-o" ]]; then outpath="\$arg"; elif [[ "\$arg" == http* ]]; then url="\$arg"; fi + prev="\$arg" +done +if [[ "\$url" == *"updates.mapledeploy.ca"* ]] && [[ -n "\$outpath" ]]; then + name=\$(basename "\$url") + if [[ -f "\${CDN_DIR}/\${name}" ]]; then + cp "\${CDN_DIR}/\${name}" "\$outpath" + exit 0 + fi + exit 22 +fi +exit 0 +MOCK + /bin/chmod +x "${MOCK_BIN}/curl" +} + +# Build and sign the manifest for the mock CDN's current contents, the way +# scripts/sign-artifacts.sh signs the real set. upgrade.sh itself is not in +# the mock CDN (this suite tests upgrade.sh's own downloads); its manifest +# entry is irrelevant here and omitted. +sign_manifest() { + { + echo "mapledeploy-coolify-artifacts-v1" + printf 'serial %s\n' "${ARTIFACT_TEST_SERIAL:-$(date +%s)}" + local f hash + for f in .env.production docker-compose.prod.yml docker-compose.yml upgrade-postgres.sh; do + hash=$(sha256sum "${CDN_DIR}/${f}" | cut -d' ' -f1) + printf '%s %s\n' "$hash" "$f" + done + } > "${CDN_DIR}/artifacts.manifest" + openssl pkeyutl -sign -inkey "${KEY_DIR}/sec.pem" -rawin \ + -in "${CDN_DIR}/artifacts.manifest" -out "${BATS_TEST_TMPDIR}/sig.bin" + openssl base64 -in "${BATS_TEST_TMPDIR}/sig.bin" -out "${CDN_DIR}/artifacts.manifest.sig" +} + +assert_source_tree_untouched() { + [ "$(cat "${COOLIFY_SOURCE_DIR}/docker-compose.yml")" = "old-compose" ] + [ "$(cat "${COOLIFY_SOURCE_DIR}/docker-compose.prod.yml")" = "old-prod-compose" ] + [ "$(cat "${COOLIFY_SOURCE_DIR}/.env.production")" = "OLD_ENV=1" ] + [ "$(cat "${COOLIFY_SOURCE_DIR}/upgrade-postgres.sh")" = "old-postgres" ] + [ "$(cat "${COOLIFY_SOURCE_DIR}/.env")" = "REGISTRY_URL=forgejo.mapledeploy.ca" ] +} + +@test "verified artifact set is installed into the source tree" { + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + # Verification passed; the run then stops at the mocked-away compose + # image extraction, which is the expected boundary for this suite. + [ "$status" -ne 0 ] + [[ "$output" == *"Failed to parse docker-compose configuration"* ]] + [[ "$output" != *"Artifact verification failed"* ]] + [ "$(cat "${COOLIFY_SOURCE_DIR}/docker-compose.yml")" = "new-compose" ] + [ "$(cat "${COOLIFY_SOURCE_DIR}/docker-compose.prod.yml")" = "new-prod-compose" ] + [ "$(cat "${COOLIFY_SOURCE_DIR}/.env.production")" = "NEW_ENV=1" ] + [ "$(cat "${COOLIFY_SOURCE_DIR}/upgrade-postgres.sh")" = "new-postgres" ] +} + +@test "tampered artifact is refused and the live tree is untouched" { + echo "EVIL=1" >> "${CDN_DIR}/.env.production" + + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + [ "$status" -ne 0 ] + [[ "$output" == *"Artifact verification failed: sha256 mismatch for .env.production"* ]] + grep -q "sha256 mismatch for .env.production" "${COOLIFY_SOURCE_DIR}/.upgrade-status" + assert_source_tree_untouched +} + +@test "missing manifest on the CDN aborts before anything is written" { + rm "${CDN_DIR}/artifacts.manifest" + + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + [ "$status" -ne 0 ] + [[ "$output" == *"download of artifacts.manifest failed"* ]] + assert_source_tree_untouched +} + +@test "manifest signed with a different key is refused" { + openssl genpkey -algorithm ED25519 -out "${BATS_TEST_TMPDIR}/rogue.pem" 2>/dev/null + openssl pkeyutl -sign -inkey "${BATS_TEST_TMPDIR}/rogue.pem" -rawin \ + -in "${CDN_DIR}/artifacts.manifest" -out "${BATS_TEST_TMPDIR}/rogue.sig.bin" + openssl base64 -in "${BATS_TEST_TMPDIR}/rogue.sig.bin" -out "${CDN_DIR}/artifacts.manifest.sig" + + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + [ "$status" -ne 0 ] + [[ "$output" == *"manifest signature verification failed"* ]] + assert_source_tree_untouched +} + +@test "manifest missing an entry for a fetched artifact is refused" { + grep -v "docker-compose.prod.yml" "${CDN_DIR}/artifacts.manifest" > "${CDN_DIR}/manifest.tmp" + mv "${CDN_DIR}/manifest.tmp" "${CDN_DIR}/artifacts.manifest" + openssl pkeyutl -sign -inkey "${KEY_DIR}/sec.pem" -rawin \ + -in "${CDN_DIR}/artifacts.manifest" -out "${BATS_TEST_TMPDIR}/resign.bin" + openssl base64 -in "${BATS_TEST_TMPDIR}/resign.bin" -out "${CDN_DIR}/artifacts.manifest.sig" + + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + [ "$status" -ne 0 ] + [[ "$output" == *"manifest has no entry for docker-compose.prod.yml"* ]] + assert_source_tree_untouched +} + +@test "signed rollback (lower serial than installed) is refused" { + echo "99999999999" > "${COOLIFY_SOURCE_DIR}/.artifact-serial" + + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + [ "$status" -ne 0 ] + [[ "$output" == *"rollback refused"* ]] + assert_source_tree_untouched + [ "$(cat "${COOLIFY_SOURCE_DIR}/.artifact-serial")" = "99999999999" ] +} + +@test "manifest without a serial line is refused" { + grep -v "^serial " "${CDN_DIR}/artifacts.manifest" > "${CDN_DIR}/manifest.tmp" + mv "${CDN_DIR}/manifest.tmp" "${CDN_DIR}/artifacts.manifest" + openssl pkeyutl -sign -inkey "${KEY_DIR}/sec.pem" -rawin \ + -in "${CDN_DIR}/artifacts.manifest" -out "${BATS_TEST_TMPDIR}/nos.bin" + openssl base64 -in "${BATS_TEST_TMPDIR}/nos.bin" -out "${CDN_DIR}/artifacts.manifest.sig" + + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + [ "$status" -ne 0 ] + [[ "$output" == *"manifest serial missing or malformed"* ]] + assert_source_tree_untouched +} + +@test "successful verification records the manifest serial" { + run bash "${SCRIPT_DIR}/upgrade.sh" 4.9.9 1.0.0 forgejo.mapledeploy.ca + [[ "$output" != *"Artifact verification failed"* ]] + expected_serial=$(awk '$1 == "serial" { print $2; exit }' "${CDN_DIR}/artifacts.manifest") + [ "$(cat "${COOLIFY_SOURCE_DIR}/.artifact-serial")" = "$expected_serial" ] +} + +@test "unsafe argument values are refused before anything runs" { + run bash "${SCRIPT_DIR}/upgrade.sh" "4.9.9'; touch /tmp/pwned;'" 1.0.0 forgejo.mapledeploy.ca + [ "$status" -ne 0 ] + [[ "$output" == *"refusing unsafe argument value"* ]] + assert_source_tree_untouched +} diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index fcb403c29..5eec06c97 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -2,9 +2,33 @@ ## Do not modify this file. You will lose the ability to autoupdate! CDN="https://updates.mapledeploy.ca/coolify" + +# MapleDeploy: artifact verification (H5, cdn-artifact-signing). +# Every file fetched from the CDN is checked against a signed manifest +# before anything is moved into place or executed. The Ed25519 public +# key is pinned here; the private key lives only in 1Password and the +# manifest is signed at release time by scripts/sign-artifacts.sh. +# Keep in sync with scripts/artifact-signing-pubkey.pem (CI asserts this). +ARTIFACT_PUBKEY_PEM='-----BEGIN PUBLIC KEY----- +MCowBQYDK2VwAyEAS2KmuRRjkdub0vjbO7wfmIdo60xYSvxx2hJ7oRYU+/k= +-----END PUBLIC KEY-----' +# Test override only. No reachable invocation path lets an unprivileged +# party set this: UpdateCoolify runs the script through a root SSH +# heredoc (sshd accepts no client env), and any sudo path applies +# env_reset. Whoever CAN set it already runs arbitrary root commands. +if [ -n "${MAPLEDEPLOY_ARTIFACT_PUBKEY_PEM_FILE:-}" ]; then + ARTIFACT_PUBKEY_PEM=$(cat "$MAPLEDEPLOY_ARTIFACT_PUBKEY_PEM_FILE") +fi + +# COOLIFY_SOURCE_DIR is a test override (same trust rationale as above); +# production always uses the default. The detached restart section below +# keeps literal paths (it never runs under tests and must match the +# production layout exactly). +SOURCE_DIR="${COOLIFY_SOURCE_DIR:-/data/coolify/source}" + LATEST_IMAGE=${1:-latest} LATEST_HELPER_VERSION=${2:-latest} -ENV_FILE="/data/coolify/source/.env" +ENV_FILE="${SOURCE_DIR}/.env" if [ -n "${3+x}" ]; then REGISTRY_URL="$3" elif [ -f "$ENV_FILE" ] && grep -q "^REGISTRY_URL=" "$ENV_FILE"; then @@ -13,10 +37,21 @@ else REGISTRY_URL="docker.io" fi SKIP_BACKUP=${4:-false} -STATUS_FILE="/data/coolify/source/.upgrade-status" + +# MapleDeploy: validate the values this script splices into shell text +# (the detached-restart heredoc single-quotes them; a quote inside a +# value would break out and run as root). Docker tag/registry grammar +# has no quote characters, so legitimate values always pass. +for _arg_check in "$LATEST_IMAGE" "$LATEST_HELPER_VERSION" "$REGISTRY_URL"; do + if ! [[ "$_arg_check" =~ ^[A-Za-z0-9._:/-]+$ ]]; then + echo "ERROR: refusing unsafe argument value: ${_arg_check}" + exit 1 + fi +done +STATUS_FILE="${SOURCE_DIR}/.upgrade-status" DATE=$(date +%Y-%m-%d-%H-%M-%S) -LOGFILE="/data/coolify/source/upgrade-${DATE}.log" +LOGFILE="${SOURCE_DIR}/upgrade-${DATE}.log" # Helper function to log with timestamp log() { @@ -53,34 +88,101 @@ echo "Helper Version: ${LATEST_HELPER_VERSION}" >>"$LOGFILE" echo "Registry URL: ${REGISTRY_URL}" >>"$LOGFILE" echo "============================================================" >>"$LOGFILE" -log_section "Step 1/6: Downloading configuration files" +log_section "Step 1/6: Downloading and verifying configuration files" write_status "1" "Downloading configuration files" echo "1/6 Downloading latest configuration files..." -log "Downloading docker-compose.yml from ${CDN}/docker-compose.yml" -curl -fsSL -L $CDN/docker-compose.yml -o /data/coolify/source/docker-compose.yml -log "Downloading docker-compose.prod.yml from ${CDN}/docker-compose.prod.yml" -curl -fsSL -L $CDN/docker-compose.prod.yml -o /data/coolify/source/docker-compose.prod.yml -log "Downloading .env.production from ${CDN}/.env.production" -curl -fsSL -L $CDN/.env.production -o /data/coolify/source/.env.production -log "Downloading upgrade-postgres.sh from ${CDN}/upgrade-postgres.sh" -curl -fsSL -L $CDN/upgrade-postgres.sh -o /data/coolify/source/upgrade-postgres.sh -chmod +x /data/coolify/source/upgrade-postgres.sh -log "Configuration files downloaded successfully" + +# MapleDeploy: artifact verification (H5). Download the signed manifest +# and every artifact into a staging dir, verify the manifest signature, +# its monotonic serial, and each file's sha256, and only then move files +# into place. A failure leaves the live source tree untouched and aborts +# the upgrade loudly. The staging dir lives INSIDE the source dir so the +# installs below are same-filesystem renames, not cross-device copies. +STAGING=$(mktemp -d "${SOURCE_DIR}/.artifact-staging.XXXXXX") \ + || { write_status "error" "mktemp failed"; exit 1; } +trap 'rm -rf "$STAGING"' EXIT + +verify_fail() { + log "ERROR: $1" + write_status "error" "Artifact verification failed: $1" + echo " ERROR: Artifact verification failed: $1. Aborting upgrade." + exit 1 +} + +fetch() { + local name="$1" + log "Downloading ${name} from ${CDN}/${name}" + curl -fsSL "${CDN}/${name}" -o "${STAGING}/${name}" \ + || verify_fail "download of ${name} failed" +} + +VERIFIED_FILES="docker-compose.yml docker-compose.prod.yml .env.production upgrade-postgres.sh" + +fetch "artifacts.manifest" +fetch "artifacts.manifest.sig" +for f in $VERIFIED_FILES; do + fetch "$f" +done + +printf '%s\n' "$ARTIFACT_PUBKEY_PEM" > "${STAGING}/pubkey.pem" +openssl base64 -d -in "${STAGING}/artifacts.manifest.sig" -out "${STAGING}/sig.bin" \ + || verify_fail "signature is not valid base64" +if ! openssl pkeyutl -verify -pubin -inkey "${STAGING}/pubkey.pem" -rawin \ + -in "${STAGING}/artifacts.manifest" -sigfile "${STAGING}/sig.bin" >/dev/null 2>&1; then + verify_fail "manifest signature verification failed" +fi +head -1 "${STAGING}/artifacts.manifest" | grep -q '^mapledeploy-coolify-artifacts-v1$' \ + || verify_fail "manifest header missing or unrecognized" +log "Manifest signature verified" + +# Rollback protection: the signed serial must not be lower than the one +# this VM last installed. Without this, a CDN attacker could replay any +# OLDER validly-signed artifact set and silently revert fixes. +SERIAL_FILE="${SOURCE_DIR}/.artifact-serial" +MANIFEST_SERIAL=$(awk '$1 == "serial" { print $2; exit }' "${STAGING}/artifacts.manifest") +[[ "$MANIFEST_SERIAL" =~ ^[0-9]+$ ]] || verify_fail "manifest serial missing or malformed" +if [ -f "$SERIAL_FILE" ]; then + INSTALLED_SERIAL=$(cat "$SERIAL_FILE" 2>/dev/null || echo "") + if [[ "$INSTALLED_SERIAL" =~ ^[0-9]+$ ]] && [ "$MANIFEST_SERIAL" -lt "$INSTALLED_SERIAL" ]; then + verify_fail "manifest serial ${MANIFEST_SERIAL} is older than installed ${INSTALLED_SERIAL} (rollback refused)" + fi +fi + +for f in $VERIFIED_FILES; do + expected=$(awk -v name="$f" '$2 == name { print $1 }' "${STAGING}/artifacts.manifest") + [ -n "$expected" ] || verify_fail "manifest has no entry for ${f}" + actual=$(sha256sum "${STAGING}/${f}" | cut -d' ' -f1) + [ "$expected" = "$actual" ] || verify_fail "sha256 mismatch for ${f}" + log "Verified ${f} (${actual})" +done + +# Same-filesystem renames (staging is inside SOURCE_DIR), each checked: +# a failed install mid-loop must abort loudly, never continue into an +# upgrade running a mixed old/new artifact set. +for f in $VERIFIED_FILES; do + mv "${STAGING}/${f}" "${SOURCE_DIR}/${f}" \ + || verify_fail "could not install ${f}" +done +chmod +x "${SOURCE_DIR}/upgrade-postgres.sh" \ + || verify_fail "could not mark upgrade-postgres.sh executable" +echo "$MANIFEST_SERIAL" > "$SERIAL_FILE" \ + || log "WARNING: could not record artifact serial ${MANIFEST_SERIAL}" +log "Configuration files downloaded and verified successfully (serial ${MANIFEST_SERIAL})" echo " Done." # Extract all images from docker-compose configuration log "Extracting all images from docker-compose configuration..." -COMPOSE_FILES="-f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml" +COMPOSE_FILES="-f ${SOURCE_DIR}/docker-compose.yml -f ${SOURCE_DIR}/docker-compose.prod.yml" # Check if custom compose file exists -if [ -f /data/coolify/source/docker-compose.custom.yml ]; then - COMPOSE_FILES="$COMPOSE_FILES -f /data/coolify/source/docker-compose.custom.yml" +if [ -f "${SOURCE_DIR}/docker-compose.custom.yml" ]; then + COMPOSE_FILES="$COMPOSE_FILES -f ${SOURCE_DIR}/docker-compose.custom.yml" log "Including custom docker-compose.yml in image extraction" fi # Check if PostgreSQL upgrade override exists -if [ -f /data/coolify/source/docker-compose.postgres-upgrade.yml ]; then - COMPOSE_FILES="$COMPOSE_FILES -f /data/coolify/source/docker-compose.postgres-upgrade.yml" +if [ -f "${SOURCE_DIR}/docker-compose.postgres-upgrade.yml" ]; then + COMPOSE_FILES="$COMPOSE_FILES -f ${SOURCE_DIR}/docker-compose.postgres-upgrade.yml" log "Including PostgreSQL upgrade compose override in image extraction" fi @@ -115,7 +217,7 @@ write_status "2" "Updating environment configuration" echo "" echo "2/6 Updating environment configuration..." log "Merging .env.production values into .env" -awk -F '=' '!seen[$1]++' "$ENV_FILE" /data/coolify/source/.env.production > "$ENV_FILE.tmp" && mv "$ENV_FILE.tmp" "$ENV_FILE" +awk -F '=' '!seen[$1]++' "$ENV_FILE" "${SOURCE_DIR}/.env.production" > "$ENV_FILE.tmp" && mv "$ENV_FILE.tmp" "$ENV_FILE" log "Environment file merged successfully" update_env_var() { diff --git a/tests/Unit/UpdateCoolifyTest.php b/tests/Unit/UpdateCoolifyTest.php index 833fe3603..a46a914ed 100644 --- a/tests/Unit/UpdateCoolifyTest.php +++ b/tests/Unit/UpdateCoolifyTest.php @@ -43,6 +43,36 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi Mockery::close(); }); +// MapleDeploy: artifact verification (H5). Mirrors the script built in +// UpdateCoolify::update() — upgrade.sh must be hash-verified against the +// Ed25519-signed manifest before it is moved into place and executed. +function updateCoolifyExpectedVerifiedCommand(string $upgradeScriptUrl, string $args): string +{ + $cdnBase = dirname($upgradeScriptUrl); + $manifestUrl = escapeshellarg($cdnBase.'/artifacts.manifest'); + $manifestSigUrl = escapeshellarg($cdnBase.'/artifacts.manifest.sig'); + $upgradeUrl = escapeshellarg($upgradeScriptUrl); + + return implode("\n", [ + 'set -euo pipefail', + 'STAGING=$(mktemp -d)', + "trap 'rm -rf \"\$STAGING\"' EXIT", + "curl -fsSL {$manifestUrl} -o \"\$STAGING/artifacts.manifest\"", + "curl -fsSL {$manifestSigUrl} -o \"\$STAGING/artifacts.manifest.sig\"", + "curl -fsSL {$upgradeUrl} -o \"\$STAGING/upgrade.sh\"", + "printf '%s\\n' '-----BEGIN PUBLIC KEY-----' 'MCowBQYDK2VwAyEAS2KmuRRjkdub0vjbO7wfmIdo60xYSvxx2hJ7oRYU+/k=' '-----END PUBLIC KEY-----' > \"\$STAGING/pubkey.pem\"", + 'openssl base64 -d -in "$STAGING/artifacts.manifest.sig" -out "$STAGING/sig.bin"', + 'openssl pkeyutl -verify -pubin -inkey "$STAGING/pubkey.pem" -rawin -in "$STAGING/artifacts.manifest" -sigfile "$STAGING/sig.bin" >/dev/null', + 'head -1 "$STAGING/artifacts.manifest" | grep -q \'^mapledeploy-coolify-artifacts-v1$\'', + 'EXPECTED=$(awk \'$2 == "upgrade.sh" { print $1 }\' "$STAGING/artifacts.manifest")', + 'test -n "$EXPECTED"', + 'ACTUAL=$(sha256sum "$STAGING/upgrade.sh" | cut -d\' \' -f1)', + 'test "$EXPECTED" = "$ACTUAL"', + 'cp "$STAGING/upgrade.sh" /data/coolify/source/upgrade.sh', + "bash \"\$STAGING/upgrade.sh\" {$args}", + ]); +} + it('has UpdateCoolify action class', function () { expect(class_exists(UpdateCoolify::class))->toBeTrue(); }); @@ -122,8 +152,10 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi (new UpdateCoolify)->handle(); expect(Activity::query()->latest('id')->first()?->getExtraProperty('command'))->toBe( - "curl -fsSL https://cdn.example.com/upgrade.sh -o /data/coolify/source/upgrade.sh\n". - "bash /data/coolify/source/upgrade.sh '4.0.10' '1.0.14' 'ghcr.io'" + updateCoolifyExpectedVerifiedCommand( + 'https://cdn.example.com/upgrade.sh', + "'4.0.10' '1.0.14' 'ghcr.io'" + ) ); }); @@ -151,8 +183,10 @@ function updateCoolifyTestCreateRootServerAndSettings(array $settings = []): voi (new UpdateCoolify)->handle(); expect(Activity::query()->latest('id')->first()?->getExtraProperty('command'))->toBe( - "curl -fsSL https://cdn.example.com/upgrade.sh -o /data/coolify/source/upgrade.sh\n". - "bash /data/coolify/source/upgrade.sh '4.0.10' '1.0.14' 'docker.io'" + updateCoolifyExpectedVerifiedCommand( + 'https://cdn.example.com/upgrade.sh', + "'4.0.10' '1.0.14' 'docker.io'" + ) ); });