320 lines
12 KiB
Bash
Executable File
320 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
#
|
||
# Generic, host/repo/branch-aware deployment.
|
||
#
|
||
# One script for every project and every environment in the fleet. It is driven
|
||
# entirely by targets.json, so onboarding a new environment never means writing
|
||
# another copy of this logic inside a project repository.
|
||
#
|
||
# Invoked by the webhook daemon as:
|
||
# cd-deploy --repo OWNER/NAME --ref refs/heads/BRANCH --sha FULLSHA [--event push]
|
||
#
|
||
# Deliberate properties, each one a lesson from the scripts this replaces:
|
||
#
|
||
# * It never touches git. The compose stacks live in uas-ng and are refreshed
|
||
# by that repo's own updater timer. A deploy script that runs `git reset
|
||
# --hard` owns two jobs badly instead of one job well.
|
||
# * It deploys the immutable <branch>-sha-<short> tag, then re-pins to the
|
||
# resolved digest. Mutable tags like :latest and :test have already drifted
|
||
# once in this fleet and served a broken build.
|
||
# * It verifies the image's org.opencontainers.image.revision label equals the
|
||
# commit that triggered the deploy, before changing anything.
|
||
# * It never runs `docker compose down`, and most emphatically never
|
||
# `down -v` -- `up -d` recreates exactly the services whose image changed.
|
||
# * A target belonging to another host is a clean no-op, logged as such, and
|
||
# never reported as a success.
|
||
|
||
set -Eeuo pipefail
|
||
|
||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||
readonly SCRIPT_DIR
|
||
readonly CD_TARGET="${SCRIPT_DIR}/cd-target"
|
||
|
||
readonly STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/cd-webhook"
|
||
readonly NO_MATCH=3
|
||
|
||
REPO=""
|
||
REF=""
|
||
SHA=""
|
||
EVENT="push"
|
||
DRY_RUN=0
|
||
|
||
# ---------------------------------------------------------------- logging ---
|
||
|
||
log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
|
||
warn() { log "WARN $*" >&2; }
|
||
die() { log "ERROR $*" >&2; exit 1; }
|
||
|
||
usage() {
|
||
sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||
exit "${1:-0}"
|
||
}
|
||
|
||
# ------------------------------------------------------------ arg parsing ---
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--repo) REPO="${2:?--repo needs a value}"; shift 2 ;;
|
||
--ref) REF="${2:?--ref needs a value}"; shift 2 ;;
|
||
--sha) SHA="${2:?--sha needs a value}"; shift 2 ;;
|
||
--event) EVENT="${2:?--event needs a value}"; shift 2 ;;
|
||
--dry-run) DRY_RUN=1; shift ;;
|
||
-h|--help) usage 0 ;;
|
||
*) die "unknown argument: $1 (try --help)" ;;
|
||
esac
|
||
done
|
||
|
||
[[ -n "$REPO" ]] || die "--repo is required"
|
||
[[ -n "$REF" ]] || die "--ref is required"
|
||
[[ -n "$SHA" ]] || die "--sha is required"
|
||
|
||
if [[ "$EVENT" != "push" ]]; then
|
||
log "ignoring event '${EVENT}' for ${REPO} (only 'push' deploys)"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$REF" != refs/heads/* ]]; then
|
||
log "ignoring non-branch ref '${REF}' for ${REPO} (tags and deletes never deploy)"
|
||
exit 0
|
||
fi
|
||
BRANCH="${REF#refs/heads/}"
|
||
|
||
if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then
|
||
die "--sha must be a full 40-character hex commit, got: ${SHA}"
|
||
fi
|
||
|
||
# The all-zero SHA is how git spells "this ref was deleted".
|
||
if [[ "$SHA" == "0000000000000000000000000000000000000000" ]]; then
|
||
log "ignoring branch deletion of ${REPO}@${BRANCH}"
|
||
exit 0
|
||
fi
|
||
|
||
# ------------------------------------------------------ target resolution ---
|
||
|
||
set +e
|
||
resolved="$("$CD_TARGET" resolve --repo "$REPO" --branch "$BRANCH")"
|
||
resolve_rc=$?
|
||
set -e
|
||
|
||
if [[ $resolve_rc -eq $NO_MATCH ]]; then
|
||
log "no target on $(hostname) for ${REPO}@${BRANCH} -- nothing to do here"
|
||
exit 0
|
||
elif [[ $resolve_rc -ne 0 ]]; then
|
||
die "target lookup failed for ${REPO}@${BRANCH} (exit ${resolve_rc})"
|
||
fi
|
||
|
||
eval "$resolved"
|
||
|
||
mkdir -p "$STATE_DIR"
|
||
readonly LOG_FILE="${STATE_DIR}/${CD_NAME}.log"
|
||
exec > >(stdbuf -oL tee -a "$LOG_FILE") 2>&1
|
||
|
||
log "=============================================================="
|
||
log "target ${CD_NAME} (${CD_ENV} on ${CD_RESOLVED_HOST})"
|
||
log "repo ${REPO}@${BRANCH}"
|
||
log "commit ${SHA}"
|
||
log "stack ${CD_STACK_DIR}/${CD_COMPOSE_FILE} [project ${CD_COMPOSE_PROJECT}]"
|
||
[[ $DRY_RUN -eq 1 ]] && log "mode DRY RUN -- nothing will be changed"
|
||
|
||
[[ -d "$CD_STACK_DIR" ]] || die "stack directory missing: ${CD_STACK_DIR}"
|
||
[[ -f "${CD_STACK_DIR}/${CD_COMPOSE_FILE}" ]] \
|
||
|| die "compose file missing: ${CD_STACK_DIR}/${CD_COMPOSE_FILE}"
|
||
|
||
# ----------------------------------------------------------------- notify ---
|
||
|
||
notify() {
|
||
local status="$1" message="$2" icon topic
|
||
case "$status" in
|
||
ok) icon="✅" ;;
|
||
fail) icon="❌" ;;
|
||
*) icon="ℹ️" ;;
|
||
esac
|
||
topic="${REPO//\//-}"
|
||
curl -fsS --max-time 10 \
|
||
-H "Title: ${CD_NAME} deploy" \
|
||
-d "${icon} ${CD_NAME} (${CD_ENV}): ${message}" \
|
||
"${CD_NTFY_BASE_URL}/${topic}" >/dev/null 2>&1 \
|
||
|| warn "ntfy notification failed (the deploy itself is unaffected)"
|
||
}
|
||
|
||
# ------------------------------------------------------------------- lock ---
|
||
|
||
# Serialise per target, not per host: a test deploy must not block a prod one.
|
||
readonly LOCK_FILE="${STATE_DIR}/${CD_NAME}.lock"
|
||
exec 9>"$LOCK_FILE"
|
||
log "waiting for deploy lock (${CD_LOCK_WAIT_SECONDS}s max)"
|
||
if ! flock -w "$CD_LOCK_WAIT_SECONDS" 9; then
|
||
notify fail "timed out waiting for the deploy lock after ${CD_LOCK_WAIT_SECONDS}s"
|
||
die "timed out waiting for deploy lock after ${CD_LOCK_WAIT_SECONDS}s"
|
||
fi
|
||
log "lock acquired"
|
||
|
||
# -------------------------------------------------------------- image tag ---
|
||
|
||
tag="$CD_IMAGE_TAG_TEMPLATE"
|
||
tag="${tag//\{branch\}/$BRANCH}"
|
||
tag="${tag//\{env\}/$CD_ENV}"
|
||
tag="${tag//\{sha\}/$SHA}"
|
||
tag="${tag//\{short7\}/${SHA:0:7}}"
|
||
tag="${tag//\{short12\}/${SHA:0:12}}"
|
||
readonly CANDIDATE="${CD_IMAGE_REPO}:${tag}"
|
||
|
||
log "candidate ${CANDIDATE}"
|
||
|
||
# ------------------------------------------------------- wait for the image --
|
||
|
||
# The push webhook fires the moment the commit lands, which is well before CI
|
||
# has finished building. Poll rather than fail: the running container keeps
|
||
# serving throughout.
|
||
wait_for_image() {
|
||
local elapsed=0
|
||
while true; do
|
||
if docker pull "$CANDIDATE" >/dev/null 2>&1; then
|
||
log "image available after ${elapsed}s"
|
||
return 0
|
||
fi
|
||
if (( elapsed >= CD_IMAGE_WAIT_SECONDS )); then
|
||
return 1
|
||
fi
|
||
log "image not published yet, retrying in ${CD_IMAGE_POLL_INTERVAL}s (${elapsed}s elapsed)"
|
||
sleep "$CD_IMAGE_POLL_INTERVAL"
|
||
elapsed=$(( elapsed + CD_IMAGE_POLL_INTERVAL ))
|
||
done
|
||
}
|
||
|
||
log "waiting for CI to publish the image (up to ${CD_IMAGE_WAIT_SECONDS}s)"
|
||
if [[ $DRY_RUN -eq 1 ]]; then
|
||
log "dry run: skipping image wait"
|
||
elif ! wait_for_image; then
|
||
notify fail "image ${CANDIDATE} never appeared (waited ${CD_IMAGE_WAIT_SECONDS}s) -- did CI fail?"
|
||
die "image ${CANDIDATE} did not appear within ${CD_IMAGE_WAIT_SECONDS}s"
|
||
fi
|
||
|
||
# --------------------------------------------------- verify and pin digest ---
|
||
|
||
image_digest_ref() {
|
||
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$1" 2>/dev/null \
|
||
| grep "^${CD_IMAGE_REPO}@" | head -n1
|
||
}
|
||
|
||
if [[ $DRY_RUN -eq 1 ]]; then
|
||
DEPLOY_IMAGE="$CANDIDATE"
|
||
log "dry run: would deploy ${DEPLOY_IMAGE}"
|
||
else
|
||
revision="$(docker image inspect \
|
||
--format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
|
||
"$CANDIDATE" 2>/dev/null || true)"
|
||
|
||
if [[ -z "$revision" || "$revision" == "<no value>" ]]; then
|
||
warn "image carries no org.opencontainers.image.revision label; cannot prove provenance"
|
||
elif [[ "$revision" != "$SHA" ]]; then
|
||
notify fail "image ${CANDIDATE} is built from ${revision:0:12}, not ${SHA:0:12} -- refusing to deploy"
|
||
die "provenance mismatch: ${CANDIDATE} declares revision ${revision}, expected ${SHA}"
|
||
else
|
||
log "provenance revision label matches ${SHA:0:12}"
|
||
fi
|
||
|
||
DEPLOY_IMAGE="$(image_digest_ref "$CANDIDATE")"
|
||
if [[ -z "$DEPLOY_IMAGE" ]]; then
|
||
warn "could not resolve a digest for ${CANDIDATE}; deploying by tag instead"
|
||
DEPLOY_IMAGE="$CANDIDATE"
|
||
else
|
||
log "pinned ${DEPLOY_IMAGE}"
|
||
fi
|
||
fi
|
||
readonly DEPLOY_IMAGE
|
||
|
||
# --------------------------------------------- remember what is running now ---
|
||
|
||
PREVIOUS_IMAGE="$(
|
||
container_image="$(docker inspect --format '{{.Image}}' "$CD_CONTAINER" 2>/dev/null || true)"
|
||
[[ -n "$container_image" ]] && image_digest_ref "$container_image" || true
|
||
)"
|
||
readonly PREVIOUS_IMAGE
|
||
|
||
if [[ -n "$PREVIOUS_IMAGE" ]]; then
|
||
log "current ${PREVIOUS_IMAGE}"
|
||
else
|
||
log "current (nothing running -- first deploy, or container absent)"
|
||
fi
|
||
|
||
# ----------------------------------------------------------------- deploy ---
|
||
|
||
compose_up() {
|
||
local image="$1"
|
||
# `up -d` recreates only what actually changed. `down` is deliberately not
|
||
# used here: it causes avoidable downtime, and `down -v` would destroy the
|
||
# data volumes these stacks depend on.
|
||
( cd "$CD_STACK_DIR" \
|
||
&& env "${CD_IMAGE_ENV_VAR}=${image}" \
|
||
"${CD_PULL_POLICY_ENV_VAR}=missing" \
|
||
timeout "$CD_COMPOSE_TIMEOUT_SECONDS" \
|
||
docker compose -f "$CD_COMPOSE_FILE" -p "$CD_COMPOSE_PROJECT" up -d --no-build )
|
||
}
|
||
|
||
check_health() {
|
||
local attempt=1 body
|
||
while (( attempt <= CD_HEALTH_RETRIES )); do
|
||
body="$(curl -fsS --max-time 5 "$CD_HEALTH_URL" 2>/dev/null || true)"
|
||
if [[ -n "$body" ]]; then
|
||
if python3 -c '
|
||
import json, sys
|
||
key, want = sys.argv[1], sys.argv[2]
|
||
try:
|
||
data = json.loads(sys.stdin.read())
|
||
except Exception:
|
||
sys.exit(1)
|
||
sys.exit(0 if str(data.get(key, "")).lower() == want.lower() else 1)
|
||
' "$CD_HEALTH_EXPECT_KEY" "$CD_HEALTH_EXPECT_VALUE" <<<"$body"; then
|
||
log "health ok after ${attempt} attempt(s): ${body}"
|
||
return 0
|
||
fi
|
||
fi
|
||
log "health not ready (attempt ${attempt}/${CD_HEALTH_RETRIES}), retrying in ${CD_HEALTH_INTERVAL}s"
|
||
sleep "$CD_HEALTH_INTERVAL"
|
||
(( attempt++ ))
|
||
done
|
||
warn "health check never passed: last response was '${body:-<no response>}'"
|
||
return 1
|
||
}
|
||
|
||
if [[ $DRY_RUN -eq 1 ]]; then
|
||
log "dry run: would run docker compose up -d with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}"
|
||
log "dry run: would health-check ${CD_HEALTH_URL}"
|
||
log "dry run complete -- no changes made"
|
||
exit 0
|
||
fi
|
||
|
||
log "deploying ${DEPLOY_IMAGE}"
|
||
if ! compose_up "$DEPLOY_IMAGE"; then
|
||
notify fail "docker compose up failed for ${SHA:0:12} -- stack left as-is"
|
||
die "docker compose up failed"
|
||
fi
|
||
|
||
if check_health; then
|
||
log "=== deploy succeeded: ${CD_NAME} now runs ${SHA:0:12} ==="
|
||
notify ok "deployed ${SHA:0:12} (${DEPLOY_IMAGE##*@})"
|
||
exit 0
|
||
fi
|
||
|
||
# ---------------------------------------------------------------- rollback ---
|
||
|
||
if [[ "$CD_ROLLBACK_ON_FAILURE" != "True" && "$CD_ROLLBACK_ON_FAILURE" != "true" ]]; then
|
||
notify fail "health check failed for ${SHA:0:12}; rollback disabled, stack left on the new image"
|
||
die "health check failed and rollback is disabled for this target"
|
||
fi
|
||
|
||
if [[ -z "$PREVIOUS_IMAGE" ]]; then
|
||
notify fail "health check failed for ${SHA:0:12} and there is no previous image to roll back to"
|
||
die "health check failed; no previous image recorded, leaving the stack as it is"
|
||
fi
|
||
|
||
warn "health check failed -- rolling back to ${PREVIOUS_IMAGE}"
|
||
if compose_up "$PREVIOUS_IMAGE" && check_health; then
|
||
notify fail "deploy of ${SHA:0:12} failed health check; rolled back to the previous image successfully"
|
||
die "deploy failed health check; rolled back to ${PREVIOUS_IMAGE}"
|
||
fi
|
||
|
||
notify fail "deploy of ${SHA:0:12} failed AND rollback failed -- ${CD_NAME} needs manual attention now"
|
||
die "deploy failed and rollback also failed; manual intervention required"
|