fisher
952de1ea8e
The health router is mounted at prefix="/api" in domaindingo's main.py, so /health 404s. Checking it would have failed every deploy and rolled back a perfectly good image. Also adopts three conventions from the vault's DomainDingo Development note: recreate a single named service (--no-deps --force-recreate), validate with compose config --quiet before touching the running container, and write the rollback record in the documented format and location.
373 lines
14 KiB
Bash
Executable File
373 lines
14 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
#
|
||
# Generic, host/repo/branch-aware deployment.
|
||
#
|
||
# One script for every project and every environment in the fleet. It is driven
|
||
# entirely by targets.json, so onboarding a new environment never means writing
|
||
# another copy of this logic inside a project repository.
|
||
#
|
||
# Invoked by the webhook daemon as:
|
||
# cd-deploy --repo OWNER/NAME --ref refs/heads/BRANCH --sha FULLSHA [--event push]
|
||
#
|
||
# Deliberate properties, each one a lesson from the scripts this replaces:
|
||
#
|
||
# * It never touches git. The compose stacks live in uas-ng and are refreshed
|
||
# by that repo's own updater timer. A deploy script that runs `git reset
|
||
# --hard` owns two jobs badly instead of one job well.
|
||
# * It deploys the immutable <branch>-sha-<short> tag, then re-pins to the
|
||
# resolved digest. Mutable tags like :latest and :test have already drifted
|
||
# once in this fleet and served a broken build.
|
||
# * It verifies the image's org.opencontainers.image.revision label equals the
|
||
# commit that triggered the deploy, before changing anything.
|
||
# * It never runs `docker compose down`, and most emphatically never
|
||
# `down -v` -- `up -d` recreates exactly the services whose image changed.
|
||
# * A target belonging to another host is a clean no-op, logged as such, and
|
||
# never reported as a success.
|
||
|
||
set -Eeuo pipefail
|
||
|
||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
|
||
readonly SCRIPT_DIR
|
||
readonly CD_TARGET="${SCRIPT_DIR}/cd-target"
|
||
|
||
readonly STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/cd-webhook"
|
||
readonly NO_MATCH=3
|
||
|
||
REPO=""
|
||
REF=""
|
||
SHA=""
|
||
EVENT="push"
|
||
DRY_RUN=0
|
||
|
||
# ---------------------------------------------------------------- logging ---
|
||
|
||
log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
|
||
warn() { log "WARN $*" >&2; }
|
||
die() { log "ERROR $*" >&2; exit 1; }
|
||
|
||
usage() {
|
||
sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||
exit "${1:-0}"
|
||
}
|
||
|
||
# ------------------------------------------------------------ arg parsing ---
|
||
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--repo) REPO="${2:?--repo needs a value}"; shift 2 ;;
|
||
--ref) REF="${2:?--ref needs a value}"; shift 2 ;;
|
||
--sha) SHA="${2:?--sha needs a value}"; shift 2 ;;
|
||
--event) EVENT="${2:?--event needs a value}"; shift 2 ;;
|
||
--dry-run) DRY_RUN=1; shift ;;
|
||
-h|--help) usage 0 ;;
|
||
*) die "unknown argument: $1 (try --help)" ;;
|
||
esac
|
||
done
|
||
|
||
[[ -n "$REPO" ]] || die "--repo is required"
|
||
[[ -n "$REF" ]] || die "--ref is required"
|
||
[[ -n "$SHA" ]] || die "--sha is required"
|
||
|
||
if [[ "$EVENT" != "push" ]]; then
|
||
log "ignoring event '${EVENT}' for ${REPO} (only 'push' deploys)"
|
||
exit 0
|
||
fi
|
||
|
||
if [[ "$REF" != refs/heads/* ]]; then
|
||
log "ignoring non-branch ref '${REF}' for ${REPO} (tags and deletes never deploy)"
|
||
exit 0
|
||
fi
|
||
BRANCH="${REF#refs/heads/}"
|
||
|
||
if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then
|
||
die "--sha must be a full 40-character hex commit, got: ${SHA}"
|
||
fi
|
||
|
||
# The all-zero SHA is how git spells "this ref was deleted".
|
||
if [[ "$SHA" == "0000000000000000000000000000000000000000" ]]; then
|
||
log "ignoring branch deletion of ${REPO}@${BRANCH}"
|
||
exit 0
|
||
fi
|
||
|
||
# ------------------------------------------------------ target resolution ---
|
||
|
||
set +e
|
||
resolved="$("$CD_TARGET" resolve --repo "$REPO" --branch "$BRANCH")"
|
||
resolve_rc=$?
|
||
set -e
|
||
|
||
if [[ $resolve_rc -eq $NO_MATCH ]]; then
|
||
log "no target on $(hostname) for ${REPO}@${BRANCH} -- nothing to do here"
|
||
exit 0
|
||
elif [[ $resolve_rc -ne 0 ]]; then
|
||
die "target lookup failed for ${REPO}@${BRANCH} (exit ${resolve_rc})"
|
||
fi
|
||
|
||
eval "$resolved"
|
||
|
||
mkdir -p "$STATE_DIR"
|
||
readonly LOG_FILE="${STATE_DIR}/${CD_NAME}.log"
|
||
readonly STATUS_FILE="${STATE_DIR}/${CD_NAME}.status"
|
||
exec > >(stdbuf -oL tee -a "$LOG_FILE") 2>&1
|
||
|
||
log "=============================================================="
|
||
log "target ${CD_NAME} (${CD_ENV} on ${CD_RESOLVED_HOST})"
|
||
log "repo ${REPO}@${BRANCH}"
|
||
log "commit ${SHA}"
|
||
log "stack ${CD_STACK_DIR}/${CD_COMPOSE_FILE} [project ${CD_COMPOSE_PROJECT}]"
|
||
[[ $DRY_RUN -eq 1 ]] && log "mode DRY RUN -- nothing will be changed"
|
||
|
||
[[ -d "$CD_STACK_DIR" ]] || die "stack directory missing: ${CD_STACK_DIR}"
|
||
[[ -f "${CD_STACK_DIR}/${CD_COMPOSE_FILE}" ]] \
|
||
|| die "compose file missing: ${CD_STACK_DIR}/${CD_COMPOSE_FILE}"
|
||
|
||
# ----------------------------------------------------------------- notify ---
|
||
|
||
notify() {
|
||
local status="$1" message="$2" icon topic
|
||
case "$status" in
|
||
ok) icon="✅" ;;
|
||
fail) icon="❌" ;;
|
||
*) icon="ℹ️" ;;
|
||
esac
|
||
|
||
# Every terminal outcome notifies, so this is also where the machine-readable
|
||
# record of that outcome belongs. cd-status reads this file rather than
|
||
# grepping the log's prose, which quietly reported "no completed deploy"
|
||
# for a healthy target whenever a log line was reworded.
|
||
printf '%s\t%s\t%s\t%s\n' \
|
||
"$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$status" "${SHA:0:12}" "$message" \
|
||
> "$STATUS_FILE"
|
||
|
||
topic="${REPO//\//-}"
|
||
curl -fsS --max-time 10 \
|
||
-H "Title: ${CD_NAME} deploy" \
|
||
-d "${icon} ${CD_NAME} (${CD_ENV}): ${message}" \
|
||
"${CD_NTFY_BASE_URL}/${topic}" >/dev/null 2>&1 \
|
||
|| warn "ntfy notification failed (the deploy itself is unaffected)"
|
||
}
|
||
|
||
# ------------------------------------------------------------------- lock ---
|
||
|
||
# Serialise per target, not per host: a test deploy must not block a prod one.
|
||
readonly LOCK_FILE="${STATE_DIR}/${CD_NAME}.lock"
|
||
exec 9>"$LOCK_FILE"
|
||
log "waiting for deploy lock (${CD_LOCK_WAIT_SECONDS}s max)"
|
||
if ! flock -w "$CD_LOCK_WAIT_SECONDS" 9; then
|
||
notify fail "timed out waiting for the deploy lock after ${CD_LOCK_WAIT_SECONDS}s"
|
||
die "timed out waiting for deploy lock after ${CD_LOCK_WAIT_SECONDS}s"
|
||
fi
|
||
log "lock acquired"
|
||
|
||
# -------------------------------------------------------------- image tag ---
|
||
|
||
tag="$CD_IMAGE_TAG_TEMPLATE"
|
||
tag="${tag//\{branch\}/$BRANCH}"
|
||
tag="${tag//\{env\}/$CD_ENV}"
|
||
tag="${tag//\{sha\}/$SHA}"
|
||
tag="${tag//\{short7\}/${SHA:0:7}}"
|
||
tag="${tag//\{short12\}/${SHA:0:12}}"
|
||
readonly CANDIDATE="${CD_IMAGE_REPO}:${tag}"
|
||
|
||
log "candidate ${CANDIDATE}"
|
||
|
||
# ------------------------------------------------------- wait for the image --
|
||
|
||
# The push webhook fires the moment the commit lands, which is well before CI
|
||
# has finished building. Poll rather than fail: the running container keeps
|
||
# serving throughout.
|
||
wait_for_image() {
|
||
local elapsed=0
|
||
while true; do
|
||
if docker pull "$CANDIDATE" >/dev/null 2>&1; then
|
||
log "image available after ${elapsed}s"
|
||
return 0
|
||
fi
|
||
if (( elapsed >= CD_IMAGE_WAIT_SECONDS )); then
|
||
return 1
|
||
fi
|
||
log "image not published yet, retrying in ${CD_IMAGE_POLL_INTERVAL}s (${elapsed}s elapsed)"
|
||
sleep "$CD_IMAGE_POLL_INTERVAL"
|
||
elapsed=$(( elapsed + CD_IMAGE_POLL_INTERVAL ))
|
||
done
|
||
}
|
||
|
||
log "waiting for CI to publish the image (up to ${CD_IMAGE_WAIT_SECONDS}s)"
|
||
if [[ $DRY_RUN -eq 1 ]]; then
|
||
log "dry run: skipping image wait"
|
||
elif ! wait_for_image; then
|
||
notify fail "image ${CANDIDATE} never appeared (waited ${CD_IMAGE_WAIT_SECONDS}s) -- did CI fail?"
|
||
die "image ${CANDIDATE} did not appear within ${CD_IMAGE_WAIT_SECONDS}s"
|
||
fi
|
||
|
||
# --------------------------------------------------- verify and pin digest ---
|
||
|
||
image_digest_ref() {
|
||
docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$1" 2>/dev/null \
|
||
| grep "^${CD_IMAGE_REPO}@" | head -n1
|
||
}
|
||
|
||
if [[ $DRY_RUN -eq 1 ]]; then
|
||
DEPLOY_IMAGE="$CANDIDATE"
|
||
log "dry run: would deploy ${DEPLOY_IMAGE}"
|
||
else
|
||
revision="$(docker image inspect \
|
||
--format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
|
||
"$CANDIDATE" 2>/dev/null || true)"
|
||
|
||
if [[ -z "$revision" || "$revision" == "<no value>" ]]; then
|
||
warn "image carries no org.opencontainers.image.revision label; cannot prove provenance"
|
||
elif [[ "$revision" != "$SHA" ]]; then
|
||
notify fail "image ${CANDIDATE} is built from ${revision:0:12}, not ${SHA:0:12} -- refusing to deploy"
|
||
die "provenance mismatch: ${CANDIDATE} declares revision ${revision}, expected ${SHA}"
|
||
else
|
||
log "provenance revision label matches ${SHA:0:12}"
|
||
fi
|
||
|
||
DEPLOY_IMAGE="$(image_digest_ref "$CANDIDATE")"
|
||
if [[ -z "$DEPLOY_IMAGE" ]]; then
|
||
warn "could not resolve a digest for ${CANDIDATE}; deploying by tag instead"
|
||
DEPLOY_IMAGE="$CANDIDATE"
|
||
else
|
||
log "pinned ${DEPLOY_IMAGE}"
|
||
fi
|
||
fi
|
||
readonly DEPLOY_IMAGE
|
||
|
||
# --------------------------------------------- remember what is running now ---
|
||
|
||
RunningImageId="$(docker inspect --format '{{.Image}}' "${CD_CONTAINER}" 2>/dev/null || true)"
|
||
PREVIOUS_IMAGE=""
|
||
PREVIOUS_REVISION=""
|
||
if [[ -n "${RunningImageId}" ]]; then
|
||
PREVIOUS_IMAGE="$(image_digest_ref "${RunningImageId}")"
|
||
PREVIOUS_REVISION="$(docker image inspect \
|
||
--format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
|
||
"${RunningImageId}" 2>/dev/null || true)"
|
||
fi
|
||
readonly PREVIOUS_IMAGE PREVIOUS_REVISION
|
||
|
||
if [[ -n "${PREVIOUS_IMAGE}" ]]; then
|
||
log "current ${PREVIOUS_IMAGE} (revision ${PREVIOUS_REVISION:0:12})"
|
||
else
|
||
log "current (nothing running -- first deploy, or container absent)"
|
||
fi
|
||
|
||
# ----------------------------------------------------------------- deploy ---
|
||
|
||
compose_up() {
|
||
local Image="$1"
|
||
# Matches the manual runbook in the vault's "DomainDingo Development" note:
|
||
# recreate exactly one service, leave its dependencies alone, and never run
|
||
# `docker compose down` -- that causes avoidable downtime, and `down -v`
|
||
# would destroy the bind-mounted data these stacks depend on.
|
||
#
|
||
# `--pull always` is deliberately absent: the image is already pinned to a
|
||
# digest and pulled, so there is nothing left to resolve.
|
||
( cd "${CD_STACK_DIR}" \
|
||
&& env "${CD_IMAGE_ENV_VAR}=${Image}" \
|
||
"${CD_PULL_POLICY_ENV_VAR}=never" \
|
||
timeout "${CD_COMPOSE_TIMEOUT_SECONDS}" \
|
||
docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \
|
||
up -d --no-build --no-deps --force-recreate "${CD_COMPOSE_SERVICE}" )
|
||
}
|
||
|
||
compose_validate() {
|
||
local Image="$1"
|
||
( cd "${CD_STACK_DIR}" \
|
||
&& env "${CD_IMAGE_ENV_VAR}=${Image}" \
|
||
"${CD_PULL_POLICY_ENV_VAR}=never" \
|
||
docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \
|
||
config --quiet )
|
||
}
|
||
|
||
# Write the rollback record where the manual runbook looks for it, in the same
|
||
# format, so an operator following that note can recover from an automated
|
||
# deploy without knowing this tool exists.
|
||
write_rollback_record() {
|
||
local Image="$1" Revision="$2" Dir="${CD_STACK_DIR}/${CD_ROLLBACK_RECORD_DIR}"
|
||
local Record="${Dir}/${CD_COMPOSE_SERVICE}-rollback-$(date -u +%Y%m%d-%H%M%S).env"
|
||
|
||
[[ -d "${Dir}" ]] || { warn "no rollback record directory at ${Dir}"; return 0; }
|
||
|
||
( umask 077
|
||
printf 'DD_ROLLBACK_IMAGE=%s\nDD_ROLLBACK_REVISION=%s\nDD_ROLLBACK_CAPTURED_AT=%s\n' \
|
||
"${Image}" "${Revision}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "${Record}" )
|
||
log "rollback recorded at ${Record}"
|
||
}
|
||
|
||
check_health() {
|
||
local attempt=1 body
|
||
while (( attempt <= CD_HEALTH_RETRIES )); do
|
||
body="$(curl -fsS --max-time 5 "$CD_HEALTH_URL" 2>/dev/null || true)"
|
||
if [[ -n "$body" ]]; then
|
||
if python3 -c '
|
||
import json, sys
|
||
key, want = sys.argv[1], sys.argv[2]
|
||
try:
|
||
data = json.loads(sys.stdin.read())
|
||
except Exception:
|
||
sys.exit(1)
|
||
sys.exit(0 if str(data.get(key, "")).lower() == want.lower() else 1)
|
||
' "$CD_HEALTH_EXPECT_KEY" "$CD_HEALTH_EXPECT_VALUE" <<<"$body"; then
|
||
log "health ok after ${attempt} attempt(s): ${body}"
|
||
return 0
|
||
fi
|
||
fi
|
||
log "health not ready (attempt ${attempt}/${CD_HEALTH_RETRIES}), retrying in ${CD_HEALTH_INTERVAL}s"
|
||
sleep "$CD_HEALTH_INTERVAL"
|
||
(( attempt++ ))
|
||
done
|
||
warn "health check never passed: last response was '${body:-<no response>}'"
|
||
return 1
|
||
}
|
||
|
||
if [[ $DRY_RUN -eq 1 ]]; then
|
||
log "dry run: would recreate service ${CD_COMPOSE_SERVICE} with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}"
|
||
log "dry run: would health-check ${CD_HEALTH_URL}"
|
||
log "dry run complete -- no changes made"
|
||
exit 0
|
||
fi
|
||
|
||
# Catch a broken compose file or a bad interpolation before the running
|
||
# container is touched.
|
||
if ! compose_validate "${DEPLOY_IMAGE}"; then
|
||
notify fail "compose config is invalid for ${SHA:0:12} -- nothing was changed"
|
||
die "docker compose config failed; the running stack was not touched"
|
||
fi
|
||
|
||
[[ -n "${PREVIOUS_IMAGE}" ]] && write_rollback_record "${PREVIOUS_IMAGE}" "${PREVIOUS_REVISION}"
|
||
|
||
log "deploying ${DEPLOY_IMAGE}"
|
||
if ! compose_up "${DEPLOY_IMAGE}"; then
|
||
notify fail "docker compose up failed for ${SHA:0:12} -- stack left as-is"
|
||
die "docker compose up failed"
|
||
fi
|
||
|
||
if check_health; then
|
||
log "=== deploy succeeded: ${CD_NAME} now runs ${SHA:0:12} ==="
|
||
notify ok "deployed ${SHA:0:12} (${DEPLOY_IMAGE##*@})"
|
||
exit 0
|
||
fi
|
||
|
||
# ---------------------------------------------------------------- rollback ---
|
||
|
||
if [[ "$CD_ROLLBACK_ON_FAILURE" != "true" ]]; then
|
||
notify fail "health check failed for ${SHA:0:12}; rollback disabled, stack left on the new image"
|
||
die "health check failed and rollback is disabled for this target"
|
||
fi
|
||
|
||
if [[ -z "$PREVIOUS_IMAGE" ]]; then
|
||
notify fail "health check failed for ${SHA:0:12} and there is no previous image to roll back to"
|
||
die "health check failed; no previous image recorded, leaving the stack as it is"
|
||
fi
|
||
|
||
warn "health check failed -- rolling back to ${PREVIOUS_IMAGE}"
|
||
if compose_up "$PREVIOUS_IMAGE" && check_health; then
|
||
notify fail "deploy of ${SHA:0:12} failed health check; rolled back to the previous image successfully"
|
||
die "deploy failed health check; rolled back to ${PREVIOUS_IMAGE}"
|
||
fi
|
||
|
||
notify fail "deploy of ${SHA:0:12} failed AND rollback failed -- ${CD_NAME} needs manual attention now"
|
||
die "deploy failed and rollback also failed; manual intervention required"
|