#!/usr/bin/env bash # # Generic, host/repo/branch-aware deployment. # # One script for every project and every environment in the fleet. It is driven # entirely by targets.json, so onboarding a new environment never means writing # another copy of this logic inside a project repository. # # Invoked by the webhook daemon as: # cd-deploy --repo OWNER/NAME --ref refs/heads/BRANCH --sha FULLSHA [--event push] # # Deliberate properties, each one a lesson from the scripts this replaces: # # * It never touches git. The compose stacks live in uas-ng and are refreshed # by that repo's own updater timer. A deploy script that runs `git reset # --hard` owns two jobs badly instead of one job well. # * It deploys the immutable -sha- tag, then re-pins to the # resolved digest. Mutable tags like :latest and :test have already drifted # once in this fleet and served a broken build. # * It verifies the image's org.opencontainers.image.revision label equals the # commit that triggered the deploy, before changing anything. # * It never runs `docker compose down`, and most emphatically never # `down -v` -- `up -d` recreates exactly the services whose image changed. # * A target belonging to another host is a clean no-op, logged as such, and # never reported as a success. set -Eeuo pipefail SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" readonly SCRIPT_DIR readonly CD_TARGET="${SCRIPT_DIR}/cd-target" readonly STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/cd-webhook" readonly NO_MATCH=3 REPO="" REF="" SHA="" EVENT="push" DRY_RUN=0 # ---------------------------------------------------------------- logging --- log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } warn() { log "WARN $*" >&2; } die() { log "ERROR $*" >&2; exit 1; } usage() { sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' exit "${1:-0}" } # ------------------------------------------------------------ arg parsing --- while [[ $# -gt 0 ]]; do case "$1" in --repo) REPO="${2:?--repo needs a value}"; shift 2 ;; --ref) REF="${2:?--ref needs a value}"; shift 2 ;; --sha) SHA="${2:?--sha needs a value}"; shift 2 ;; --event) EVENT="${2:?--event needs a value}"; shift 2 ;; --dry-run) DRY_RUN=1; shift ;; -h|--help) usage 0 ;; *) die "unknown argument: $1 (try --help)" ;; esac done [[ -n "${REPO}" ]] || die "--repo is required" [[ -n "${REF}" ]] || die "--ref is required" [[ -n "${SHA}" ]] || die "--sha is required" if [[ "${EVENT}" != "push" ]]; then log "ignoring event '${EVENT}' for ${REPO} (only 'push' deploys)" exit 0 fi if [[ "${REF}" != refs/heads/* ]]; then log "ignoring non-branch ref '${REF}' for ${REPO} (tags and deletes never deploy)" exit 0 fi BRANCH="${REF#refs/heads/}" if [[ ! "${SHA}" =~ ^[0-9a-f]{40}$ ]]; then die "--sha must be a full 40-character hex commit, got: ${SHA}" fi # The all-zero SHA is how git spells "this ref was deleted". if [[ "${SHA}" == "0000000000000000000000000000000000000000" ]]; then log "ignoring branch deletion of ${REPO}@${BRANCH}" exit 0 fi # ------------------------------------------------------ target resolution --- set +e Resolved="$("${CD_TARGET}" resolve --repo "${REPO}" --branch "${BRANCH}")" ResolveRc=$? set -e if [[ ${ResolveRc} -eq ${NO_MATCH} ]]; then log "no target on $(hostname) for ${REPO}@${BRANCH} -- nothing to do here" exit 0 elif [[ ${ResolveRc} -ne 0 ]]; then die "target lookup failed for ${REPO}@${BRANCH} (exit ${ResolveRc})" fi eval "${Resolved}" mkdir -p "${STATE_DIR}" readonly LOG_FILE="${STATE_DIR}/${CD_NAME}.log" readonly STATUS_FILE="${STATE_DIR}/${CD_NAME}.status" exec > >(stdbuf -oL tee -a "${LOG_FILE}") 2>&1 log "==============================================================" log "target ${CD_NAME} (${CD_ENV} on ${CD_RESOLVED_HOST})" log "repo ${REPO}@${BRANCH}" log "commit ${SHA}" log "stack ${CD_STACK_DIR}/${CD_COMPOSE_FILE} [project ${CD_COMPOSE_PROJECT}]" [[ ${DRY_RUN} -eq 1 ]] && log "mode DRY RUN -- nothing will be changed" [[ -d "${CD_STACK_DIR}" ]] || die "stack directory missing: ${CD_STACK_DIR}" [[ -f "${CD_STACK_DIR}/${CD_COMPOSE_FILE}" ]] \ || die "compose file missing: ${CD_STACK_DIR}/${CD_COMPOSE_FILE}" # ----------------------------------------------------------------- notify --- notify() { local Status="$1" Message="$2" Icon Topic case "${Status}" in ok) Icon="✅" ;; fail) Icon="❌" ;; *) Icon="â„šī¸" ;; esac # Every terminal outcome notifies, so this is also where the machine-readable # record of that outcome belongs. cd-status reads this file rather than # grepping the log's prose, which quietly reported "no completed deploy" # for a healthy target whenever a log line was reworded. printf '%s\t%s\t%s\t%s\n' \ "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "${Status}" "${SHA:0:12}" "${Message}" \ > "${STATUS_FILE}" Topic="${REPO//\//-}" curl -fsS --max-time 10 \ -H "Title: ${CD_NAME} deploy" \ -d "${Icon} ${CD_NAME} (${CD_ENV}): ${Message}" \ "${CD_NTFY_BASE_URL}/${Topic}" >/dev/null 2>&1 \ || warn "ntfy notification failed (the deploy itself is unaffected)" } # ------------------------------------------------------------------- lock --- # Serialise per target, not per host: a test deploy must not block a prod one. readonly LOCK_FILE="${STATE_DIR}/${CD_NAME}.lock" exec 9>"${LOCK_FILE}" log "waiting for deploy lock (${CD_LOCK_WAIT_SECONDS}s max)" if ! flock -w "${CD_LOCK_WAIT_SECONDS}" 9; then notify fail "timed out waiting for the deploy lock after ${CD_LOCK_WAIT_SECONDS}s" die "timed out waiting for deploy lock after ${CD_LOCK_WAIT_SECONDS}s" fi log "lock acquired" # -------------------------------------------------------------- image tag --- Tag="${CD_IMAGE_TAG_TEMPLATE}" Tag="${Tag//\{branch\}/$BRANCH}" Tag="${Tag//\{env\}/$CD_ENV}" Tag="${Tag//\{sha\}/$SHA}" Tag="${Tag//\{short7\}/${SHA:0:7}}" Tag="${Tag//\{short12\}/${SHA:0:12}}" readonly CANDIDATE="${CD_IMAGE_REPO}:${Tag}" log "candidate ${CANDIDATE}" # ------------------------------------------------------- wait for the image -- # The push webhook fires the moment the commit lands, which is well before CI # has finished building. Poll rather than fail: the running container keeps # serving throughout. wait_for_image() { local Elapsed=0 while true; do if docker pull "${CANDIDATE}" >/dev/null 2>&1; then log "image available after ${Elapsed}s" return 0 fi if (( Elapsed >= CD_IMAGE_WAIT_SECONDS )); then return 1 fi log "image not published yet, retrying in ${CD_IMAGE_POLL_INTERVAL}s (${Elapsed}s elapsed)" sleep "${CD_IMAGE_POLL_INTERVAL}" Elapsed=$(( Elapsed + CD_IMAGE_POLL_INTERVAL )) done } log "waiting for CI to publish the image (up to ${CD_IMAGE_WAIT_SECONDS}s)" if [[ ${DRY_RUN} -eq 1 ]]; then log "dry run: skipping image wait" elif ! wait_for_image; then notify fail "image ${CANDIDATE} never appeared (waited ${CD_IMAGE_WAIT_SECONDS}s) -- did CI fail?" die "image ${CANDIDATE} did not appear within ${CD_IMAGE_WAIT_SECONDS}s" fi # --------------------------------------------------- verify and pin digest --- image_digest_ref() { docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$1" 2>/dev/null \ | grep "^${CD_IMAGE_REPO}@" | head -n1 } if [[ ${DRY_RUN} -eq 1 ]]; then DEPLOY_IMAGE="${CANDIDATE}" log "dry run: would deploy ${DEPLOY_IMAGE}" else Revision="$(docker image inspect \ --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \ "${CANDIDATE}" 2>/dev/null || true)" if [[ -z "${Revision}" || "${Revision}" == "" ]]; then warn "image carries no org.opencontainers.image.revision label; cannot prove provenance" elif [[ "${Revision}" != "${SHA}" ]]; then notify fail "image ${CANDIDATE} is built from ${Revision:0:12}, not ${SHA:0:12} -- refusing to deploy" die "provenance mismatch: ${CANDIDATE} declares revision ${Revision}, expected ${SHA}" else log "provenance revision label matches ${SHA:0:12}" fi DEPLOY_IMAGE="$(image_digest_ref "${CANDIDATE}")" if [[ -z "${DEPLOY_IMAGE}" ]]; then warn "could not resolve a digest for ${CANDIDATE}; deploying by tag instead" DEPLOY_IMAGE="${CANDIDATE}" else log "pinned ${DEPLOY_IMAGE}" fi fi readonly DEPLOY_IMAGE # --------------------------------------------- remember what is running now --- RunningImageId="$(docker inspect --format '{{.Image}}' "${CD_CONTAINER}" 2>/dev/null || true)" PREVIOUS_IMAGE="" PREVIOUS_REVISION="" if [[ -n "${RunningImageId}" ]]; then PREVIOUS_IMAGE="$(image_digest_ref "${RunningImageId}")" PREVIOUS_REVISION="$(docker image inspect \ --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \ "${RunningImageId}" 2>/dev/null || true)" fi readonly PREVIOUS_IMAGE PREVIOUS_REVISION if [[ -n "${PREVIOUS_IMAGE}" ]]; then log "current ${PREVIOUS_IMAGE} (revision ${PREVIOUS_REVISION:0:12})" else log "current (nothing running -- first deploy, or container absent)" fi # ----------------------------------------------------------------- deploy --- compose_up() { local Image="$1" # Matches the manual runbook in the vault's "DomainDingo Development" note: # recreate exactly one service, leave its dependencies alone, and never run # `docker compose down` -- that causes avoidable downtime, and `down -v` # would destroy the bind-mounted data these stacks depend on. # # `--pull always` is deliberately absent: the image is already pinned to a # digest and pulled, so there is nothing left to resolve. ( cd "${CD_STACK_DIR}" \ && env "${CD_IMAGE_ENV_VAR}=${Image}" \ "${CD_PULL_POLICY_ENV_VAR}=never" \ timeout "${CD_COMPOSE_TIMEOUT_SECONDS}" \ docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \ up -d --no-build --no-deps --force-recreate "${CD_COMPOSE_SERVICE}" ) } compose_validate() { local Image="$1" ( cd "${CD_STACK_DIR}" \ && env "${CD_IMAGE_ENV_VAR}=${Image}" \ "${CD_PULL_POLICY_ENV_VAR}=never" \ docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \ config --quiet ) } # Write the rollback record where the manual runbook looks for it, in the same # format, so an operator following that note can recover from an automated # deploy without knowing this tool exists. write_rollback_record() { local Image="$1" Revision="$2" Dir="${CD_STACK_DIR}/${CD_ROLLBACK_RECORD_DIR}" local Record="${Dir}/${CD_COMPOSE_SERVICE}-rollback-$(date -u +%Y%m%d-%H%M%S).env" [[ -d "${Dir}" ]] || { warn "no rollback record directory at ${Dir}"; return 0; } ( umask 077 printf 'DD_ROLLBACK_IMAGE=%s\nDD_ROLLBACK_REVISION=%s\nDD_ROLLBACK_CAPTURED_AT=%s\n' \ "${Image}" "${Revision}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "${Record}" ) log "rollback recorded at ${Record}" } check_health() { local Attempt=1 Body while (( Attempt <= CD_HEALTH_RETRIES )); do Body="$(curl -fsS --max-time 5 "${CD_HEALTH_URL}" 2>/dev/null || true)" if [[ -n "${Body}" ]]; then if python3 -c ' import json, sys key, want = sys.argv[1], sys.argv[2] try: data = json.loads(sys.stdin.read()) except Exception: sys.exit(1) sys.exit(0 if str(data.get(key, "")).lower() == want.lower() else 1) ' "${CD_HEALTH_EXPECT_KEY}" "${CD_HEALTH_EXPECT_VALUE}" <<<"${Body}"; then log "health ok after ${Attempt} attempt(s): ${Body}" return 0 fi fi log "health not ready (attempt ${Attempt}/${CD_HEALTH_RETRIES}), retrying in ${CD_HEALTH_INTERVAL}s" sleep "${CD_HEALTH_INTERVAL}" (( Attempt++ )) done warn "health check never passed: last response was '${Body:-}'" return 1 } if [[ ${DRY_RUN} -eq 1 ]]; then log "dry run: would recreate service ${CD_COMPOSE_SERVICE} with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}" log "dry run: would health-check ${CD_HEALTH_URL}" log "dry run complete -- no changes made" exit 0 fi # Catch a broken compose file or a bad interpolation before the running # container is touched. if ! compose_validate "${DEPLOY_IMAGE}"; then notify fail "compose config is invalid for ${SHA:0:12} -- nothing was changed" die "docker compose config failed; the running stack was not touched" fi [[ -n "${PREVIOUS_IMAGE}" ]] && write_rollback_record "${PREVIOUS_IMAGE}" "${PREVIOUS_REVISION}" log "deploying ${DEPLOY_IMAGE}" if ! compose_up "${DEPLOY_IMAGE}"; then notify fail "docker compose up failed for ${SHA:0:12} -- stack left as-is" die "docker compose up failed" fi if check_health; then log "=== deploy succeeded: ${CD_NAME} now runs ${SHA:0:12} ===" notify ok "deployed ${SHA:0:12} (${DEPLOY_IMAGE##*@})" exit 0 fi # ---------------------------------------------------------------- rollback --- if [[ "${CD_ROLLBACK_ON_FAILURE}" != "true" ]]; then notify fail "health check failed for ${SHA:0:12}; rollback disabled, stack left on the new image" die "health check failed and rollback is disabled for this target" fi if [[ -z "${PREVIOUS_IMAGE}" ]]; then notify fail "health check failed for ${SHA:0:12} and there is no previous image to roll back to" die "health check failed; no previous image recorded, leaving the stack as it is" fi warn "health check failed -- rolling back to ${PREVIOUS_IMAGE}" if compose_up "${PREVIOUS_IMAGE}" && check_health; then notify fail "deploy of ${SHA:0:12} failed health check; rolled back to the previous image successfully" die "deploy failed health check; rolled back to ${PREVIOUS_IMAGE}" fi notify fail "deploy of ${SHA:0:12} failed AND rollback failed -- ${CD_NAME} needs manual attention now" die "deploy failed and rollback also failed; manual intervention required"