my-cd-webhook/bin/cd-deploy

#!/usr/bin/env bash
#
# Generic, host/repo/branch-aware deployment.
#
# One script for every project and every environment in the fleet. It is driven
# entirely by targets.json, so onboarding a new environment never means writing
# another copy of this logic inside a project repository.
#
# Invoked by the webhook daemon as:
#     cd-deploy --repo OWNER/NAME --ref refs/heads/BRANCH --sha FULLSHA [--event push]
#
# Deliberate properties, each one a lesson from the scripts this replaces:
#
#   * It never touches git. The compose stacks live in uas-ng and are refreshed
#     by that repo's own updater timer. A deploy script that runs `git reset
#     --hard` owns two jobs badly instead of one job well.
#   * It deploys the immutable <branch>-sha-<short> tag, then re-pins to the
#     resolved digest. Mutable tags like :latest and :test have already drifted
#     once in this fleet and served a broken build.
#   * It verifies the image's org.opencontainers.image.revision label equals the
#     commit that triggered the deploy, before changing anything.
#   * It never runs `docker compose down`, and most emphatically never
#     `down -v` -- `up -d` recreates exactly the services whose image changed.
#   * A target belonging to another host is a clean no-op, logged as such, and
#     never reported as a success.

set -Eeuo pipefail

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
readonly SCRIPT_DIR
readonly CD_TARGET="${SCRIPT_DIR}/cd-target"

readonly STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/cd-webhook"
readonly NO_MATCH=3

REPO=""
REF=""
SHA=""
EVENT="push"
DRY_RUN=0

# ---------------------------------------------------------------- logging ---

log()  { printf '%s  %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
warn() { log "WARN  $*" >&2; }
die()  { log "ERROR $*" >&2; exit 1; }

usage() {
    sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
    exit "${1:-0}"
}

# ------------------------------------------------------------ arg parsing ---

while [[ $# -gt 0 ]]; do
    case "$1" in
        --repo)     REPO="${2:?--repo needs a value}"; shift 2 ;;
        --ref)      REF="${2:?--ref needs a value}";   shift 2 ;;
        --sha)      SHA="${2:?--sha needs a value}";   shift 2 ;;
        --event)    EVENT="${2:?--event needs a value}"; shift 2 ;;
        --dry-run)  DRY_RUN=1; shift ;;
        -h|--help)  usage 0 ;;
        *)          die "unknown argument: $1 (try --help)" ;;
    esac
done

[[ -n "${REPO}" ]] || die "--repo is required"
[[ -n "${REF}"  ]] || die "--ref is required"
[[ -n "${SHA}"  ]] || die "--sha is required"

if [[ "${EVENT}" != "push" ]]; then
    log "ignoring event '${EVENT}' for ${REPO} (only 'push' deploys)"
    exit 0
fi

if [[ "${REF}" != refs/heads/* ]]; then
    log "ignoring non-branch ref '${REF}' for ${REPO} (tags and deletes never deploy)"
    exit 0
fi
BRANCH="${REF#refs/heads/}"

if [[ ! "${SHA}" =~ ^[0-9a-f]{40}$ ]]; then
    die "--sha must be a full 40-character hex commit, got: ${SHA}"
fi

# The all-zero SHA is how git spells "this ref was deleted".
if [[ "${SHA}" == "0000000000000000000000000000000000000000" ]]; then
    log "ignoring branch deletion of ${REPO}@${BRANCH}"
    exit 0
fi

# ------------------------------------------------------ target resolution ---

set +e
Resolved="$("${CD_TARGET}" resolve --repo "${REPO}" --branch "${BRANCH}")"
ResolveRc=$?
set -e

if [[ ${ResolveRc} -eq ${NO_MATCH} ]]; then
    log "no target on $(hostname) for ${REPO}@${BRANCH} -- nothing to do here"
    exit 0
elif [[ ${ResolveRc} -ne 0 ]]; then
    die "target lookup failed for ${REPO}@${BRANCH} (exit ${ResolveRc})"
fi

eval "${Resolved}"

mkdir -p "${STATE_DIR}"
readonly LOG_FILE="${STATE_DIR}/${CD_NAME}.log"
readonly STATUS_FILE="${STATE_DIR}/${CD_NAME}.status"
exec > >(stdbuf -oL tee -a "${LOG_FILE}") 2>&1

log "=============================================================="
log "target      ${CD_NAME}  (${CD_ENV} on ${CD_RESOLVED_HOST})"
log "repo        ${REPO}@${BRANCH}"
log "commit      ${SHA}"
log "stack       ${CD_STACK_DIR}/${CD_COMPOSE_FILE}  [project ${CD_COMPOSE_PROJECT}]"
[[ ${DRY_RUN} -eq 1 ]] && log "mode        DRY RUN -- nothing will be changed"

[[ -d "${CD_STACK_DIR}" ]] || die "stack directory missing: ${CD_STACK_DIR}"
[[ -f "${CD_STACK_DIR}/${CD_COMPOSE_FILE}" ]] \
    || die "compose file missing: ${CD_STACK_DIR}/${CD_COMPOSE_FILE}"

# ----------------------------------------------------------------- notify ---

notify() {
    local Status="$1" Message="$2" Icon Topic
    case "${Status}" in
        ok)   Icon="✅" ;;
        fail) Icon="❌" ;;
        *)    Icon="ℹ️" ;;
    esac

    # Every terminal outcome notifies, so this is also where the machine-readable
    # record of that outcome belongs. cd-status reads this file rather than
    # grepping the log's prose, which quietly reported "no completed deploy"
    # for a healthy target whenever a log line was reworded.
    printf '%s\t%s\t%s\t%s\n' \
        "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "${Status}" "${SHA:0:12}" "${Message}" \
        > "${STATUS_FILE}"

    Topic="${REPO//\//-}"
    curl -fsS --max-time 10 \
        -H "Title: ${CD_NAME} deploy" \
        -d "${Icon} ${CD_NAME} (${CD_ENV}): ${Message}" \
        "${CD_NTFY_BASE_URL}/${Topic}" >/dev/null 2>&1 \
        || warn "ntfy notification failed (the deploy itself is unaffected)"
}

# ------------------------------------------------------------------- lock ---

# Serialise per target, not per host: a test deploy must not block a prod one.
readonly LOCK_FILE="${STATE_DIR}/${CD_NAME}.lock"
exec 9>"${LOCK_FILE}"
log "waiting for deploy lock (${CD_LOCK_WAIT_SECONDS}s max)"
if ! flock -w "${CD_LOCK_WAIT_SECONDS}" 9; then
    notify fail "timed out waiting for the deploy lock after ${CD_LOCK_WAIT_SECONDS}s"
    die "timed out waiting for deploy lock after ${CD_LOCK_WAIT_SECONDS}s"
fi
log "lock acquired"

# -------------------------------------------------------------- image tag ---

Tag="${CD_IMAGE_TAG_TEMPLATE}"
Tag="${Tag//\{branch\}/$BRANCH}"
Tag="${Tag//\{env\}/$CD_ENV}"
Tag="${Tag//\{sha\}/$SHA}"
Tag="${Tag//\{short7\}/${SHA:0:7}}"
Tag="${Tag//\{short12\}/${SHA:0:12}}"
readonly CANDIDATE="${CD_IMAGE_REPO}:${Tag}"

log "candidate   ${CANDIDATE}"

# ------------------------------------------------------- wait for the image --

# The push webhook fires the moment the commit lands, which is well before CI
# has finished building. Poll rather than fail: the running container keeps
# serving throughout.
wait_for_image() {
    local Elapsed=0
    while true; do
        if docker pull "${CANDIDATE}" >/dev/null 2>&1; then
            log "image available after ${Elapsed}s"
            return 0
        fi
        if (( Elapsed >= CD_IMAGE_WAIT_SECONDS )); then
            return 1
        fi
        log "image not published yet, retrying in ${CD_IMAGE_POLL_INTERVAL}s (${Elapsed}s elapsed)"
        sleep "${CD_IMAGE_POLL_INTERVAL}"
        Elapsed=$(( Elapsed + CD_IMAGE_POLL_INTERVAL ))
    done
}

log "waiting for CI to publish the image (up to ${CD_IMAGE_WAIT_SECONDS}s)"
if [[ ${DRY_RUN} -eq 1 ]]; then
    log "dry run: skipping image wait"
elif ! wait_for_image; then
    notify fail "image ${CANDIDATE} never appeared (waited ${CD_IMAGE_WAIT_SECONDS}s) -- did CI fail?"
    die "image ${CANDIDATE} did not appear within ${CD_IMAGE_WAIT_SECONDS}s"
fi

# --------------------------------------------------- verify and pin digest ---

image_digest_ref() {
    docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$1" 2>/dev/null \
        | grep "^${CD_IMAGE_REPO}@" | head -n1
}

if [[ ${DRY_RUN} -eq 1 ]]; then
    DEPLOY_IMAGE="${CANDIDATE}"
    log "dry run: would deploy ${DEPLOY_IMAGE}"
else
    Revision="$(docker image inspect \
        --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
        "${CANDIDATE}" 2>/dev/null || true)"

    if [[ -z "${Revision}" || "${Revision}" == "<no value>" ]]; then
        warn "image carries no org.opencontainers.image.revision label; cannot prove provenance"
    elif [[ "${Revision}" != "${SHA}" ]]; then
        notify fail "image ${CANDIDATE} is built from ${Revision:0:12}, not ${SHA:0:12} -- refusing to deploy"
        die "provenance mismatch: ${CANDIDATE} declares revision ${Revision}, expected ${SHA}"
    else
        log "provenance  revision label matches ${SHA:0:12}"
    fi

    DEPLOY_IMAGE="$(image_digest_ref "${CANDIDATE}")"
    if [[ -z "${DEPLOY_IMAGE}" ]]; then
        warn "could not resolve a digest for ${CANDIDATE}; deploying by tag instead"
        DEPLOY_IMAGE="${CANDIDATE}"
    else
        log "pinned      ${DEPLOY_IMAGE}"
    fi
fi
readonly DEPLOY_IMAGE

# --------------------------------------------- remember what is running now ---

RunningImageId="$(docker inspect --format '{{.Image}}' "${CD_CONTAINER}" 2>/dev/null || true)"
PREVIOUS_IMAGE=""
PREVIOUS_REVISION=""
if [[ -n "${RunningImageId}" ]]; then
    PREVIOUS_IMAGE="$(image_digest_ref "${RunningImageId}")"
    PREVIOUS_REVISION="$(docker image inspect \
        --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
        "${RunningImageId}" 2>/dev/null || true)"
fi
readonly PREVIOUS_IMAGE PREVIOUS_REVISION

if [[ -n "${PREVIOUS_IMAGE}" ]]; then
    log "current     ${PREVIOUS_IMAGE} (revision ${PREVIOUS_REVISION:0:12})"
else
    log "current     (nothing running -- first deploy, or container absent)"
fi

# ----------------------------------------------------------------- deploy ---

compose_up() {
    local Image="$1"
    # Matches the manual runbook in the vault's "DomainDingo Development" note:
    # recreate exactly one service, leave its dependencies alone, and never run
    # `docker compose down` -- that causes avoidable downtime, and `down -v`
    # would destroy the bind-mounted data these stacks depend on.
    #
    # `--pull always` is deliberately absent: the image is already pinned to a
    # digest and pulled, so there is nothing left to resolve.
    ( cd "${CD_STACK_DIR}" \
      && env "${CD_IMAGE_ENV_VAR}=${Image}" \
             "${CD_PULL_POLICY_ENV_VAR}=never" \
         timeout "${CD_COMPOSE_TIMEOUT_SECONDS}" \
         docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \
             up -d --no-build --no-deps --force-recreate "${CD_COMPOSE_SERVICE}" )
}

compose_validate() {
    local Image="$1"
    ( cd "${CD_STACK_DIR}" \
      && env "${CD_IMAGE_ENV_VAR}=${Image}" \
             "${CD_PULL_POLICY_ENV_VAR}=never" \
         docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \
             config --quiet )
}

# Write the rollback record where the manual runbook looks for it, in the same
# format, so an operator following that note can recover from an automated
# deploy without knowing this tool exists.
write_rollback_record() {
    local Image="$1" Revision="$2" Dir="${CD_STACK_DIR}/${CD_ROLLBACK_RECORD_DIR}"
    local Record="${Dir}/${CD_COMPOSE_SERVICE}-rollback-$(date -u +%Y%m%d-%H%M%S).env"

    [[ -d "${Dir}" ]] || { warn "no rollback record directory at ${Dir}"; return 0; }

    ( umask 077
      printf 'DD_ROLLBACK_IMAGE=%s\nDD_ROLLBACK_REVISION=%s\nDD_ROLLBACK_CAPTURED_AT=%s\n' \
          "${Image}" "${Revision}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "${Record}" )
    log "rollback    recorded at ${Record}"
}

check_health() {
    local Attempt=1 Body
    while (( Attempt <= CD_HEALTH_RETRIES )); do
        Body="$(curl -fsS --max-time 5 "${CD_HEALTH_URL}" 2>/dev/null || true)"
        if [[ -n "${Body}" ]]; then
            if python3 -c '
import json, sys
key, want = sys.argv[1], sys.argv[2]
try:
    data = json.loads(sys.stdin.read())
except Exception:
    sys.exit(1)
sys.exit(0 if str(data.get(key, "")).lower() == want.lower() else 1)
' "${CD_HEALTH_EXPECT_KEY}" "${CD_HEALTH_EXPECT_VALUE}" <<<"${Body}"; then
                log "health      ok after ${Attempt} attempt(s): ${Body}"
                return 0
            fi
        fi
        log "health      not ready (attempt ${Attempt}/${CD_HEALTH_RETRIES}), retrying in ${CD_HEALTH_INTERVAL}s"
        sleep "${CD_HEALTH_INTERVAL}"
        (( Attempt++ ))
    done
    warn "health check never passed: last response was '${Body:-<no response>}'"
    return 1
}

if [[ ${DRY_RUN} -eq 1 ]]; then
    log "dry run: would recreate service ${CD_COMPOSE_SERVICE} with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}"
    log "dry run: would health-check ${CD_HEALTH_URL}"
    log "dry run complete -- no changes made"
    exit 0
fi

# Catch a broken compose file or a bad interpolation before the running
# container is touched.
if ! compose_validate "${DEPLOY_IMAGE}"; then
    notify fail "compose config is invalid for ${SHA:0:12} -- nothing was changed"
    die "docker compose config failed; the running stack was not touched"
fi

[[ -n "${PREVIOUS_IMAGE}" ]] && write_rollback_record "${PREVIOUS_IMAGE}" "${PREVIOUS_REVISION}"

log "deploying   ${DEPLOY_IMAGE}"
if ! compose_up "${DEPLOY_IMAGE}"; then
    notify fail "docker compose up failed for ${SHA:0:12} -- stack left as-is"
    die "docker compose up failed"
fi

if check_health; then
    log "=== deploy succeeded: ${CD_NAME} now runs ${SHA:0:12} ==="
    notify ok "deployed ${SHA:0:12} (${DEPLOY_IMAGE##*@})"
    exit 0
fi

# ---------------------------------------------------------------- rollback ---

if [[ "${CD_ROLLBACK_ON_FAILURE}" != "true" ]]; then
    notify fail "health check failed for ${SHA:0:12}; rollback disabled, stack left on the new image"
    die "health check failed and rollback is disabled for this target"
fi

if [[ -z "${PREVIOUS_IMAGE}" ]]; then
    notify fail "health check failed for ${SHA:0:12} and there is no previous image to roll back to"
    die "health check failed; no previous image recorded, leaving the stack as it is"
fi

warn "health check failed -- rolling back to ${PREVIOUS_IMAGE}"
if compose_up "${PREVIOUS_IMAGE}" && check_health; then
    notify fail "deploy of ${SHA:0:12} failed health check; rolled back to the previous image successfully"
    die "deploy failed health check; rolled back to ${PREVIOUS_IMAGE}"
fi

notify fail "deploy of ${SHA:0:12} failed AND rollback failed -- ${CD_NAME} needs manual attention now"
die "deploy failed and rollback also failed; manual intervention required"