From 952de1ea8e41e6f8d2357993b07ef7c44bf69fbf Mon Sep 17 00:00:00 2001 From: fisher Date: Sun, 23 Aug 2026 08:53:26 +0000 Subject: [PATCH] Fix health endpoint, align with the documented manual runbook The health router is mounted at prefix="/api" in domaindingo's main.py, so /health 404s. Checking it would have failed every deploy and rolled back a perfectly good image. Also adopts three conventions from the vault's DomainDingo Development note: recreate a single named service (--no-deps --force-recreate), validate with compose config --quiet before touching the running container, and write the rollback record in the documented format and location. --- bin/cd-deploy | 79 +++++++++++++++++++++++++++++++++++++++------------ docs/hints.md | 18 ++++++++++++ targets.json | 31 +++++++------------- 3 files changed, 89 insertions(+), 39 deletions(-) diff --git a/bin/cd-deploy b/bin/cd-deploy index 1e9ae50..6be151c 100755 --- a/bin/cd-deploy +++ b/bin/cd-deploy @@ -236,14 +236,19 @@ readonly DEPLOY_IMAGE # --------------------------------------------- remember what is running now --- -PREVIOUS_IMAGE="$( - container_image="$(docker inspect --format '{{.Image}}' "$CD_CONTAINER" 2>/dev/null || true)" - [[ -n "$container_image" ]] && image_digest_ref "$container_image" || true -)" -readonly PREVIOUS_IMAGE +RunningImageId="$(docker inspect --format '{{.Image}}' "${CD_CONTAINER}" 2>/dev/null || true)" +PREVIOUS_IMAGE="" +PREVIOUS_REVISION="" +if [[ -n "${RunningImageId}" ]]; then + PREVIOUS_IMAGE="$(image_digest_ref "${RunningImageId}")" + PREVIOUS_REVISION="$(docker image inspect \ + --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \ + "${RunningImageId}" 2>/dev/null || true)" +fi +readonly PREVIOUS_IMAGE PREVIOUS_REVISION -if [[ -n "$PREVIOUS_IMAGE" ]]; then - log "current ${PREVIOUS_IMAGE}" +if [[ -n "${PREVIOUS_IMAGE}" ]]; then + log "current ${PREVIOUS_IMAGE} (revision ${PREVIOUS_REVISION:0:12})" else log "current (nothing running -- first deploy, or container absent)" fi @@ -251,15 +256,44 @@ fi # ----------------------------------------------------------------- deploy --- compose_up() { - local image="$1" - # `up -d` recreates only what actually changed. `down` is deliberately not - # used here: it causes avoidable downtime, and `down -v` would destroy the - # data volumes these stacks depend on. - ( cd "$CD_STACK_DIR" \ - && env "${CD_IMAGE_ENV_VAR}=${image}" \ - "${CD_PULL_POLICY_ENV_VAR}=missing" \ - timeout "$CD_COMPOSE_TIMEOUT_SECONDS" \ - docker compose -f "$CD_COMPOSE_FILE" -p "$CD_COMPOSE_PROJECT" up -d --no-build ) + local Image="$1" + # Matches the manual runbook in the vault's "DomainDingo Development" note: + # recreate exactly one service, leave its dependencies alone, and never run + # `docker compose down` -- that causes avoidable downtime, and `down -v` + # would destroy the bind-mounted data these stacks depend on. + # + # `--pull always` is deliberately absent: the image is already pinned to a + # digest and pulled, so there is nothing left to resolve. + ( cd "${CD_STACK_DIR}" \ + && env "${CD_IMAGE_ENV_VAR}=${Image}" \ + "${CD_PULL_POLICY_ENV_VAR}=never" \ + timeout "${CD_COMPOSE_TIMEOUT_SECONDS}" \ + docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \ + up -d --no-build --no-deps --force-recreate "${CD_COMPOSE_SERVICE}" ) +} + +compose_validate() { + local Image="$1" + ( cd "${CD_STACK_DIR}" \ + && env "${CD_IMAGE_ENV_VAR}=${Image}" \ + "${CD_PULL_POLICY_ENV_VAR}=never" \ + docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \ + config --quiet ) +} + +# Write the rollback record where the manual runbook looks for it, in the same +# format, so an operator following that note can recover from an automated +# deploy without knowing this tool exists. +write_rollback_record() { + local Image="$1" Revision="$2" Dir="${CD_STACK_DIR}/${CD_ROLLBACK_RECORD_DIR}" + local Record="${Dir}/${CD_COMPOSE_SERVICE}-rollback-$(date -u +%Y%m%d-%H%M%S).env" + + [[ -d "${Dir}" ]] || { warn "no rollback record directory at ${Dir}"; return 0; } + + ( umask 077 + printf 'DD_ROLLBACK_IMAGE=%s\nDD_ROLLBACK_REVISION=%s\nDD_ROLLBACK_CAPTURED_AT=%s\n' \ + "${Image}" "${Revision}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "${Record}" ) + log "rollback recorded at ${Record}" } check_health() { @@ -289,14 +323,23 @@ sys.exit(0 if str(data.get(key, "")).lower() == want.lower() else 1) } if [[ $DRY_RUN -eq 1 ]]; then - log "dry run: would run docker compose up -d with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}" + log "dry run: would recreate service ${CD_COMPOSE_SERVICE} with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}" log "dry run: would health-check ${CD_HEALTH_URL}" log "dry run complete -- no changes made" exit 0 fi +# Catch a broken compose file or a bad interpolation before the running +# container is touched. +if ! compose_validate "${DEPLOY_IMAGE}"; then + notify fail "compose config is invalid for ${SHA:0:12} -- nothing was changed" + die "docker compose config failed; the running stack was not touched" +fi + +[[ -n "${PREVIOUS_IMAGE}" ]] && write_rollback_record "${PREVIOUS_IMAGE}" "${PREVIOUS_REVISION}" + log "deploying ${DEPLOY_IMAGE}" -if ! compose_up "$DEPLOY_IMAGE"; then +if ! compose_up "${DEPLOY_IMAGE}"; then notify fail "docker compose up failed for ${SHA:0:12} -- stack left as-is" die "docker compose up failed" fi diff --git a/docs/hints.md b/docs/hints.md index 65d2851..fdac58e 100644 --- a/docs/hints.md +++ b/docs/hints.md @@ -37,3 +37,21 @@ Short notes on things that were not obvious. Prune stale ones. - **`docker compose down -v` destroys the data volumes** for these stacks. The retired paradicsomleves test script used it. `cd-deploy` only ever runs `up -d`. + +- **domaindingo's health endpoint is `/api/health`, not `/health`.** The route is + declared as `/health` in `src/domaindingo/api/status.py`, but `main.py` mounts + the router with `prefix="/api"`. Checking `/health` gets a 404, which reads as + a failed deploy and triggers a rollback. The vault's + "DomainDingo Development" note is the authority here. + +- **The manual runbook recreates one service, not the project.** + `docker compose up -d --pull always --no-deps --force-recreate `. + `cd-deploy` matches it (minus `--pull always`, since it has already pulled a + pinned digest), so an automated deploy and a hand-run one converge on the same + state. + +- **Rollback records are a documented convention.** + `-rollback-.env` in `backup-test`/`backup-prod`, + holding `DD_ROLLBACK_IMAGE`, `DD_ROLLBACK_REVISION`, `DD_ROLLBACK_CAPTURED_AT`. + `cd-deploy` writes the same format in the same place, so an operator following + the manual note can recover from an automated deploy. diff --git a/targets.json b/targets.json index 0244c85..7a479c8 100644 --- a/targets.json +++ b/targets.json @@ -1,6 +1,5 @@ { "version": 1, - "_comment": [ "Single source of truth for the fleet's continuous-deployment targets.", "A target is uniquely identified by (repo, branch). The 'host' field decides", @@ -10,20 +9,16 @@ "Adding an environment is an edit here plus a re-run of install/install.sh on", "the owning host. It is never a change to the project repository." ], - "defaults": { "ntfy_base_url": "https://ntfy.fisher.hu", - "image_wait_seconds": 900, "image_poll_interval": 15, - "health_retries": 30, "health_interval": 5, - "lock_wait_seconds": 600, - "compose_timeout_seconds": 300 + "compose_timeout_seconds": 300, + "rollback_record_dir": "." }, - "hosts": { "s5.fisher.hu": { "bind": "10.255.255.1", @@ -34,7 +29,6 @@ "port": 21600 } }, - "targets": [ { "name": "domaindingo-test", @@ -43,24 +37,21 @@ "branch": "test", "env": "test", "host": "s5.fisher.hu", - "stack_dir": "/home/fisher/S/uas-ng/docker/domaindingo/s5.fisher.hu", "compose_file": "docker-compose.yml", "compose_project": "domaindingo-test", "container": "domaindingo-test", - + "compose_service": "domaindingo-test", "image_repo": "gitea.fisher.hu/webdev/domaindingo", "image_tag_template": "{branch}-sha-{short7}", "image_env_var": "DOMAINDINGO_TEST_IMAGE", "pull_policy_env_var": "DOMAINDINGO_TEST_PULL_POLICY", - - "health_url": "http://127.0.0.1:9101/health", + "health_url": "http://127.0.0.1:9101/api/health", "health_expect_key": "db", "health_expect_value": "ok", - - "rollback_on_failure": true + "rollback_on_failure": true, + "rollback_record_dir": "backup-test" }, - { "name": "domaindingo-prod", "enabled": true, @@ -68,22 +59,20 @@ "branch": "prod", "env": "prod", "host": "s5.fisher.hu", - "stack_dir": "/home/fisher/S/uas-ng/docker/domaindingo/s5.fisher.hu", "compose_file": "docker-compose-prod.yml", "compose_project": "domaindingo-prod", "container": "domaindingo-prod", - + "compose_service": "domaindingo-prod", "image_repo": "gitea.fisher.hu/webdev/domaindingo", "image_tag_template": "{branch}-sha-{short7}", "image_env_var": "DOMAINDINGO_PROD_IMAGE", "pull_policy_env_var": "DOMAINDINGO_PROD_PULL_POLICY", - - "health_url": "http://127.0.0.1:9103/health", + "health_url": "http://127.0.0.1:9103/api/health", "health_expect_key": "db", "health_expect_value": "ok", - - "rollback_on_failure": true + "rollback_on_failure": true, + "rollback_record_dir": "backup-prod" } ] }