Fix health endpoint, align with the documented manual runbook
The health router is mounted at prefix="/api" in domaindingo's main.py, so /health 404s. Checking it would have failed every deploy and rolled back a perfectly good image. Also adopts three conventions from the vault's DomainDingo Development note: recreate a single named service (--no-deps --force-recreate), validate with compose config --quiet before touching the running container, and write the rollback record in the documented format and location.
This commit is contained in:
fisher
+61
-18
@@ -236,14 +236,19 @@ readonly DEPLOY_IMAGE
|
||||
|
||||
# --------------------------------------------- remember what is running now ---
|
||||
|
||||
PREVIOUS_IMAGE="$(
|
||||
container_image="$(docker inspect --format '{{.Image}}' "$CD_CONTAINER" 2>/dev/null || true)"
|
||||
[[ -n "$container_image" ]] && image_digest_ref "$container_image" || true
|
||||
)"
|
||||
readonly PREVIOUS_IMAGE
|
||||
RunningImageId="$(docker inspect --format '{{.Image}}' "${CD_CONTAINER}" 2>/dev/null || true)"
|
||||
PREVIOUS_IMAGE=""
|
||||
PREVIOUS_REVISION=""
|
||||
if [[ -n "${RunningImageId}" ]]; then
|
||||
PREVIOUS_IMAGE="$(image_digest_ref "${RunningImageId}")"
|
||||
PREVIOUS_REVISION="$(docker image inspect \
|
||||
--format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \
|
||||
"${RunningImageId}" 2>/dev/null || true)"
|
||||
fi
|
||||
readonly PREVIOUS_IMAGE PREVIOUS_REVISION
|
||||
|
||||
if [[ -n "$PREVIOUS_IMAGE" ]]; then
|
||||
log "current ${PREVIOUS_IMAGE}"
|
||||
if [[ -n "${PREVIOUS_IMAGE}" ]]; then
|
||||
log "current ${PREVIOUS_IMAGE} (revision ${PREVIOUS_REVISION:0:12})"
|
||||
else
|
||||
log "current (nothing running -- first deploy, or container absent)"
|
||||
fi
|
||||
@@ -251,15 +256,44 @@ fi
|
||||
# ----------------------------------------------------------------- deploy ---
|
||||
|
||||
compose_up() {
|
||||
local image="$1"
|
||||
# `up -d` recreates only what actually changed. `down` is deliberately not
|
||||
# used here: it causes avoidable downtime, and `down -v` would destroy the
|
||||
# data volumes these stacks depend on.
|
||||
( cd "$CD_STACK_DIR" \
|
||||
&& env "${CD_IMAGE_ENV_VAR}=${image}" \
|
||||
"${CD_PULL_POLICY_ENV_VAR}=missing" \
|
||||
timeout "$CD_COMPOSE_TIMEOUT_SECONDS" \
|
||||
docker compose -f "$CD_COMPOSE_FILE" -p "$CD_COMPOSE_PROJECT" up -d --no-build )
|
||||
local Image="$1"
|
||||
# Matches the manual runbook in the vault's "DomainDingo Development" note:
|
||||
# recreate exactly one service, leave its dependencies alone, and never run
|
||||
# `docker compose down` -- that causes avoidable downtime, and `down -v`
|
||||
# would destroy the bind-mounted data these stacks depend on.
|
||||
#
|
||||
# `--pull always` is deliberately absent: the image is already pinned to a
|
||||
# digest and pulled, so there is nothing left to resolve.
|
||||
( cd "${CD_STACK_DIR}" \
|
||||
&& env "${CD_IMAGE_ENV_VAR}=${Image}" \
|
||||
"${CD_PULL_POLICY_ENV_VAR}=never" \
|
||||
timeout "${CD_COMPOSE_TIMEOUT_SECONDS}" \
|
||||
docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \
|
||||
up -d --no-build --no-deps --force-recreate "${CD_COMPOSE_SERVICE}" )
|
||||
}
|
||||
|
||||
compose_validate() {
|
||||
local Image="$1"
|
||||
( cd "${CD_STACK_DIR}" \
|
||||
&& env "${CD_IMAGE_ENV_VAR}=${Image}" \
|
||||
"${CD_PULL_POLICY_ENV_VAR}=never" \
|
||||
docker compose -f "${CD_COMPOSE_FILE}" -p "${CD_COMPOSE_PROJECT}" \
|
||||
config --quiet )
|
||||
}
|
||||
|
||||
# Write the rollback record where the manual runbook looks for it, in the same
|
||||
# format, so an operator following that note can recover from an automated
|
||||
# deploy without knowing this tool exists.
|
||||
write_rollback_record() {
|
||||
local Image="$1" Revision="$2" Dir="${CD_STACK_DIR}/${CD_ROLLBACK_RECORD_DIR}"
|
||||
local Record="${Dir}/${CD_COMPOSE_SERVICE}-rollback-$(date -u +%Y%m%d-%H%M%S).env"
|
||||
|
||||
[[ -d "${Dir}" ]] || { warn "no rollback record directory at ${Dir}"; return 0; }
|
||||
|
||||
( umask 077
|
||||
printf 'DD_ROLLBACK_IMAGE=%s\nDD_ROLLBACK_REVISION=%s\nDD_ROLLBACK_CAPTURED_AT=%s\n' \
|
||||
"${Image}" "${Revision}" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" > "${Record}" )
|
||||
log "rollback recorded at ${Record}"
|
||||
}
|
||||
|
||||
check_health() {
|
||||
@@ -289,14 +323,23 @@ sys.exit(0 if str(data.get(key, "")).lower() == want.lower() else 1)
|
||||
}
|
||||
|
||||
if [[ $DRY_RUN -eq 1 ]]; then
|
||||
log "dry run: would run docker compose up -d with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}"
|
||||
log "dry run: would recreate service ${CD_COMPOSE_SERVICE} with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}"
|
||||
log "dry run: would health-check ${CD_HEALTH_URL}"
|
||||
log "dry run complete -- no changes made"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Catch a broken compose file or a bad interpolation before the running
|
||||
# container is touched.
|
||||
if ! compose_validate "${DEPLOY_IMAGE}"; then
|
||||
notify fail "compose config is invalid for ${SHA:0:12} -- nothing was changed"
|
||||
die "docker compose config failed; the running stack was not touched"
|
||||
fi
|
||||
|
||||
[[ -n "${PREVIOUS_IMAGE}" ]] && write_rollback_record "${PREVIOUS_IMAGE}" "${PREVIOUS_REVISION}"
|
||||
|
||||
log "deploying ${DEPLOY_IMAGE}"
|
||||
if ! compose_up "$DEPLOY_IMAGE"; then
|
||||
if ! compose_up "${DEPLOY_IMAGE}"; then
|
||||
notify fail "docker compose up failed for ${SHA:0:12} -- stack left as-is"
|
||||
die "docker compose up failed"
|
||||
fi
|
||||
|
||||
@@ -37,3 +37,21 @@ Short notes on things that were not obvious. Prune stale ones.
|
||||
- **`docker compose down -v` destroys the data volumes** for these stacks. The
|
||||
retired paradicsomleves test script used it. `cd-deploy` only ever runs
|
||||
`up -d`.
|
||||
|
||||
- **domaindingo's health endpoint is `/api/health`, not `/health`.** The route is
|
||||
declared as `/health` in `src/domaindingo/api/status.py`, but `main.py` mounts
|
||||
the router with `prefix="/api"`. Checking `/health` gets a 404, which reads as
|
||||
a failed deploy and triggers a rollback. The vault's
|
||||
"DomainDingo Development" note is the authority here.
|
||||
|
||||
- **The manual runbook recreates one service, not the project.**
|
||||
`docker compose up -d --pull always --no-deps --force-recreate <service>`.
|
||||
`cd-deploy` matches it (minus `--pull always`, since it has already pulled a
|
||||
pinned digest), so an automated deploy and a hand-run one converge on the same
|
||||
state.
|
||||
|
||||
- **Rollback records are a documented convention.**
|
||||
`<service>-rollback-<UTC timestamp>.env` in `backup-test`/`backup-prod`,
|
||||
holding `DD_ROLLBACK_IMAGE`, `DD_ROLLBACK_REVISION`, `DD_ROLLBACK_CAPTURED_AT`.
|
||||
`cd-deploy` writes the same format in the same place, so an operator following
|
||||
the manual note can recover from an automated deploy.
|
||||
|
||||
+10
-21
@@ -1,6 +1,5 @@
|
||||
{
|
||||
"version": 1,
|
||||
|
||||
"_comment": [
|
||||
"Single source of truth for the fleet's continuous-deployment targets.",
|
||||
"A target is uniquely identified by (repo, branch). The 'host' field decides",
|
||||
@@ -10,20 +9,16 @@
|
||||
"Adding an environment is an edit here plus a re-run of install/install.sh on",
|
||||
"the owning host. It is never a change to the project repository."
|
||||
],
|
||||
|
||||
"defaults": {
|
||||
"ntfy_base_url": "https://ntfy.fisher.hu",
|
||||
|
||||
"image_wait_seconds": 900,
|
||||
"image_poll_interval": 15,
|
||||
|
||||
"health_retries": 30,
|
||||
"health_interval": 5,
|
||||
|
||||
"lock_wait_seconds": 600,
|
||||
"compose_timeout_seconds": 300
|
||||
"compose_timeout_seconds": 300,
|
||||
"rollback_record_dir": "."
|
||||
},
|
||||
|
||||
"hosts": {
|
||||
"s5.fisher.hu": {
|
||||
"bind": "10.255.255.1",
|
||||
@@ -34,7 +29,6 @@
|
||||
"port": 21600
|
||||
}
|
||||
},
|
||||
|
||||
"targets": [
|
||||
{
|
||||
"name": "domaindingo-test",
|
||||
@@ -43,24 +37,21 @@
|
||||
"branch": "test",
|
||||
"env": "test",
|
||||
"host": "s5.fisher.hu",
|
||||
|
||||
"stack_dir": "/home/fisher/S/uas-ng/docker/domaindingo/s5.fisher.hu",
|
||||
"compose_file": "docker-compose.yml",
|
||||
"compose_project": "domaindingo-test",
|
||||
"container": "domaindingo-test",
|
||||
|
||||
"compose_service": "domaindingo-test",
|
||||
"image_repo": "gitea.fisher.hu/webdev/domaindingo",
|
||||
"image_tag_template": "{branch}-sha-{short7}",
|
||||
"image_env_var": "DOMAINDINGO_TEST_IMAGE",
|
||||
"pull_policy_env_var": "DOMAINDINGO_TEST_PULL_POLICY",
|
||||
|
||||
"health_url": "http://127.0.0.1:9101/health",
|
||||
"health_url": "http://127.0.0.1:9101/api/health",
|
||||
"health_expect_key": "db",
|
||||
"health_expect_value": "ok",
|
||||
|
||||
"rollback_on_failure": true
|
||||
"rollback_on_failure": true,
|
||||
"rollback_record_dir": "backup-test"
|
||||
},
|
||||
|
||||
{
|
||||
"name": "domaindingo-prod",
|
||||
"enabled": true,
|
||||
@@ -68,22 +59,20 @@
|
||||
"branch": "prod",
|
||||
"env": "prod",
|
||||
"host": "s5.fisher.hu",
|
||||
|
||||
"stack_dir": "/home/fisher/S/uas-ng/docker/domaindingo/s5.fisher.hu",
|
||||
"compose_file": "docker-compose-prod.yml",
|
||||
"compose_project": "domaindingo-prod",
|
||||
"container": "domaindingo-prod",
|
||||
|
||||
"compose_service": "domaindingo-prod",
|
||||
"image_repo": "gitea.fisher.hu/webdev/domaindingo",
|
||||
"image_tag_template": "{branch}-sha-{short7}",
|
||||
"image_env_var": "DOMAINDINGO_PROD_IMAGE",
|
||||
"pull_policy_env_var": "DOMAINDINGO_PROD_PULL_POLICY",
|
||||
|
||||
"health_url": "http://127.0.0.1:9103/health",
|
||||
"health_url": "http://127.0.0.1:9103/api/health",
|
||||
"health_expect_key": "db",
|
||||
"health_expect_value": "ok",
|
||||
|
||||
"rollback_on_failure": true
|
||||
"rollback_on_failure": true,
|
||||
"rollback_record_dir": "backup-prod"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user