From 3a6fafb5c7c95db82b8f0f382fc1b1f7312a3259 Mon Sep 17 00:00:00 2001 From: fisher Date: Sun, 23 Aug 2026 07:49:52 +0000 Subject: [PATCH] Generic host/repo/branch-aware CD on adnanh/webhook Replaces the per-project deploy scripts and self-contained webhook receivers with one manifest-driven implementation that lives outside the application repositories and can be updated independently of them. First targets: domaindingo test and prod on s5. --- .github/copilot-instructions.md | 12 + .gitignore | 5 + AGENTS.md | 12 + CLAUDE.md | 12 + README.md | 174 ++++++++++ ai-context/infrastructure/git-instructions.md | 57 ++++ .../infrastructure/gitea-to-github-release.md | 110 ++++++ .../rootless-docker-gitea-runner.md | 95 ++++++ ai-context/services/cloudflare-r2.md | 58 ++++ ai-context/services/ntfy.md | 48 +++ ai-context/style/working-style.md | 32 ++ bin/cd-deploy | 319 ++++++++++++++++++ bin/cd-render-hooks | 237 +++++++++++++ bin/cd-status | 71 ++++ bin/cd-target | 183 ++++++++++ docs/OPERATIONS.md | 124 +++++++ docs/active-context.md | 17 + etc/cd-webhook.service | 33 ++ install/install.sh | 135 ++++++++ install/uninstall.sh | 36 ++ secrets.env.example | 12 + targets.json | 90 +++++ 22 files changed, 1872 insertions(+) create mode 100644 .github/copilot-instructions.md create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 README.md create mode 100644 ai-context/infrastructure/git-instructions.md create mode 100644 ai-context/infrastructure/gitea-to-github-release.md create mode 100644 ai-context/infrastructure/rootless-docker-gitea-runner.md create mode 100644 ai-context/services/cloudflare-r2.md create mode 100644 ai-context/services/ntfy.md create mode 100644 ai-context/style/working-style.md create mode 100755 bin/cd-deploy create mode 100755 bin/cd-render-hooks create mode 100755 bin/cd-status create mode 100755 bin/cd-target create mode 100644 docs/OPERATIONS.md create mode 100644 docs/active-context.md create mode 100644 etc/cd-webhook.service create mode 100755 install/install.sh create mode 100755 install/uninstall.sh create mode 100644 secrets.env.example create mode 100644 targets.json diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..9b1d1b7 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,12 @@ +# Agent instructions + +This project's knowledge base lives in `ai-context/` and `docs/`. **Do not read +it preemptively.** When a task may touch project infrastructure, services, or +repository operations, consult the routing index `docs/active-context.md` and +open an entry only if its trigger matches your task. + +Files under `ai-context/` are read-only — the install script overwrites them on +update, so local edits are lost. + +Record project-specific discoveries (gotchas, quirks, non-obvious tool +behaviour) as short entries in `docs/hints.md`; prune stale ones. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..41cd5d2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# Rendered artefacts and anything host-local. The repository holds the manifest +# and the scripts; secrets and generated hooks belong only on the host. +hooks.json +secrets.env +*.log diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..9b1d1b7 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,12 @@ +# Agent instructions + +This project's knowledge base lives in `ai-context/` and `docs/`. **Do not read +it preemptively.** When a task may touch project infrastructure, services, or +repository operations, consult the routing index `docs/active-context.md` and +open an entry only if its trigger matches your task. + +Files under `ai-context/` are read-only — the install script overwrites them on +update, so local edits are lost. + +Record project-specific discoveries (gotchas, quirks, non-obvious tool +behaviour) as short entries in `docs/hints.md`; prune stale ones. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..9b1d1b7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,12 @@ +# Agent instructions + +This project's knowledge base lives in `ai-context/` and `docs/`. **Do not read +it preemptively.** When a task may touch project infrastructure, services, or +repository operations, consult the routing index `docs/active-context.md` and +open an entry only if its trigger matches your task. + +Files under `ai-context/` are read-only — the install script overwrites them on +update, so local edits are lost. + +Record project-specific discoveries (gotchas, quirks, non-obvious tool +behaviour) as short entries in `docs/hints.md`; prune stale ones. diff --git a/README.md b/README.md new file mode 100644 index 0000000..d7c0bb2 --- /dev/null +++ b/README.md @@ -0,0 +1,174 @@ +# my-cd-webhook + +Fleet-wide continuous deployment built on [adnanh/webhook](https://github.com/adnanh/webhook), +replacing the per-project deploy scripts and hand-rolled webhook receivers that +used to live inside each application repository. + +This repository is checked out on every deploying host (currently `s4` and `s5`) +and updated independently of the projects it deploys. + +## Why this exists + +Every project had grown its own deployment mechanism, and each one was a +slightly different 200-line bash script plus, in some cases, a bespoke Python +HTTP server. They shared no code, drifted apart, and could only be changed by +committing to the application repository and redeploying it — which is exactly +the thing that is hardest to do when deployment is broken. + +Concretely, the patterns this replaces: + +| Project | What was there | Problem | +|---|---|---| +| `webdev/paradicsomleves` | `scripts/webhook.py` (275 lines), `webhook-install.sh`, `webhook-uninstall.sh` | A whole HTTP server, HMAC implementation, systemd watchdog integration and unit generator, maintained inside an image-gallery app. Removed in `660a309`. | +| `webdev/domaindingo` | `scripts/deploy-{dev,test,prod}.sh` | Three near-identical scripts, run by hand. A `hostname != s2` guard that `exit 0`s on mismatch — so a wrong-host run *reports success* — and which is now simply wrong, because test and prod both moved to s5. `deploy-prod.sh` has no host guard at all. | +| `webdev/grindex` | nothing | The pipeline stops at "image pushed"; deployment is a human running `docker compose up -d` on s4. | + +The webhook was never the problem. A *self-contained* webhook, reimplemented +per project, was. + +## Design + +**One manifest.** [`targets.json`](targets.json) maps `(repo, branch)` onto a +host, an environment, a compose stack and an image. It is the only file that +knows this, and adding an environment is an edit here — never a change to an +application repository. + +**Host-aware by construction.** Every host runs the same daemon from the same +checkout and acts only on the targets whose `host` matches. A push for another +host's target is a logged no-op, never a silent success. Host matching accepts +both `s5` and `s5.fisher.hu`, because the fleet uses both. + +**Deploys are immutable and provenance-checked.** The receiver deploys the +`-sha-` tag, verifies the image's +`org.opencontainers.image.revision` label equals the commit that triggered it, +then re-pins to the resolved digest before handing it to compose. Mutable tags +have already drifted once in this fleet and served a broken build for months. + +**It never touches git.** The compose stacks live in `sysadmin/uas-ng` and are +refreshed by that repository's own updater timer. A deploy script that also runs +`git reset --hard` is doing two jobs badly. + +**It never runs `docker compose down`.** `up -d` recreates exactly the services +whose image changed. The old test script used `down -v`, which destroys data +volumes. + +**Secrets stay on the host.** `secrets.env` (mode 0600) holds one webhook secret +per repository. The rendered hooks file is generated on the host and is never +committed. + +## Layout + +``` +targets.json the manifest — the single source of truth +bin/cd-target manifest queries (resolve / list / host-config) +bin/cd-deploy the one generic deploy script +bin/cd-render-hooks targets.json -> adnanh/webhook hooks.json for this host +bin/cd-status what is this host deploying, and is it healthy +etc/cd-webhook.service systemd user unit template +install/install.sh idempotent installer +install/uninstall.sh removal (--purge also drops secrets and logs) +secrets.env.example template for the host-local secrets file +``` + +## How a deploy runs + +``` +git push origin test + │ + ▼ +Gitea push webhook ──► http://10.255.255.1:20090/hooks/webdev-domaindingo + │ (WireGuard only; never exposed publicly) + ▼ +adnanh/webhook verify HMAC-SHA256, require X-Gitea-Event: push, + require a ref we actually deploy + │ + ▼ +bin/cd-deploy --repo webdev/domaindingo --ref refs/heads/test --sha <40-hex> + │ + ├─ resolve (repo, branch, this host) in targets.json ──► no match? log, exit 0 + ├─ take the per-target lock + ├─ poll the registry for -sha- (CI is still building) + ├─ verify the revision label matches the pushed commit + ├─ resolve tag -> digest, and deploy the digest + ├─ docker compose up -d + ├─ health check + └─ roll back to the previous digest if health fails + │ + ▼ + ntfy +``` + +## Installing on a host + +Requires `adnanh/webhook` on `PATH` (or `WEBHOOK_BIN` set), plus `docker`, +`python3`, `curl` and `flock`. + +```bash +git clone ssh://git@gitea.fisher.hu:2221/sysadmin/my-cd-webhook.git ~/S/my-cd-webhook +cd ~/S/my-cd-webhook +./install/install.sh # creates the secrets template on the first run +$EDITOR ~/.config/cd-webhook/secrets.env +./install/install.sh # renders hooks, installs and starts the service +``` + +The installer is idempotent — re-run it after editing `targets.json`, rotating a +secret, or pulling a new version of this repository. + +It prints the exact webhook URL for each repository at the end. In Gitea, under +**Repository → Settings → Webhooks → Add Webhook → Gitea**: + +| Field | Value | +|---|---| +| Target URL | `http://:20090/hooks/` | +| HTTP Method | `POST` | +| POST Content Type | `application/json` | +| Secret | the matching value from `secrets.env` | +| Trigger On | Push Events | +| Branch filter | `*` (branch selection lives in `targets.json`) | + +## Adding a target + +Add an object to `targets.json` and re-run `install/install.sh` on the owning +host. Nothing else. If the repository is new to that host, add its secret to +`secrets.env` first and configure the webhook in Gitea. + +Fields: `name`, `enabled`, `repo`, `branch`, `env`, `host`, `stack_dir`, +`compose_file`, `compose_project`, `container`, `image_repo`, +`image_tag_template`, `image_env_var`, `pull_policy_env_var`, `health_url`, +`health_expect_key`, `health_expect_value`, `rollback_on_failure`. + +`image_tag_template` understands `{branch}`, `{env}`, `{sha}`, `{short7}` and +`{short12}`. + +## Current targets + +| Target | Repo | Branch | Host | Route | +|---|---|---|---|---| +| `domaindingo-test` | `webdev/domaindingo` | `test` | s5 | `ddt.fisher.hu` | +| `domaindingo-prod` | `webdev/domaindingo` | `prod` | s5 | `dd.fisher.hu` | + +## Testing without deploying + +```bash +./bin/cd-status # health of this host's receiver +./bin/cd-target list --host s5.fisher.hu # what s5 would deploy +./bin/cd-render-hooks --host s5.fisher.hu --redact # the hooks file, secrets masked +./bin/cd-deploy --repo webdev/domaindingo --ref refs/heads/test \ + --sha $(git rev-parse HEAD) --dry-run +``` + +`CD_TARGETS_FILE` points the tooling at a scratch manifest for testing. It +changes which manifest is read, never which host this machine may act as. + +## Not yet done + +- **Port `20090` is not yet in the Port registry.** Register it before this goes + live on a second host. +- **Gitea (s5) must be able to reach s4 on `10.255.255.12:20090`** over + WireGuard. Verify before adding grindex. +- **grindex is deliberately absent from the manifest.** Its stack lives at + `/home/fisher/S/traefik-systems/grindex/` on s4, which is not in git and whose + compose project names have not been verified on the host. Adding it from + guesswork would be exactly the sloppiness this repo exists to remove. +- **`webdev/domaindingo` still carries `scripts/deploy-*.sh`.** They should be + deleted there once this is live, the way paradicsomleves' were. diff --git a/ai-context/infrastructure/git-instructions.md b/ai-context/infrastructure/git-instructions.md new file mode 100644 index 0000000..76ab537 --- /dev/null +++ b/ai-context/infrastructure/git-instructions.md @@ -0,0 +1,57 @@ +# Gitea workflow + +## Branch flow + +`main` → `test` → `dev` → feature branches. + +`test` is the default branch, protected, merge-only. + +For a brand-new empty repo: create `test` from `main`, then `dev` from `test`. + +## Posting with `tea` + +When posting to Gitea with `tea` (issues, comments, PRs): + +- Run it in the **foreground**. +- **Use the right body argument** — there is no `--body` flag and no + read-from-file option: + - `tea issues create` / `tea pulls create`: body is `--description` / `-d` + - `tea comment []`: body is a **positional** argument (no flag) +- **Never put `\n` escape sequences in the body.** `tea` stores the string + verbatim — it does not interpret escapes, so any `\n` you pass appears + *literally* in the posted text. The body must already contain real newlines: + write it to a file (the Write tool, or `printf` — never `echo "...\n..."`), + then pass it by command substitution, e.g. `-d "$(cat body.md)"` or + `tea comment 42 "$(cat body.md)"`. Inline heredocs and `\n`-laden argument + strings are what cause the literal `\n`. +- **Confirm it actually posted** — check the exit code *and* re-read the created + item; verify the body renders with real line breaks, not literal `\n`. `tea` + can fail silently. +- **Write subcommands hang on inherited stdin.** Even with the body passed as an + argument, `tea` (`comment`, `issues edit`, `issues close`, `pulls create`, …) + reads stdin; in a non-TTY shell the inherited stdin never sends EOF, so the + command blocks until it is killed. **Fix: redirect stdin — append `` (note: `tea issues view ` + does *not* print the full body). +- **Show comments:** `tea issue --comments`. Comments are hidden by default, + but this flag renders them — you do **not** need the API to read comments. +- **Edit an existing issue body:** `tea issues edit -d "$(cat body.md)" + -public + HostName github.com + User git + IdentityFile ~/.ssh/id_ed25519_-public + IdentitiesOnly yes +``` + +Remote becomes `github--public:OWNER/REPO.git` — no URL/user/password at +release time, key scoped to one repo, `IdentitiesOnly yes` stops SSH offering +unrelated keys. `IdentityFile` may point at the `.pub` when the private key is in +`ssh-agent` (agent signs); otherwise point at the private key. The release script +should check the alias exists before pushing, failing early with guidance. + +Once the alias and deploy key are in place for the project, remove this note's +entry from `docs/active-context.md` — the setup is one-time and shouldn't keep +occupying agent context. + +## Release procedure + +Keep `$REL_REPO` as a separate local folder from the dev clone. Each release: + +1. Pick + verify `$RELEASE_REF` in Gitea (tests/CI green). +2. Read `$VERSION` from package metadata. +3. Rebuild `$REL_REPO`'s tracked content from the allow-list at `$RELEASE_REF` + via `git archive` (committed files only). +4. Commit as a single `Release $VERSION`, tag `v$VERSION` — all local. +5. Optionally push to GitHub once `$REL_REMOTE` exists, and optionally create a + GitHub Release. PyPI publishing, if any, is a separate explicit step. + +### Allow-list export (sketch) + +Keep the allow-list in the script, reviewed like code: + +```bash +#!/usr/bin/env bash +set -euo pipefail +DEV_REF="${1:-main}" +STAGE="$(mktemp -d)" + +PATHS=( # Anything not listed does NOT ship. + src pyproject.toml README.md LICENSE + examples/app.service examples/config.example.yaml examples/content.example.yaml +) + +git archive "$DEV_REF" -- "${PATHS[@]}" | tar -x -C "$STAGE" # committed only + +# Guard: refuse if a non-example file slipped into examples/ +if find "$STAGE/examples" -type f ! -name '*.example.*' ! -name '*.service' | grep -q .; then + echo "Refusing: non-example file in examples/ — possible private data" >&2 + exit 1 +fi + +# Sync into $REL_REPO, preserving its .git, then commit/tag (push later): +# rsync -a --delete --exclude='.git' "$STAGE"/ "$REL_REPO"/ +# git -C "$REL_REPO" add -A +# git -C "$REL_REPO" commit -m "Release $VERSION" +# git -C "$REL_REPO" tag "v$VERSION" +# git -C "$REL_REPO" push -u --follow-tags origin main # once a remote exists +``` diff --git a/ai-context/infrastructure/rootless-docker-gitea-runner.md b/ai-context/infrastructure/rootless-docker-gitea-runner.md new file mode 100644 index 0000000..d784c3c --- /dev/null +++ b/ai-context/infrastructure/rootless-docker-gitea-runner.md @@ -0,0 +1,95 @@ +# Rootless Docker Gitea Runner Notes + +Reusable setup for running Gitea Actions jobs on a **rootless** Docker host when +jobs need Docker access (Playwright, Buildx). The runner runs as a container; +job containers reach the host's rootless socket. Substitute the rootless socket +path `/run/user//docker.sock` for your runner user (examples use `3100`). + +## Runner compose + +Mount the rootless socket into the runner container: + +```yaml +services: + gitea-runner-1: + image: gitea/act_runner:latest + container_name: s4-runner-1 + restart: unless-stopped + environment: + GITEA_INSTANCE_URL: https://gitea.example.com + GITEA_RUNNER_REGISTRATION_TOKEN: ${GITEA_RUNNER_REGISTRATION_TOKEN} + GITEA_RUNNER_NAME: s4-runner-1 + CONFIG_FILE: /data/config.yaml + volumes: + - runner1_data:/data + - ./config.yaml:/data/config.yaml:ro + - /run/user/3100/docker.sock:/var/run/docker.sock +volumes: + runner1_data: +``` + +## Runner config + +```yaml +container: + docker_host: "-" # critical + options: "-v /run/user/3100/docker.sock:/var/run/docker.sock" + valid_volumes: + - /run/user/3100/docker.sock # source path only +``` + +`docker_host: "-"` stops `act_runner` from adding its own socket mount on top of +the one in `options`, which otherwise fails job creation with +`Duplicate mount point: /var/run/docker.sock`. Keep `valid_volumes` as the host +source path only (not the `src:dst` bind form). + +## Workflow patterns + +- **Buildx**: use `docker/setup-buildx-action@v3` with `driver: docker`. The + default driver starts a separate BuildKit container that misbehaves on this + runner shape; the `docker` driver uses the mounted daemon directly. +- **No setup-node cache**: drop `cache: pnpm` from `actions/setup-node@v4`; the + job network often can't reach the Gitea Actions cache service (minutes of + restore/save timeouts). Use `pnpm install --frozen-lockfile` instead. Revisit + only once the cache-service network path is confirmed. +- **Playwright**: split browser E2E from validation — build + upload artifact in + a normal job, then run Playwright in the official image + (`mcr.microsoft.com/playwright:v1.58.2-noble`) against the downloaded artifact. + +## Host checks + +```sh +docker context show; echo "$DOCKER_HOST"; ls -l /run/user/3100/docker.sock; docker info +# DOCKER_HOST should be unix:///run/user/3100/docker.sock +docker run --rm -v /run/user/3100/docker.sock:/var/run/docker.sock docker:27-cli docker version +``` + +If that container can't use the socket, fix rootless Docker before touching the +runner. + +## Troubleshooting + +- **`Duplicate mount point: /var/run/docker.sock`** — set `docker_host: "-"`, + keep the bind only in `options`, `valid_volumes` = source path only, recreate. +- **`WARNING: IPv4 forwarding is disabled`** may persist even after host + `net.ipv4.ip_forward=1`. `systemctl --user restart docker` and re-check + `docker info`. If jobs create containers and reach their networks, treat it as + stale output, not a failure. +- **Buildx/build fails** — use `driver: docker`; confirm `docker version` works + through the mounted socket. + +## Useful commands + +```sh +docker ps --format '{{.ID}} {{.Image}} {{.Status}} {{.Names}}' +docker logs --tail 120 s4-runner-1 +docker compose up -d --force-recreate gitea-runner-1 +tea actions runs --repo owner/repo +tea actions runs logs --job --repo owner/repo +``` + +## Known-good (Grindex / current runner) + +Host `s4.fisher.hu`; container `s4-runner-1`; image `gitea/act_runner:latest`; +socket `/run/user/3100/docker.sock`; workflows in `.gitea/workflows`; Buildx +`driver: docker`; setup-node cache disabled. diff --git a/ai-context/services/cloudflare-r2.md b/ai-context/services/cloudflare-r2.md new file mode 100644 index 0000000..67aa729 --- /dev/null +++ b/ai-context/services/cloudflare-r2.md @@ -0,0 +1,58 @@ +# Cloudflare R2 CI Setup + +Upload a build artifact (e.g. a status JSON) to an R2 bucket from CI, using +`awscli` against R2's S3-compatible API. + +## Cloudflare side + +Create a bucket and a write-capable API token/key pair; note the account ID. + +## CI config + +Variables: `R2_BUCKET_NAME`, `R2_ACCOUNT_ID`. +Secrets: `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`. + +The keys must be passed as `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` — +`awscli` reads those names. Also required: `AWS_DEFAULT_REGION=auto` and +endpoint `https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com`. + +## Job + +```yaml +publish_status: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: { python-version: "3.12" } + - run: pip install awscli + - name: Publish to R2 + env: + R2_BUCKET_NAME: ${{ vars.R2_BUCKET_NAME }} + R2_ACCOUNT_ID: ${{ vars.R2_ACCOUNT_ID }} + AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} + run: | + set -eu + # If publish is optional, skip on missing config; use exit 1 if mandatory. + [ -n "${R2_BUCKET_NAME:-}" ] && [ -n "${R2_ACCOUNT_ID:-}" ] \ + && [ -n "${AWS_ACCESS_KEY_ID:-}" ] && [ -n "${AWS_SECRET_ACCESS_KEY:-}" ] \ + || { echo "Skipping: R2 config not set."; exit 0; } + f="$(mktemp -d)/project-name.json" + cat > "$f" <-sha- tag, then re-pins to the +# resolved digest. Mutable tags like :latest and :test have already drifted +# once in this fleet and served a broken build. +# * It verifies the image's org.opencontainers.image.revision label equals the +# commit that triggered the deploy, before changing anything. +# * It never runs `docker compose down`, and most emphatically never +# `down -v` -- `up -d` recreates exactly the services whose image changed. +# * A target belonging to another host is a clean no-op, logged as such, and +# never reported as a success. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly SCRIPT_DIR +readonly CD_TARGET="${SCRIPT_DIR}/cd-target" + +readonly STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/cd-webhook" +readonly NO_MATCH=3 + +REPO="" +REF="" +SHA="" +EVENT="push" +DRY_RUN=0 + +# ---------------------------------------------------------------- logging --- + +log() { printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } +warn() { log "WARN $*" >&2; } +die() { log "ERROR $*" >&2; exit 1; } + +usage() { + sed -n '3,30p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit "${1:-0}" +} + +# ------------------------------------------------------------ arg parsing --- + +while [[ $# -gt 0 ]]; do + case "$1" in + --repo) REPO="${2:?--repo needs a value}"; shift 2 ;; + --ref) REF="${2:?--ref needs a value}"; shift 2 ;; + --sha) SHA="${2:?--sha needs a value}"; shift 2 ;; + --event) EVENT="${2:?--event needs a value}"; shift 2 ;; + --dry-run) DRY_RUN=1; shift ;; + -h|--help) usage 0 ;; + *) die "unknown argument: $1 (try --help)" ;; + esac +done + +[[ -n "$REPO" ]] || die "--repo is required" +[[ -n "$REF" ]] || die "--ref is required" +[[ -n "$SHA" ]] || die "--sha is required" + +if [[ "$EVENT" != "push" ]]; then + log "ignoring event '${EVENT}' for ${REPO} (only 'push' deploys)" + exit 0 +fi + +if [[ "$REF" != refs/heads/* ]]; then + log "ignoring non-branch ref '${REF}' for ${REPO} (tags and deletes never deploy)" + exit 0 +fi +BRANCH="${REF#refs/heads/}" + +if [[ ! "$SHA" =~ ^[0-9a-f]{40}$ ]]; then + die "--sha must be a full 40-character hex commit, got: ${SHA}" +fi + +# The all-zero SHA is how git spells "this ref was deleted". +if [[ "$SHA" == "0000000000000000000000000000000000000000" ]]; then + log "ignoring branch deletion of ${REPO}@${BRANCH}" + exit 0 +fi + +# ------------------------------------------------------ target resolution --- + +set +e +resolved="$("$CD_TARGET" resolve --repo "$REPO" --branch "$BRANCH")" +resolve_rc=$? +set -e + +if [[ $resolve_rc -eq $NO_MATCH ]]; then + log "no target on $(hostname) for ${REPO}@${BRANCH} -- nothing to do here" + exit 0 +elif [[ $resolve_rc -ne 0 ]]; then + die "target lookup failed for ${REPO}@${BRANCH} (exit ${resolve_rc})" +fi + +eval "$resolved" + +mkdir -p "$STATE_DIR" +readonly LOG_FILE="${STATE_DIR}/${CD_NAME}.log" +exec > >(stdbuf -oL tee -a "$LOG_FILE") 2>&1 + +log "==============================================================" +log "target ${CD_NAME} (${CD_ENV} on ${CD_RESOLVED_HOST})" +log "repo ${REPO}@${BRANCH}" +log "commit ${SHA}" +log "stack ${CD_STACK_DIR}/${CD_COMPOSE_FILE} [project ${CD_COMPOSE_PROJECT}]" +[[ $DRY_RUN -eq 1 ]] && log "mode DRY RUN -- nothing will be changed" + +[[ -d "$CD_STACK_DIR" ]] || die "stack directory missing: ${CD_STACK_DIR}" +[[ -f "${CD_STACK_DIR}/${CD_COMPOSE_FILE}" ]] \ + || die "compose file missing: ${CD_STACK_DIR}/${CD_COMPOSE_FILE}" + +# ----------------------------------------------------------------- notify --- + +notify() { + local status="$1" message="$2" icon topic + case "$status" in + ok) icon="✅" ;; + fail) icon="❌" ;; + *) icon="ℹ️" ;; + esac + topic="${REPO//\//-}" + curl -fsS --max-time 10 \ + -H "Title: ${CD_NAME} deploy" \ + -d "${icon} ${CD_NAME} (${CD_ENV}): ${message}" \ + "${CD_NTFY_BASE_URL}/${topic}" >/dev/null 2>&1 \ + || warn "ntfy notification failed (deploy outcome itself is unaffected)" +} + +# ------------------------------------------------------------------- lock --- + +# Serialise per target, not per host: a test deploy must not block a prod one. +readonly LOCK_FILE="${STATE_DIR}/${CD_NAME}.lock" +exec 9>"$LOCK_FILE" +log "waiting for deploy lock (${CD_LOCK_WAIT_SECONDS}s max)" +if ! flock -w "$CD_LOCK_WAIT_SECONDS" 9; then + notify fail "timed out waiting for the deploy lock after ${CD_LOCK_WAIT_SECONDS}s" + die "timed out waiting for deploy lock after ${CD_LOCK_WAIT_SECONDS}s" +fi +log "lock acquired" + +# -------------------------------------------------------------- image tag --- + +tag="$CD_IMAGE_TAG_TEMPLATE" +tag="${tag//\{branch\}/$BRANCH}" +tag="${tag//\{env\}/$CD_ENV}" +tag="${tag//\{sha\}/$SHA}" +tag="${tag//\{short7\}/${SHA:0:7}}" +tag="${tag//\{short12\}/${SHA:0:12}}" +readonly CANDIDATE="${CD_IMAGE_REPO}:${tag}" + +log "candidate ${CANDIDATE}" + +# ------------------------------------------------------- wait for the image -- + +# The push webhook fires the moment the commit lands, which is well before CI +# has finished building. Poll rather than fail: the running container keeps +# serving throughout. +wait_for_image() { + local elapsed=0 + while true; do + if docker pull "$CANDIDATE" >/dev/null 2>&1; then + log "image available after ${elapsed}s" + return 0 + fi + if (( elapsed >= CD_IMAGE_WAIT_SECONDS )); then + return 1 + fi + log "image not published yet, retrying in ${CD_IMAGE_POLL_INTERVAL}s (${elapsed}s elapsed)" + sleep "$CD_IMAGE_POLL_INTERVAL" + elapsed=$(( elapsed + CD_IMAGE_POLL_INTERVAL )) + done +} + +log "waiting for CI to publish the image (up to ${CD_IMAGE_WAIT_SECONDS}s)" +if [[ $DRY_RUN -eq 1 ]]; then + log "dry run: skipping image wait" +elif ! wait_for_image; then + notify fail "image ${CANDIDATE} never appeared (waited ${CD_IMAGE_WAIT_SECONDS}s) -- did CI fail?" + die "image ${CANDIDATE} did not appear within ${CD_IMAGE_WAIT_SECONDS}s" +fi + +# --------------------------------------------------- verify and pin digest --- + +image_digest_ref() { + docker image inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "$1" 2>/dev/null \ + | grep "^${CD_IMAGE_REPO}@" | head -n1 +} + +if [[ $DRY_RUN -eq 1 ]]; then + DEPLOY_IMAGE="$CANDIDATE" + log "dry run: would deploy ${DEPLOY_IMAGE}" +else + revision="$(docker image inspect \ + --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' \ + "$CANDIDATE" 2>/dev/null || true)" + + if [[ -z "$revision" || "$revision" == "" ]]; then + warn "image carries no org.opencontainers.image.revision label; cannot prove provenance" + elif [[ "$revision" != "$SHA" ]]; then + notify fail "image ${CANDIDATE} is built from ${revision:0:12}, not ${SHA:0:12} -- refusing to deploy" + die "provenance mismatch: ${CANDIDATE} declares revision ${revision}, expected ${SHA}" + else + log "provenance revision label matches ${SHA:0:12}" + fi + + DEPLOY_IMAGE="$(image_digest_ref "$CANDIDATE")" + if [[ -z "$DEPLOY_IMAGE" ]]; then + warn "could not resolve a digest for ${CANDIDATE}; deploying by tag instead" + DEPLOY_IMAGE="$CANDIDATE" + else + log "pinned ${DEPLOY_IMAGE}" + fi +fi +readonly DEPLOY_IMAGE + +# --------------------------------------------- remember what is running now --- + +PREVIOUS_IMAGE="$( + container_image="$(docker inspect --format '{{.Image}}' "$CD_CONTAINER" 2>/dev/null || true)" + [[ -n "$container_image" ]] && image_digest_ref "$container_image" || true +)" +readonly PREVIOUS_IMAGE + +if [[ -n "$PREVIOUS_IMAGE" ]]; then + log "current ${PREVIOUS_IMAGE}" +else + log "current (nothing running -- first deploy, or container absent)" +fi + +# ----------------------------------------------------------------- deploy --- + +compose_up() { + local image="$1" + # `up -d` recreates only what actually changed. `down` is deliberately not + # used here: it causes avoidable downtime, and `down -v` would destroy the + # data volumes these stacks depend on. + ( cd "$CD_STACK_DIR" \ + && env "${CD_IMAGE_ENV_VAR}=${image}" \ + "${CD_PULL_POLICY_ENV_VAR}=missing" \ + timeout "$CD_COMPOSE_TIMEOUT_SECONDS" \ + docker compose -f "$CD_COMPOSE_FILE" -p "$CD_COMPOSE_PROJECT" up -d --no-build ) +} + +check_health() { + local attempt=1 body + while (( attempt <= CD_HEALTH_RETRIES )); do + body="$(curl -fsS --max-time 5 "$CD_HEALTH_URL" 2>/dev/null || true)" + if [[ -n "$body" ]]; then + if python3 -c ' +import json, sys +key, want = sys.argv[1], sys.argv[2] +try: + data = json.loads(sys.stdin.read()) +except Exception: + sys.exit(1) +sys.exit(0 if str(data.get(key, "")).lower() == want.lower() else 1) +' "$CD_HEALTH_EXPECT_KEY" "$CD_HEALTH_EXPECT_VALUE" <<<"$body"; then + log "health ok after ${attempt} attempt(s): ${body}" + return 0 + fi + fi + log "health not ready (attempt ${attempt}/${CD_HEALTH_RETRIES}), retrying in ${CD_HEALTH_INTERVAL}s" + sleep "$CD_HEALTH_INTERVAL" + (( attempt++ )) + done + warn "health check never passed: last response was '${body:-}'" + return 1 +} + +if [[ $DRY_RUN -eq 1 ]]; then + log "dry run: would run docker compose up -d with ${CD_IMAGE_ENV_VAR}=${DEPLOY_IMAGE}" + log "dry run: would health-check ${CD_HEALTH_URL}" + log "dry run complete -- no changes made" + exit 0 +fi + +log "deploying ${DEPLOY_IMAGE}" +if ! compose_up "$DEPLOY_IMAGE"; then + notify fail "docker compose up failed for ${SHA:0:12} -- stack left as-is" + die "docker compose up failed" +fi + +if check_health; then + log "=== deploy succeeded: ${CD_NAME} now runs ${SHA:0:12} ===" + notify ok "deployed ${SHA:0:12} (${DEPLOY_IMAGE##*@})" + exit 0 +fi + +# ---------------------------------------------------------------- rollback --- + +if [[ "$CD_ROLLBACK_ON_FAILURE" != "True" && "$CD_ROLLBACK_ON_FAILURE" != "true" ]]; then + notify fail "health check failed for ${SHA:0:12}; rollback disabled, stack left on the new image" + die "health check failed and rollback is disabled for this target" +fi + +if [[ -z "$PREVIOUS_IMAGE" ]]; then + notify fail "health check failed for ${SHA:0:12} and there is no previous image to roll back to" + die "health check failed; no previous image recorded, leaving the stack as it is" +fi + +warn "health check failed -- rolling back to ${PREVIOUS_IMAGE}" +if compose_up "$PREVIOUS_IMAGE" && check_health; then + notify fail "deploy of ${SHA:0:12} failed health check; rolled back to the previous image successfully" + die "deploy failed health check; rolled back to ${PREVIOUS_IMAGE}" +fi + +notify fail "deploy of ${SHA:0:12} failed AND rollback failed -- ${CD_NAME} needs manual attention now" +die "deploy failed and rollback also failed; manual intervention required" diff --git a/bin/cd-render-hooks b/bin/cd-render-hooks new file mode 100755 index 0000000..969564b --- /dev/null +++ b/bin/cd-render-hooks @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Render an adnanh/webhook hooks file for this host from targets.json. + +One hook is emitted per repository that this host actually deploys, not one per +environment. That keeps the Gitea side trivial -- a repository gets exactly one +webhook, configured once -- while the branch/environment mapping stays in +targets.json where it can change without anyone touching Gitea. + +Secrets never live in this repository. They are read from a host-local file +(default ~/.config/cd-webhook/secrets.env, mode 0600) as: + + CD_WEBHOOK_SECRET_WEBDEV_DOMAINDINGO= + +A repository with no secret is a hard error: a hook without HMAC validation +would accept a deploy request from anyone who can reach the port. + +Usage: + cd-render-hooks [--host HOST] [--output PATH] [--redact] +""" + +from __future__ import annotations + +import argparse +import importlib.machinery +import importlib.util +import json +import os +import re +import stat +import sys +from pathlib import Path + +BIN_DIR = Path(__file__).resolve().parent +REPO_ROOT = BIN_DIR.parent + + +def _load_cd_target(): + """Import bin/cd-target, whose hyphenated name blocks a plain import. + + Sharing the module rather than re-implementing the lookup keeps one + definition of how a (repo, branch, host) triple is matched. + """ + spec = importlib.util.spec_from_loader( + "cd_target", + importlib.machinery.SourceFileLoader("cd_target", str(BIN_DIR / "cd-target")), + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +_cd_target = _load_cd_target() +load = _cd_target.load +pick_host = _cd_target.pick_host +host_matches = _cd_target.host_matches + +DEFAULT_CONFIG_DIR = Path( + os.environ.get("XDG_CONFIG_HOME", Path.home() / ".config") +) / "cd-webhook" +DEFAULT_SECRETS = DEFAULT_CONFIG_DIR / "secrets.env" +DEFAULT_OUTPUT = DEFAULT_CONFIG_DIR / "hooks.json" + +REDACTED = "" + + +def secret_var(repo: str) -> str: + """webdev/domaindingo -> CD_WEBHOOK_SECRET_WEBDEV_DOMAINDINGO""" + return "CD_WEBHOOK_SECRET_" + re.sub(r"[^A-Za-z0-9]+", "_", repo).upper() + + +def hook_id(repo: str) -> str: + """webdev/domaindingo -> webdev-domaindingo (becomes the URL path).""" + return re.sub(r"[^A-Za-z0-9]+", "-", repo).lower().strip("-") + + +def load_secrets(path: Path) -> dict[str, str]: + if not path.exists(): + sys.exit( + f"cd-render-hooks: no secrets file at {path}\n" + f" Create it (mode 0600) with one line per repository, e.g.\n" + f" CD_WEBHOOK_SECRET_WEBDEV_DOMAINDINGO=" + ) + + mode = path.stat().st_mode + if mode & (stat.S_IRWXG | stat.S_IRWXO): + sys.exit( + f"cd-render-hooks: {path} is group/world accessible.\n" + f" Run: chmod 600 {path}" + ) + + secrets: dict[str, str] = {} + for raw in path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + secrets[key.strip()] = value.strip().strip('"').strip("'") + return secrets + + +def build_hook(repo: str, branches: list[str], secret: str) -> dict: + """One hook: HMAC-validated, push-only, restricted to known branches.""" + branch_rules = [ + { + "match": { + "type": "value", + "value": f"refs/heads/{branch}", + "parameter": {"source": "payload", "name": "ref"}, + } + } + for branch in branches + ] + + return { + "id": hook_id(repo), + "execute-command": str(REPO_ROOT / "bin" / "cd-deploy"), + "command-working-directory": str(REPO_ROOT), + "http-methods": ["POST"], + "incoming-payload-content-type": "application/json", + + # The deploy runs detached; Gitea gets an immediate answer and never + # times out waiting for an image build plus a health check. + "response-message": f"deploy request accepted for {repo}", + "include-command-output-in-response": False, + + # A push to a branch we do not deploy is a normal event, not a failure. + # Returning 200 keeps Gitea's delivery history readable; the daemon log + # records which rule did not match. + "trigger-rule-mismatch-http-response-code": 200, + + "pass-arguments-to-command": [ + {"source": "string", "name": "--repo"}, + {"source": "payload", "name": "repository.full_name"}, + {"source": "string", "name": "--ref"}, + {"source": "payload", "name": "ref"}, + {"source": "string", "name": "--sha"}, + {"source": "payload", "name": "after"}, + {"source": "string", "name": "--event"}, + {"source": "header", "name": "X-Gitea-Event"}, + ], + + "trigger-rule": { + "and": [ + { + "match": { + "type": "payload-hmac-sha256", + "secret": secret, + "parameter": { + "source": "header", + "name": "X-Gitea-Signature", + }, + } + }, + { + "match": { + "type": "value", + "value": "push", + "parameter": { + "source": "header", + "name": "X-Gitea-Event", + }, + } + }, + {"or": branch_rules} if len(branch_rules) > 1 else branch_rules[0], + ] + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--host") + parser.add_argument("--secrets", type=Path, default=DEFAULT_SECRETS) + parser.add_argument("--output", type=Path, default=DEFAULT_OUTPUT) + parser.add_argument( + "--redact", + action="store_true", + help="print to stdout with secrets masked, and write nothing", + ) + args = parser.parse_args() + + manifest = load() + host = pick_host(manifest, args.host) + names = {host.lower(), host.split(".", 1)[0].lower()} + + # repo -> branches, preserving manifest order for a stable diff. + by_repo: dict[str, list[str]] = {} + for target in manifest.get("targets", []): + if not target.get("enabled", True): + continue + if not host_matches(target["host"], names): + continue + by_repo.setdefault(target["repo"], []).append(target["branch"]) + + if not by_repo: + sys.exit( + f"cd-render-hooks: no enabled targets for host {host!r}.\n" + f" Either this host deploys nothing, or its name does not match " + f"any 'host' field in targets.json." + ) + + secrets = {} if args.redact else load_secrets(args.secrets) + + hooks = [] + for repo, branches in by_repo.items(): + if args.redact: + secret = REDACTED + else: + var = secret_var(repo) + secret = secrets.get(var, "") + if not secret: + sys.exit( + f"cd-render-hooks: {args.secrets} has no value for {var}\n" + f" Every deployed repository needs its Gitea webhook secret here." + ) + hooks.append(build_hook(repo, branches, secret)) + + rendered = json.dumps(hooks, indent=2) + "\n" + + if args.redact: + print(rendered, end="") + return 0 + + args.output.parent.mkdir(parents=True, exist_ok=True) + # Create with restrictive permissions before any secret reaches the disk. + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w") as handle: + handle.write(rendered) + + print(f"wrote {args.output} ({len(hooks)} hook(s) for {host})") + for repo, branches in by_repo.items(): + print(f" /hooks/{hook_id(repo)} <- {repo} [{', '.join(branches)}]") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/cd-status b/bin/cd-status new file mode 100755 index 0000000..aeed84a --- /dev/null +++ b/bin/cd-status @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# What is this host deploying, and is the receiver actually healthy? +# +# Written because "the timer is active" was never proof of anything in this +# fleet -- the same lesson the uas-ng updater documents. This shows the daemon, +# the socket, and the outcome of the last deploy for each target. + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" +readonly STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/cd-webhook" +readonly CONFIG_DIR="${XDG_CONFIG_HOME:-${HOME}/.config}/cd-webhook" +readonly UNIT_NAME="cd-webhook.service" + +bold() { printf '\n\033[1m%s\033[0m\n' "$*"; } + +bold "Host" +if eval "$("${REPO_ROOT}/bin/cd-target" host-config 2>/dev/null)"; then + printf ' %s, receiver on %s:%s\n' "$CD_HOST" "$CD_BIND" "$CD_PORT" +else + printf ' %s is not configured in targets.json\n' "$(hostname)" + printf ' answers to: %s\n' "$("${REPO_ROOT}/bin/cd-target" hostnames)" +fi + +bold "Receiver" +state="$(systemctl --user is-active "$UNIT_NAME" 2>/dev/null || true)" +printf ' %-12s %s\n' "state" "${state:-not installed}" +if [[ "$state" == "active" ]]; then + printf ' %-12s %s\n' "since" \ + "$(systemctl --user show "$UNIT_NAME" -p ActiveEnterTimestamp --value)" + if command -v ss >/dev/null 2>&1 && [[ -n "${CD_PORT:-}" ]]; then + if ss -ltn 2>/dev/null | grep -q ":${CD_PORT}\b"; then + printf ' %-12s listening on port %s\n' "socket" "$CD_PORT" + else + printf ' %-12s \033[31mNOT listening on port %s\033[0m\n' "socket" "$CD_PORT" + fi + fi +fi + +if [[ -f "${CONFIG_DIR}/hooks.json" ]]; then + bold "Hook endpoints" + python3 - "${CONFIG_DIR}/hooks.json" <<'PY' +import json, sys +with open(sys.argv[1]) as fh: + for hook in json.load(fh): + print(f" /hooks/{hook['id']}") +PY +fi + +bold "Targets on this host" +mapfile -t targets < <("${REPO_ROOT}/bin/cd-target" list 2>/dev/null || true) +if [[ ${#targets[@]} -eq 0 ]]; then + printf ' none\n' +else + for target in "${targets[@]}"; do + log="${STATE_DIR}/${target}.log" + if [[ -f "$log" ]]; then + last="$(grep -E '=== deploy succeeded|ERROR|WARN health' "$log" | tail -n1 || true)" + printf ' %-22s %s\n' "$target" "${last:-no completed deploy recorded}" + else + printf ' %-22s %s\n' "$target" "never deployed from this host" + fi + done +fi + +bold "Recent receiver log" +journalctl --user -u "$UNIT_NAME" -n 15 --no-pager 2>/dev/null \ + || printf ' (no journal entries)\n' +echo diff --git a/bin/cd-target b/bin/cd-target new file mode 100755 index 0000000..c710b37 --- /dev/null +++ b/bin/cd-target @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +"""Query targets.json. + +The manifest is the only place that knows how a (repo, branch) pair maps onto a +host, a compose stack and an image. Everything else -- the deploy script, the +hook generator, the installer -- asks this helper rather than parsing JSON +itself, so there is exactly one implementation of the matching rules. + +Subcommands: + resolve --repo OWNER/NAME --branch BRANCH [--host HOST] + Print shell-quoted KEY=VALUE lines for the matching target. + Exit 3 when nothing matches (a normal, non-error outcome: it just + means this push is not ours to act on). + + list [--host HOST] + Print the name of every enabled target owned by HOST, one per line. + + repos [--host HOST] + Print the distinct repositories HOST deploys, one per line. + + host-config [--host HOST] + Print shell-quoted BIND/PORT for HOST. + + hostnames + Print the names this machine answers to (diagnostics). + +HOST defaults to the local machine in every subcommand. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shlex +import socket +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent + +# CD_TARGETS_FILE lets a test run against a scratch manifest. It changes which +# manifest is read, never which host this machine is allowed to act as, so it +# cannot be used to make one host deploy another's targets. +MANIFEST = Path(os.environ.get("CD_TARGETS_FILE", REPO_ROOT / "targets.json")) + +NO_MATCH = 3 + + +def local_hostnames() -> set[str]: + """Every name this machine plausibly answers to. + + Hosts in this fleet are referred to by FQDN in the manifest but report a + short name from `hostname`, so both forms -- and the leading label of any + FQDN -- have to count as a match. Getting this wrong is how the old + per-project scripts ended up with a dead `hostname != s2` guard. + """ + names = {socket.gethostname()} + try: + names.add(socket.getfqdn()) + except OSError: + pass + for name in list(names): + if "." in name: + names.add(name.split(".", 1)[0]) + return {n.lower() for n in names if n} + + +def host_matches(target_host: str, names: set[str]) -> bool: + target_host = target_host.lower() + return target_host in names or target_host.split(".", 1)[0] in names + + +def load() -> dict: + try: + return json.loads(MANIFEST.read_text()) + except FileNotFoundError: + sys.exit(f"cd-target: manifest not found: {MANIFEST}") + except json.JSONDecodeError as exc: + sys.exit(f"cd-target: manifest is not valid JSON: {exc}") + + +def pick_host(manifest: dict, explicit: str | None) -> str: + """Return the manifest's canonical name for the host we are acting as.""" + if explicit: + return explicit + names = local_hostnames() + for host in manifest.get("hosts", {}): + if host_matches(host, names): + return host + # Not a configured host: return the short local name so callers report + # something truthful rather than silently adopting another host's identity. + return socket.gethostname() + + +def emit(pairs: dict[str, object]) -> None: + for key, value in pairs.items(): + print(f"{key}={shlex.quote(str(value))}") + + +def cmd_resolve(args, manifest: dict) -> int: + host = pick_host(manifest, args.host) + names = local_hostnames() if not args.host else {args.host.lower()} + defaults = manifest.get("defaults", {}) + + for target in manifest.get("targets", []): + if not target.get("enabled", True): + continue + if target["repo"] != args.repo or target["branch"] != args.branch: + continue + if not host_matches(target["host"], names): + continue + + merged = {**defaults, **target} + merged["resolved_host"] = host + emit({f"CD_{k.upper()}": v for k, v in merged.items() if not k.startswith("_")}) + return 0 + + return NO_MATCH + + +def cmd_list(args, manifest: dict) -> int: + names = local_hostnames() if not args.host else {args.host.lower()} + for target in manifest.get("targets", []): + if target.get("enabled", True) and host_matches(target["host"], names): + print(target["name"]) + return 0 + + +def cmd_repos(args, manifest: dict) -> int: + names = local_hostnames() if not args.host else {args.host.lower()} + seen: list[str] = [] + for target in manifest.get("targets", []): + if not target.get("enabled", True): + continue + if host_matches(target["host"], names) and target["repo"] not in seen: + seen.append(target["repo"]) + print("\n".join(seen)) + return 0 + + +def cmd_host_config(args, manifest: dict) -> int: + host = pick_host(manifest, args.host) + config = manifest.get("hosts", {}).get(host) + if config is None: + sys.exit( + f"cd-target: host {host!r} has no entry in targets.json 'hosts'. " + "Add one before installing here." + ) + emit({"CD_HOST": host, "CD_BIND": config["bind"], "CD_PORT": config["port"]}) + return 0 + + +def cmd_hostnames(_args, _manifest: dict) -> int: + print(" ".join(sorted(local_hostnames()))) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description="Query the CD target manifest") + sub = parser.add_subparsers(dest="command", required=True) + + p = sub.add_parser("resolve") + p.add_argument("--repo", required=True) + p.add_argument("--branch", required=True) + p.add_argument("--host") + p.set_defaults(func=cmd_resolve) + + for name, func in (("list", cmd_list), ("repos", cmd_repos), + ("host-config", cmd_host_config)): + p = sub.add_parser(name) + p.add_argument("--host") + p.set_defaults(func=func) + + p = sub.add_parser("hostnames") + p.set_defaults(func=cmd_hostnames) + + args = parser.parse_args() + return args.func(args, load()) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md new file mode 100644 index 0000000..40fc6b8 --- /dev/null +++ b/docs/OPERATIONS.md @@ -0,0 +1,124 @@ +# Operations + +Runbook for the CD webhook receiver. See [../README.md](../README.md) for the +design. + +## First question: is it actually working? + +```bash +~/S/my-cd-webhook/bin/cd-status +``` + +An active service is not proof of anything on its own — the same lesson the +`uas-ng` updater documents. `cd-status` shows the daemon state, whether the port +is genuinely listening, the hook endpoints, and the outcome of the last deploy +per target. + +## Where things are + +| What | Path | +|---|---| +| This repository | `~/S/my-cd-webhook` | +| Secrets (0600) | `~/.config/cd-webhook/secrets.env` | +| Rendered hooks (0600, generated) | `~/.config/cd-webhook/hooks.json` | +| Deploy logs and locks | `~/.local/state/cd-webhook/.log` | +| systemd unit | `~/.config/systemd/user/cd-webhook.service` | + +```bash +journalctl --user -u cd-webhook.service -f # receiver +tail -f ~/.local/state/cd-webhook/domaindingo-prod.log # a deploy +``` + +## Deploying by hand + +The deploy script is the same one the webhook calls, so a manual deploy and an +automatic one are identical: + +```bash +cd ~/S/my-cd-webhook +./bin/cd-deploy --repo webdev/domaindingo \ + --ref refs/heads/prod \ + --sha +``` + +Add `--dry-run` to see what it would do without touching anything. + +The commit must be one CI has built, because the script deploys +`-sha-` and verifies the image's revision label matches. + +## Rolling back + +A failed health check rolls back automatically when the target has +`rollback_on_failure` set. To roll back deliberately, deploy the previous +commit: + +```bash +./bin/cd-deploy --repo webdev/domaindingo --ref refs/heads/prod --sha +``` + +This works because every build is published under an immutable per-commit tag. +Deploying "the previous version" never depends on a mutable tag still meaning +what it meant yesterday — which in this fleet it once did not. + +## Rotating a webhook secret + +1. Generate: `openssl rand -hex 32` +2. Update the value in Gitea (**Repository → Settings → Webhooks → edit → Secret**). +3. Update the matching line in `~/.config/cd-webhook/secrets.env`. +4. Re-render and restart: `./install/install.sh` + +Order matters only in that there is a window between steps 2 and 3 where pushes +are rejected. Nothing deploys with a bad signature, so the failure mode is a +missed deploy, not a wrong one. + +## Adding a host + +1. Add the host to `"hosts"` in `targets.json` with its WireGuard address and + port `20090`. +2. Add its targets. +3. Commit and push this repository. +4. On the host: clone, `./install/install.sh`, fill in `secrets.env`, re-run. +5. Confirm Gitea (on s5) can reach the host's WireGuard address on that port. + +## Troubleshooting + +**Gitea shows the delivery as succeeded but nothing deployed.** +Expected when the push was to a branch this host does not deploy — the hook +returns 200 by design so the delivery history stays readable. Check the receiver +log for which rule did not match, and `cd-target list` for what this host owns. + +**Deliveries fail with a signature error.** +`secrets.env` and the Gitea webhook disagree. Re-run `install/install.sh` after +fixing, and confirm the Gitea webhook's content type is `application/json`. + +**"image ... did not appear within 900s".** +CI never published the image. Check the repository's Actions run — this is +almost always a failed build, not a deployment fault. The old container keeps +serving throughout, so the site is unaffected. + +**"provenance mismatch: ... declares revision X, expected Y".** +The tag exists but was built from a different commit. This is the tag-drift +failure mode, caught before anything changed. Do not work around it by deploying +the tag directly; find out why the tag moved. + +**The receiver will not start.** +```bash +systemctl --user status cd-webhook.service +journalctl --user -u cd-webhook.service -n 50 --no-pager +``` +Usual causes: the `webhook` binary is missing, the bind address does not exist +on this host (check `ip addr show wg0`), or the port is already taken. + +**Deploys stopped after a reboot and nobody was logged in.** +`loginctl enable-linger` should have been set by the installer. Verify with +`loginctl show-user "$(whoami)" -p Linger`. + +## Deliberate non-features + +- **No `docker compose down`.** `up -d` recreates only what changed. `down -v` + in particular would destroy the data volumes these stacks depend on. +- **No arbitrary SHA input over the webhook.** The commit always comes from the + signed push payload. +- **No deploys from tags, branch deletions, or non-push events.** +- **No git operations.** Compose stacks are `uas-ng`'s responsibility and are + refreshed by its own updater timer. diff --git a/docs/active-context.md b/docs/active-context.md new file mode 100644 index 0000000..9816db8 --- /dev/null +++ b/docs/active-context.md @@ -0,0 +1,17 @@ +# Active context — knowledge routing index + +Index of this project's knowledge-base entries. Read an entry **only when your +current task matches its trigger** — never preemptively. When an instruction has +been fully implemented and is no longer needed, delete its line. (An ai-context +update may re-add it; if it's still done, just delete it again.) + +## Entries + +- Working style — code/tool checklist, math/arithmetic options → `ai-context/style/working-style.md` +- Repository operations — Gitea project, `tea` usage, branch/PR flow, remote dev + bind addresses, package-manager notes → `docs/project-knowledge-base.md` +- Gitea workflow — branch flow (`main → test → dev → feature`) and posting comments/PRs with `tea` → `ai-context/infrastructure/git-instructions.md` +- Gitea CI on a rootless-Docker runner — Playwright/Buildx, socket mount errors → `ai-context/infrastructure/rootless-docker-gitea-runner.md` +- Publishing public releases to a separate GitHub repo → `ai-context/infrastructure/gitea-to-github-release.md` +- Uploading build artifacts to Cloudflare R2 from CI → `ai-context/services/cloudflare-r2.md` +- CI success/failure notifications via ntfy → `ai-context/services/ntfy.md` diff --git a/etc/cd-webhook.service b/etc/cd-webhook.service new file mode 100644 index 0000000..417c97a --- /dev/null +++ b/etc/cd-webhook.service @@ -0,0 +1,33 @@ +[Unit] +Description=Continuous deployment webhook receiver (adnanh/webhook) +Documentation=https://github.com/adnanh/webhook +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple + +# Placeholders are substituted by install/install.sh from targets.json. +# -hotreload picks up a re-rendered hooks file without a restart, so adding an +# environment is `cd-render-hooks` and nothing else. +# -nopanic keeps the daemon alive if the hooks file is momentarily unreadable +# mid-write rather than exiting and taking every other target down with it. +ExecStart=@WEBHOOK_BIN@ \ + -hooks @HOOKS_FILE@ \ + -ip @BIND@ \ + -port @PORT@ \ + -hotreload \ + -nopanic + +Restart=on-failure +RestartSec=5s + +# The deploy itself is a child process that talks to the user's Docker socket, +# so this stays deliberately unsandboxed apart from the cheap wins below. +NoNewPrivileges=yes +PrivateTmp=yes +ProtectControlGroups=yes +ProtectKernelTunables=yes + +[Install] +WantedBy=default.target diff --git a/install/install.sh b/install/install.sh new file mode 100755 index 0000000..9d1a85f --- /dev/null +++ b/install/install.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# +# Install the CD webhook receiver on this host. +# +# Idempotent: re-run it after editing targets.json, after rotating a secret, or +# after pulling a new version of this repository. It re-renders the hooks file +# and restarts the daemon; it never touches a project repository or uas-ng. +# +# Prerequisites: +# * adnanh/webhook on PATH (or WEBHOOK_BIN pointing at it) +# * a host-local secrets file, see secrets.env.example +# * this host present in targets.json under "hosts" + +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)" + +readonly CONFIG_DIR="${XDG_CONFIG_HOME:-${HOME}/.config}/cd-webhook" +readonly HOOKS_FILE="${CONFIG_DIR}/hooks.json" +readonly SECRETS_FILE="${CONFIG_DIR}/secrets.env" +readonly SYSTEMD_DIR="${HOME}/.config/systemd/user" +readonly UNIT_NAME="cd-webhook.service" + +info() { printf '\n\033[1m==> %s\033[0m\n' "$*"; } +ok() { printf ' ok %s\n' "$*"; } +die() { printf '\n\033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; } + +# ------------------------------------------------------------ prerequisites -- + +info "Checking prerequisites" + +for tool in python3 docker curl flock stdbuf; do + command -v "$tool" >/dev/null 2>&1 || die "required tool not found: ${tool}" +done +ok "python3, docker, curl, flock, stdbuf" + +WEBHOOK_BIN="${WEBHOOK_BIN:-$(command -v webhook || true)}" +[[ -n "$WEBHOOK_BIN" && -x "$WEBHOOK_BIN" ]] || die \ + "adnanh/webhook not found. Install it, or set WEBHOOK_BIN=/path/to/webhook. + Debian/Ubuntu: sudo apt install webhook + Or a release binary from https://github.com/adnanh/webhook/releases" +ok "webhook binary: ${WEBHOOK_BIN}" + +docker info >/dev/null 2>&1 || die \ + "cannot talk to Docker as $(whoami). The deploy runs as this user, so this + account needs working access to the Docker socket it deploys through." +ok "docker reachable as $(whoami)" + +# ------------------------------------------------------------- host config --- + +info "Resolving this host in targets.json" + +eval "$("${REPO_ROOT}/bin/cd-target" host-config)" +ok "host ${CD_HOST}, binding ${CD_BIND}:${CD_PORT}" + +mapfile -t TARGETS < <("${REPO_ROOT}/bin/cd-target" list) +[[ ${#TARGETS[@]} -gt 0 ]] || die "no enabled targets for ${CD_HOST} in targets.json" +for target in "${TARGETS[@]}"; do + ok "target ${target}" +done + +# ----------------------------------------------------------------- secrets --- + +info "Checking the secrets file" + +mkdir -p "$CONFIG_DIR" +chmod 700 "$CONFIG_DIR" + +if [[ ! -f "$SECRETS_FILE" ]]; then + install -m 0600 "${REPO_ROOT}/secrets.env.example" "$SECRETS_FILE" + die "created a template at ${SECRETS_FILE} + Fill in one secret per repository, then re-run this installer. + The same value must be set as the Secret on the Gitea webhook." +fi +chmod 600 "$SECRETS_FILE" +ok "${SECRETS_FILE}" + +# ------------------------------------------------------------------- hooks --- + +info "Rendering hooks for ${CD_HOST}" +"${REPO_ROOT}/bin/cd-render-hooks" --output "$HOOKS_FILE" --secrets "$SECRETS_FILE" + +# ------------------------------------------------------------------ systemd -- + +info "Installing the user service" + +mkdir -p "$SYSTEMD_DIR" +sed -e "s|@WEBHOOK_BIN@|${WEBHOOK_BIN}|g" \ + -e "s|@HOOKS_FILE@|${HOOKS_FILE}|g" \ + -e "s|@BIND@|${CD_BIND}|g" \ + -e "s|@PORT@|${CD_PORT}|g" \ + "${REPO_ROOT}/etc/cd-webhook.service" > "${SYSTEMD_DIR}/${UNIT_NAME}" +ok "${SYSTEMD_DIR}/${UNIT_NAME}" + +systemctl --user daemon-reload +systemctl --user enable "$UNIT_NAME" >/dev/null +systemctl --user restart "$UNIT_NAME" +ok "enabled and restarted" + +# Survive logout, same as the other user services in this fleet. +loginctl enable-linger "$(whoami)" >/dev/null 2>&1 || true + +sleep 1 +systemctl --user is-active --quiet "$UNIT_NAME" \ + || die "the service did not stay running: + systemctl --user status ${UNIT_NAME} + journalctl --user -u ${UNIT_NAME} -n 50 --no-pager" + +# ------------------------------------------------------------------ summary -- + +info "Done. Configure these webhooks in Gitea" + +echo +printf ' %-28s %s\n' "Target URL" "http://${CD_BIND}:${CD_PORT}/hooks/ (below)" +printf ' %-28s %s\n' "HTTP Method" "POST" +printf ' %-28s %s\n' "Content Type" "application/json" +printf ' %-28s %s\n' "Trigger On" "Push Events" +printf ' %-28s %s\n' "Secret" "the matching value from ${SECRETS_FILE}" +echo + +"${REPO_ROOT}/bin/cd-render-hooks" --redact >/dev/null 2>&1 || true +python3 - "$HOOKS_FILE" "$CD_BIND" "$CD_PORT" <<'PY' +import json, sys +hooks_file, bind, port = sys.argv[1], sys.argv[2], sys.argv[3] +with open(hooks_file) as fh: + hooks = json.load(fh) +for hook in hooks: + print(f" http://{bind}:{port}/hooks/{hook['id']}") +PY + +echo +echo " Logs: journalctl --user -u ${UNIT_NAME} -f" +echo " Status: ${REPO_ROOT}/bin/cd-status" +echo diff --git a/install/uninstall.sh b/install/uninstall.sh new file mode 100755 index 0000000..73f3c49 --- /dev/null +++ b/install/uninstall.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# +# Remove the CD webhook receiver from this host. +# +# Stops and deletes the user service. Deliberately leaves the secrets file and +# the deploy logs in place -- removing a receiver is not a reason to destroy +# credentials or the record of what was deployed. Pass --purge to remove them. + +set -Eeuo pipefail + +readonly CONFIG_DIR="${XDG_CONFIG_HOME:-${HOME}/.config}/cd-webhook" +readonly STATE_DIR="${XDG_STATE_HOME:-${HOME}/.local/state}/cd-webhook" +readonly SYSTEMD_DIR="${HOME}/.config/systemd/user" +readonly UNIT_NAME="cd-webhook.service" + +PURGE=0 +[[ "${1:-}" == "--purge" ]] && PURGE=1 + +echo "Stopping ${UNIT_NAME} ..." +systemctl --user disable --now "$UNIT_NAME" 2>/dev/null || true +rm -f "${SYSTEMD_DIR}/${UNIT_NAME}" +systemctl --user daemon-reload + +# The rendered hooks file holds the webhook secrets, so it goes regardless; it +# is regenerated from targets.json plus secrets.env on the next install. +rm -f "${CONFIG_DIR}/hooks.json" + +if [[ $PURGE -eq 1 ]]; then + echo "Purging secrets and deploy logs ..." + rm -rf "$CONFIG_DIR" "$STATE_DIR" +else + echo "Kept ${CONFIG_DIR}/secrets.env and deploy logs in ${STATE_DIR}" + echo " (re-run with --purge to remove them too)" +fi + +echo "Done." diff --git a/secrets.env.example b/secrets.env.example new file mode 100644 index 0000000..1f75e65 --- /dev/null +++ b/secrets.env.example @@ -0,0 +1,12 @@ +# Host-local webhook secrets. NEVER commit the real file. +# +# Copy to ~/.config/cd-webhook/secrets.env, chmod 600, and set one value per +# repository this host deploys. Each value must match the "Secret" field of the +# corresponding webhook in Gitea (Repository -> Settings -> Webhooks). +# +# Variable name: CD_WEBHOOK_SECRET_ + the repo path, uppercased, non-alphanumerics +# collapsed to underscores. webdev/domaindingo -> CD_WEBHOOK_SECRET_WEBDEV_DOMAINDINGO +# +# Generate one with: openssl rand -hex 32 + +CD_WEBHOOK_SECRET_WEBDEV_DOMAINDINGO= diff --git a/targets.json b/targets.json new file mode 100644 index 0000000..f6c3a07 --- /dev/null +++ b/targets.json @@ -0,0 +1,90 @@ +{ + "version": 1, + + "_comment": [ + "Single source of truth for the fleet's continuous-deployment targets.", + "A target is uniquely identified by (repo, branch). The 'host' field decides", + "which machine acts on it -- every host runs the same daemon from the same", + "checkout of this repo and simply ignores targets that are not its own.", + "", + "Adding an environment is an edit here plus a re-run of install/install.sh on", + "the owning host. It is never a change to the project repository." + ], + + "defaults": { + "registry": "gitea.fisher.hu", + "ntfy_base_url": "https://ntfy.fisher.hu", + + "image_wait_seconds": 900, + "image_poll_interval": 15, + + "health_retries": 30, + "health_interval": 5, + + "lock_wait_seconds": 600, + "compose_timeout_seconds": 300 + }, + + "hosts": { + "s5.fisher.hu": { + "bind": "10.255.255.1", + "port": 20090 + }, + "s4.fisher.hu": { + "bind": "10.255.255.12", + "port": 20090 + } + }, + + "targets": [ + { + "name": "domaindingo-test", + "enabled": true, + "repo": "webdev/domaindingo", + "branch": "test", + "env": "test", + "host": "s5.fisher.hu", + + "stack_dir": "/home/fisher/S/uas-ng/docker/domaindingo/s5.fisher.hu", + "compose_file": "docker-compose.yml", + "compose_project": "domaindingo-test", + "container": "domaindingo-test", + + "image_repo": "gitea.fisher.hu/webdev/domaindingo", + "image_tag_template": "{branch}-sha-{short7}", + "image_env_var": "DOMAINDINGO_TEST_IMAGE", + "pull_policy_env_var": "DOMAINDINGO_TEST_PULL_POLICY", + + "health_url": "http://127.0.0.1:9101/health", + "health_expect_key": "db", + "health_expect_value": "ok", + + "rollback_on_failure": true + }, + + { + "name": "domaindingo-prod", + "enabled": true, + "repo": "webdev/domaindingo", + "branch": "prod", + "env": "prod", + "host": "s5.fisher.hu", + + "stack_dir": "/home/fisher/S/uas-ng/docker/domaindingo/s5.fisher.hu", + "compose_file": "docker-compose-prod.yml", + "compose_project": "domaindingo-prod", + "container": "domaindingo-prod", + + "image_repo": "gitea.fisher.hu/webdev/domaindingo", + "image_tag_template": "{branch}-sha-{short7}", + "image_env_var": "DOMAINDINGO_PROD_IMAGE", + "pull_policy_env_var": "DOMAINDINGO_PROD_PULL_POLICY", + + "health_url": "http://127.0.0.1:9103/health", + "health_expect_key": "db", + "health_expect_value": "ok", + + "rollback_on_failure": true + } + ] +}