From 47fb3f2193af82227293d5b64a97b5a4e24af05b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 5 Nov 2026 11:26:50 -0500 Subject: [PATCH] ci: the weekly refresh also rebuilds :dev, and a failed unattended run opens an issue The Sunday refresh failed five weeks running (2026-10-04 to 11-01) and nothing said so; it was found by reading the run list (#6864). It also only ever refreshed :latest: an artifact whose dev source is quiet, the agent most weeks, kept its last build's base layers on :dev, because reuse skips it on every push. - refresh-dev: after promote, a refresh dispatches a forced dev run. That run is an ordinary dev build (lanes, smoke, promote), so :dev gets dev's own source with FC_CHANNEL=dev rather than main's bytes relabelled. - A forced build now pulls fresh bases, as the refresh does; otherwise forcing rebuilds the same image. - unattended-failure: any failed run that is not a push or PR opens an issue (or comments on the open one) naming the run and the failed jobs. - RELEASE_TOKEN's scope list names write:repository, which the dispatch needs. Co-Authored-By: Claude Opus 5.5 --- .forgejo/workflows/build.yml | 132 +++++++++++++++++++++++++++++++++-- 1 file changed, 127 insertions(+), 5 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 6bedc66..8c049f7 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -202,7 +202,8 @@ env: # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes: # - write:package, read:package (for docker push to git.fabledsword.com) # - write:release (for ext- release asset cache) -# - write:issue (for future issue-management automation) +# - write:issue (`unattended-failure` opens an issue) +# - write:repository (`refresh-dev` dispatches this workflow) # The injected GITHUB_TOKEN cannot be used — it lacks write:package. jobs: @@ -1446,8 +1447,11 @@ jobs: # `no-cache: true`, which is a much larger version of the same # churn #3265 is about. # - # Only on the schedule. An ordinary push wants the cached base. - pull: ${{ env.IS_REFRESH == 'true' }} + # Only on the refresh and on a forced build. An ordinary push wants + # the cached base. A forced build pulls because forcing is how the + # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that + # keeps the old base layers would be the same image again. + pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }} # ONE tag, the channel's. Every other tag is written by the step # below, registry-side. buildx here pushes the first tag to the # registry and then re-pushes the rest through the DOCKER driver, @@ -2202,6 +2206,121 @@ jobs: exit 1 fi echo "promote: $MOVED tag(s) written" + + # The refresh rebuilds `:dev` as well as `:latest`. + # + # The refresh run itself builds main and publishes `:latest`. `:dev` needs a + # build of dev's own source with FC_CHANNEL=dev baked in, so copying main's + # refreshed bytes onto `:dev` is not the same thing: the image would report + # itself as main, and the reuse check would then keep it that way. Instead + # this job dispatches an ordinary dev run with `force_build`. That run goes + # through the lanes, the smoke and `promote` like any push, and a forced + # build pulls fresh bases (see `pull:` in both build jobs). + # + # Without this, an artifact whose source is quiet on dev (the agent, most + # weeks) keeps the base layers of its last build indefinitely, because + # reuse skips its build on every push. + # + # The dispatched run shares this run's concurrency group (both are on the + # dev ref), so it queues until this run has finished. It runs whatever + # `promote` did: a failed `:latest` refresh is no reason to leave `:dev` + # stale, and `unattended-failure` reports the failure either way. + refresh-dev: + needs: [promote] + if: >- + !cancelled() && (github.event_name == 'schedule' + || format('{0}', github.event.inputs.refresh) == 'true') + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - name: Dispatch a forced dev build + env: + TOKEN: ${{ secrets.RELEASE_TOKEN }} + API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }} + run: | + set -eu + CODE=$(curl -sS --max-time 30 -o response.txt -w '%{http_code}' -X POST \ + -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \ + -d '{"ref":"dev","inputs":{"force_build":"true"}}' \ + "$API/actions/workflows/build.yml/dispatches") + echo "refresh-dev: dispatch answered HTTP $CODE" + case "$CODE" in + 2??) echo "refresh-dev: a forced dev run is queued behind this one" ;; + *) + cat response.txt >&2 + echo "refresh-dev: the dispatch was refused. A 403 means RELEASE_TOKEN" >&2 + echo "refresh-dev: lacks write:repository (see the scope list at the top)." >&2 + exit 1 ;; + esac + + # A run nobody is watching reports its own failure. + # + # A push run is watched by whoever pushed it. The weekly refresh and the dev + # run it dispatches are not, and a red run on the Actions page alone goes + # unseen. So any failed run that is not a push or a pull request opens an + # issue, or comments on the one already open, naming the run and the jobs + # that failed. A manual dispatch that fails reports here too; closing the + # issue once it is dealt with is what keeps the next failure visible. + # + # The issue is written with RELEASE_TOKEN, so it is authored by the token's + # owner, and the forge does not notify people of their own actions. It + # shows in the repository's open issues rather than as an email. + unattended-failure: + needs: [lint, extension-version, backend-lint-and-test, frontend-build, + extension-test, integration, sign-extension, build-web, smoke-web, + build-agent, promote, refresh-dev] + if: >- + failure() && github.event_name != 'push' + && github.event_name != 'pull_request' + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + steps: + - name: Open or update the failure issue + env: + TOKEN: ${{ secrets.RELEASE_TOKEN }} + API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + EVENT: ${{ github.event_name }} + REF: ${{ github.ref_name }} + NEEDS: ${{ toJSON(needs) }} + run: | + python3 - <<'EOF' + import json, os, urllib.parse, urllib.request + + TITLE = "CI: an unattended build.yml run failed" + api, token = os.environ["API"], os.environ["TOKEN"] + + def call(method, path, body=None): + req = urllib.request.Request( + api + path, method=method, + data=json.dumps(body).encode() if body is not None else None, + headers={"Authorization": f"token {token}", + "Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as resp: + return json.load(resp) + + needs = json.loads(os.environ["NEEDS"]) + failed = sorted(j for j, v in needs.items() if v.get("result") == "failure") + text = (f"Run: {os.environ['RUN_URL']}\n" + f"Trigger: `{os.environ['EVENT']}` on `{os.environ['REF']}`\n" + f"Failed jobs: {', '.join(f'`{j}`' for j in failed) or '(none reported)'}\n") + + q = urllib.parse.urlencode({"state": "open", "type": "issues", "q": TITLE}) + existing = [i for i in call("GET", f"/issues?{q}") if i["title"] == TITLE] + if existing: + n = existing[0]["number"] + call("POST", f"/issues/{n}/comments", {"body": text}) + print(f"unattended-failure: commented on open issue #{n}") + else: + body = ("A build.yml run that nobody watches failed. Read the run's " + "failing job logs; close this issue once it is fixed, so the " + "next failure opens a new one.\n\n" + text) + n = call("POST", "/issues", {"title": TITLE, "body": body})["number"] + print(f"unattended-failure: opened issue #{n}") + EOF + build-agent: # THE GATE (2026-09-23). Every lane above must have PASSED before this job # exists at all — so a red suite does not produce an image, let alone push @@ -2540,8 +2659,11 @@ jobs: # `no-cache: true`, which is a much larger version of the same # churn #3265 is about. # - # Only on the schedule. An ordinary push wants the cached base. - pull: ${{ env.IS_REFRESH == 'true' }} + # Only on the refresh and on a forced build. An ordinary push wants + # the cached base. A forced build pulls because forcing is how the + # refresh rebuilds `:dev` (see `refresh-dev`), and a rebuild that + # keeps the old base layers would be the same image again. + pull: ${{ env.IS_REFRESH == 'true' || format('{0}', github.event.inputs.force_build) == 'true' }} # ONE tag, the channel's. Every other tag is written by the step # below, registry-side. buildx here pushes the first tag to the # registry and then re-pushes the rest through the DOCKER driver,