fix(alerts): an alert's email waits for the real commit, not a savepoint (#5807)
SQLAlchemy fires after_commit and after_rollback for a savepoint too. The host-agent ingest releases one after recording its metrics, so an alert raised by a sample started its email before the outer transaction committed; twice on 2026-10-12 the outbox row hit the event's foreign key first and the email was lost. A savepoint rolling back (a docker persist failing) cleared every pending notification, the outer one's included. _after_commit now runs its jobs only on the outermost commit, and a rolled-back savepoint drops only the jobs added inside it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+38
-10
@@ -212,28 +212,56 @@ def _spawn(coro) -> None:
|
||||
task.add_done_callback(_background.discard)
|
||||
|
||||
|
||||
def _within(txn, boundary) -> bool:
|
||||
"""Whether `txn` (a savepoint, or None for the outer transaction) is
|
||||
`boundary` or opened inside it."""
|
||||
while txn is not None:
|
||||
if txn is boundary:
|
||||
return True
|
||||
txn = txn.parent
|
||||
return False
|
||||
|
||||
|
||||
def _after_commit(session: AsyncSession, fn, *args) -> None:
|
||||
"""Run fn(app, *args) once the session's transaction commits, and never if
|
||||
it rolls back: a notification or a playbook run must describe what was
|
||||
actually stored."""
|
||||
"""Run fn(app, *args) once the session's outer transaction commits, and
|
||||
never if what added it rolls back: a notification or a playbook run must
|
||||
describe what was actually stored.
|
||||
|
||||
SQLAlchemy fires after_commit and after_rollback for a savepoint too
|
||||
(#5807). Releasing one, as the host-agent ingest does after its metrics,
|
||||
is not the commit: a job started then raced the real commit, and an email
|
||||
queued for an event not yet visible broke its foreign key. A savepoint
|
||||
rolling back drops only the jobs added inside it."""
|
||||
if _app is None:
|
||||
return
|
||||
info = session.info
|
||||
sync = session.sync_session
|
||||
if _AFTER_COMMIT not in info:
|
||||
info[_AFTER_COMMIT] = []
|
||||
app = _app
|
||||
|
||||
def _committed(_sync_session) -> None:
|
||||
def _committed(sync_session) -> None:
|
||||
if sync_session.get_nested_transaction() is not None:
|
||||
return # a savepoint released: its jobs wait for the outer commit
|
||||
jobs, info[_AFTER_COMMIT] = info[_AFTER_COMMIT], []
|
||||
for job_fn, job_args in jobs:
|
||||
for _owner, job_fn, job_args in jobs:
|
||||
_spawn(job_fn(app, *job_args))
|
||||
|
||||
def _rolled_back(_sync_session) -> None:
|
||||
info[_AFTER_COMMIT] = []
|
||||
def _rolled_back(_sync_session, previous_transaction) -> None:
|
||||
# What was actually rolled back: the nearest savepoint, or the
|
||||
# outer transaction (a failed flush rolls back to one of them).
|
||||
boundary = previous_transaction
|
||||
while boundary.parent is not None and not boundary.nested:
|
||||
boundary = boundary.parent
|
||||
if not boundary.nested:
|
||||
info[_AFTER_COMMIT] = []
|
||||
else:
|
||||
info[_AFTER_COMMIT] = [job for job in info[_AFTER_COMMIT]
|
||||
if not _within(job[0], boundary)]
|
||||
|
||||
sa_event.listen(session.sync_session, "after_commit", _committed)
|
||||
sa_event.listen(session.sync_session, "after_rollback", _rolled_back)
|
||||
info[_AFTER_COMMIT].append((fn, args))
|
||||
sa_event.listen(sync, "after_commit", _committed)
|
||||
sa_event.listen(sync, "after_soft_rollback", _rolled_back)
|
||||
info[_AFTER_COMMIT].append((sync.get_nested_transaction(), fn, args))
|
||||
|
||||
|
||||
LATEST_CHUNK = 1000
|
||||
|
||||
Reference in New Issue
Block a user