fix(backup,tags): unwedge backups on NFS (#739) + tag-standardize "0 groups" (#740)

#739 — DB backups hung on NFS in uninterruptible D-state, defeating the 12-min
subprocess timeout AND Celery's hard limit, so a stuck pg_dump held the
concurrency-1 maintenance_long lane for hours — starving normalize_tags,
re-extract, audits, and the new series rescan (which is why #740 "never
applied"). Three fixes:
- _run_bounded: Popen + bounded post-kill reap; if the child is unkillable
  (D-state) we stop waiting and re-raise TimeoutExpired, freeing the slot. The
  orphan is reaped by the OS once its syscall clears.
- backup_db dumps to a LOCAL temp file then moves the finished .sql to the
  (NFS) _backups dir — pg_dump's long phase is now a DB-socket wait + local
  writes (killable) instead of an NFS write that hangs. backup_images keeps
  bounded-kill (too big to stage locally).
- recover_stalled_backup_runs: split the stall window — db 40 min (was sharing
  images' 7h), so a hung DB backup is flipped to error promptly.

#740 — Standardize tag casing showed "0 groups to change" the instant it was
clicked: onNormCommit overwrote the preview with zeros. Keep the real preview
visible and disable the button while queued; backend apply was already correct.

Tests: fake subprocess.Popen alongside run; bounded-kill fail-fast; local-temp
target; per-kind stall sweep.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/app/services/backup_service.py

@@ -15,7 +15,10 @@ lifecycle + soft/hard time limits + retention bookkeeping.
 from __future__ import annotations
 
 import json
+import os
+import shutil
 import subprocess
+import tempfile
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -26,6 +29,35 @@ _BACKUPS_DIRNAME = "_backups"
 # blocking syscall ignores that signal. These bound the worst case.
 _DB_SUBPROCESS_TIMEOUT_S = 12 * 60        # 12 min (Celery soft is 10 min)
 _IMAGES_SUBPROCESS_TIMEOUT_S = 7 * 60 * 60  # 7 hr (Celery soft is 6 hr)
+# Grace after SIGKILL to reap the child. If it can't be reaped in this window
+# (an uninterruptible NFS D-state — the failure mode that wedged the
+# concurrency-1 maintenance lane for hours, operator-flagged 2026-06-07), we
+# STOP waiting and fail fast, freeing the worker slot. The orphan is reaped by
+# the OS once its blocking syscall clears.
+_KILL_REAP_GRACE_S = 10
+
+
+def _run_bounded(cmd: list[str], timeout: int) -> None:
+    """subprocess.run(check=True, timeout) whose reaper can't itself hang.
+
+    subprocess.run's timeout path SIGKILLs the child then blocks in wait() to
+    reap it — but a process stuck in uninterruptible I/O (NFS) can't be reaped,
+    so wait() blocks for hours. Here we bound the post-kill reap and re-raise
+    TimeoutExpired regardless, so the caller fails fast instead of wedging."""
+    proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    try:
+        out, err = proc.communicate(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        try:
+            proc.communicate(timeout=_KILL_REAP_GRACE_S)
+        except subprocess.TimeoutExpired:
+            pass  # unkillable (D-state) — abandon the reap, fail fast
+        raise
+    if proc.returncode != 0:
+        raise subprocess.CalledProcessError(
+            proc.returncode, cmd, output=out, stderr=err
+        )
 
 
 def _libpq_url(sa_url: str) -> str:
@@ -84,14 +116,25 @@ def backup_db(
     ts = _now_ts()
     out_dir = _backups_dir(images_root)
     sql_path = out_dir / f"fc_db_{ts}.sql"
-    subprocess.run(
-        [
-            "pg_dump", "--no-owner", "--no-acl",
-            "-f", str(sql_path), _libpq_url(db_url),
-        ],
-        capture_output=True, check=True,
-        timeout=_DB_SUBPROCESS_TIMEOUT_S,
-    )
+    # Dump to LOCAL disk first, then move the finished file to the (NFS) backups
+    # dir. pg_dump's long phase is then a DB-socket wait + local writes — both
+    # killable — instead of an NFS write that can hang uninterruptibly. Only the
+    # final move touches NFS, and it's a bounded single-file step.
+    fd, tmp_name = tempfile.mkstemp(prefix="fc_db_", suffix=".sql")
+    os.close(fd)
+    tmp_path = Path(tmp_name)
+    try:
+        _run_bounded(
+            [
+                "pg_dump", "--no-owner", "--no-acl",
+                "-f", str(tmp_path), _libpq_url(db_url),
+            ],
+            _DB_SUBPROCESS_TIMEOUT_S,
+        )
+        shutil.move(str(tmp_path), str(sql_path))
+    finally:
+        if tmp_path.exists():
+            tmp_path.unlink(missing_ok=True)
     manifest_path = _write_manifest(
         out_dir, kind="db", ts=ts, tag=tag, triggered_by=triggered_by,
         artifact_path=sql_path,
@@ -114,15 +157,17 @@ def backup_images(
     ts = _now_ts()
     out_dir = _backups_dir(images_root)
     tar_path = out_dir / f"fc_images_{ts}.tar.zst"
-    subprocess.run(
+    # No local-temp here (the archive is hundreds of GB — it can't stage in
+    # /tmp), but bounded-kill still applies so a tar wedged on NFS fails fast
+    # rather than holding the lane for hours.
+    _run_bounded(
         [
             "tar", "--zstd", "-cf", str(tar_path),
             "-C", str(images_root.parent), images_root.name,
             f"--exclude={images_root.name}/_backups",
             f"--exclude={images_root.name}/_quarantine",
         ],
-        capture_output=True, check=True,
-        timeout=_IMAGES_SUBPROCESS_TIMEOUT_S,
+        _IMAGES_SUBPROCESS_TIMEOUT_S,
     )
     manifest_path = _write_manifest(
         out_dir, kind="images", ts=ts, tag=tag, triggered_by=triggered_by,

backend/app/tasks/maintenance.py

@@ -83,8 +83,13 @@ TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600  # 7 days
 # small buffer) so the sweep never flags in-flight work.
 #
 # Backups: images backup has time_limit=23400s (6.5h). 7h covers it
-# with a 30-min buffer; db backup at 12 min hard limit fits trivially.
+# with a 30-min buffer.
 BACKUP_STALL_THRESHOLD_MINUTES = 7 * 60
+# DB backup/restore is seconds-to-minutes (35-min hard limit). It must NOT share
+# the images' 7h window — a DB backup wedged on NFS would otherwise sit "running"
+# for 7 hours holding the concurrency-1 maintenance_long lane (operator-flagged
+# 2026-06-07). 40 min gives a small buffer over the hard limit.
+BACKUP_DB_STALL_THRESHOLD_MINUTES = 40
 # Library audit: scan_library_for_rule has time_limit=7500s (2h5m).
 # 2h15m gives a 10-min buffer.
 LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135
@@ -619,16 +624,20 @@ def recover_stalled_backup_runs() -> int:
     """
     SessionLocal = _sync_session_factory()
     now = datetime.now(UTC)
-    cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES)
-    msg = (
-        f"stranded by recovery sweep (no terminal status after "
-        f"{BACKUP_STALL_THRESHOLD_MINUTES // 60}h)"
-    )
+    db_cutoff = now - timedelta(minutes=BACKUP_DB_STALL_THRESHOLD_MINUTES)
+    slow_cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES)
+    msg = "stranded by recovery sweep (no terminal status within the stall window)"
     with SessionLocal() as session:
         result = session.execute(
             update(BackupRun)
             .where(BackupRun.status.in_(["running", "restoring"]))
-            .where(BackupRun.started_at < cutoff)
+            # db backups/restores are fast (40-min window); images run hours (7h).
+            .where(
+                or_(
+                    and_(BackupRun.kind == "db", BackupRun.started_at < db_cutoff),
+                    and_(BackupRun.kind != "db", BackupRun.started_at < slow_cutoff),
+                )
+            )
             .values(status="error", finished_at=now, error=msg)
         )
         session.commit()