fix(backup,tags): unwedge backups on NFS (#739) + tag-standardize "0 groups" (#740)

#739 — DB backups hung on NFS in uninterruptible D-state, defeating the 12-min
subprocess timeout AND Celery's hard limit, so a stuck pg_dump held the
concurrency-1 maintenance_long lane for hours — starving normalize_tags,
re-extract, audits, and the new series rescan (which is why #740 "never
applied"). Three fixes:
- _run_bounded: Popen + bounded post-kill reap; if the child is unkillable
  (D-state) we stop waiting and re-raise TimeoutExpired, freeing the slot. The
  orphan is reaped by the OS once its syscall clears.
- backup_db dumps to a LOCAL temp file then moves the finished .sql to the
  (NFS) _backups dir — pg_dump's long phase is now a DB-socket wait + local
  writes (killable) instead of an NFS write that hangs. backup_images keeps
  bounded-kill (too big to stage locally).
- recover_stalled_backup_runs: split the stall window — db 40 min (was sharing
  images' 7h), so a hung DB backup is flipped to error promptly.

#740 — Standardize tag casing showed "0 groups to change" the instant it was
clicked: onNormCommit overwrote the preview with zeros. Keep the real preview
visible and disable the button while queued; backend apply was already correct.

Tests: fake subprocess.Popen alongside run; bounded-kill fail-fast; local-temp
target; per-kind stall sweep.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/test_backup_service.py

@@ -5,6 +5,7 @@ without external binaries. The real subprocess behavior is exercised
 implicitly via the Celery task tests in test_tasks_backup.py.
 """
 import json
+import subprocess
 from pathlib import Path
 
 import pytest
@@ -16,9 +17,9 @@ pytestmark = pytest.mark.integration
 
 @pytest.fixture
 def fake_subprocess(monkeypatch):
-    """Replace subprocess.run with a fake that writes a sentinel to
-    the target path (for pg_dump's -f, for tar's -cf). Captures all
-    calls in a list."""
+    """Fake both subprocess.run (restore path) AND subprocess.Popen (the
+    bounded-kill backup path) so tests run without external binaries. Each
+    writes the target sentinel and records the cmd in a shared list."""
     calls = []
 
     class _FakeProc:
@@ -26,17 +27,33 @@ def fake_subprocess(monkeypatch):
         stdout = b""
         stderr = b""
 
-    def _fake_run(cmd, **kwargs):
-        calls.append(list(cmd))
+    def _write_sentinel(cmd):
         if cmd[0] == "pg_dump":
             i = cmd.index("-f")
             Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
         elif cmd[0] == "tar" and "-cf" in cmd:
             i = cmd.index("-cf")
             Path(cmd[i + 1]).write_bytes(b"fake tar payload")
+
+    def _fake_run(cmd, **kwargs):
+        calls.append(list(cmd))
+        _write_sentinel(cmd)
         return _FakeProc()
 
+    class _FakePopen:
+        def __init__(self, cmd, **kwargs):
+            calls.append(list(cmd))
+            self.returncode = 0
+            _write_sentinel(cmd)
+
+        def communicate(self, timeout=None):
+            return (b"", b"")
+
+        def kill(self):
+            self.returncode = -9
+
     monkeypatch.setattr("subprocess.run", _fake_run)
+    monkeypatch.setattr("subprocess.Popen", _FakePopen)
     return calls
 
 
@@ -198,3 +215,41 @@ def test_backups_dir_created_on_first_use(tmp_path):
     d = backup_service._backups_dir(tmp_path)
     assert d.is_dir()
     assert d.name == "_backups"
+
+
+# --- bounded-kill + local-temp (FC #739) -----------------------------
+
+
+def test_backup_db_dumps_to_local_temp_not_nfs_backups_dir(tmp_path, fake_subprocess):
+    """pg_dump must target a LOCAL temp path, not the (NFS) _backups dir —
+    so its long phase can't hang uninterruptibly on an NFS write."""
+    backup_service.backup_db(db_url="postgresql://u@h/d", images_root=tmp_path)
+    cmd = fake_subprocess[0]
+    dump_target = cmd[cmd.index("-f") + 1]
+    assert "_backups" not in dump_target
+    # The finished file still ends up in _backups (moved there).
+    result = backup_service.backup_db(
+        db_url="postgresql://u@h/d", images_root=tmp_path,
+    )
+    assert "_backups" in result["sql_path"]
+
+
+def test_run_bounded_fails_fast_when_unkillable(monkeypatch):
+    """A child stuck in D-state (communicate keeps timing out even after kill)
+    must NOT block the reaper — _run_bounded kills then re-raises promptly."""
+    killed = {"n": 0}
+
+    class _Hang:
+        def __init__(self, cmd, **kwargs):
+            pass
+
+        def communicate(self, timeout=None):
+            raise subprocess.TimeoutExpired(cmd="x", timeout=timeout or 0)
+
+        def kill(self):
+            killed["n"] += 1
+
+    monkeypatch.setattr("subprocess.Popen", _Hang)
+    with pytest.raises(subprocess.TimeoutExpired):
+        backup_service._run_bounded(["pg_dump"], 1)
+    assert killed["n"] == 1

tests/test_tasks_backup.py

@@ -34,16 +34,32 @@ def fake_subprocess_and_images_root(monkeypatch, tmp_path):
         stdout = b""
         stderr = b""
 
-    def _fake_run(cmd, **kwargs):
+    def _sentinel(cmd):
         if cmd[0] == "pg_dump":
             i = cmd.index("-f")
             Path(cmd[i + 1]).write_bytes(b"-- fake pg_dump\n")
         elif cmd[0] == "tar" and "-cf" in cmd:
             i = cmd.index("-cf")
             Path(cmd[i + 1]).write_bytes(b"fake tar payload")
+
+    def _fake_run(cmd, **kwargs):
+        _sentinel(cmd)
         return _FakeProc()
 
+    class _FakePopen:
+        def __init__(self, cmd, **kwargs):
+            self.returncode = 0
+            _sentinel(cmd)
+
+        def communicate(self, timeout=None):
+            return (b"", b"")
+
+        def kill(self):
+            self.returncode = -9
+
+    # backup_db/backup_images go through _run_bounded (Popen); restore via run.
     monkeypatch.setattr("subprocess.run", _fake_run)
+    monkeypatch.setattr("subprocess.Popen", _FakePopen)
 
 
 def _seed_backup(db_sync, *, kind, status, started_at, tag=None,
@@ -89,7 +105,8 @@ async def test_backup_db_task_records_failure_on_subprocess_error(db_sync, monke
 
     def _boom(*a, **kw):
         raise RuntimeError("synthetic pg_dump fail")
-    monkeypatch.setattr("subprocess.run", _boom)
+    # backup_db dumps via _run_bounded → subprocess.Popen.
+    monkeypatch.setattr("subprocess.Popen", _boom)
 
     with pytest.raises(RuntimeError):
         backup_db_task.delay().get()
@@ -237,6 +254,37 @@ def test_prune_backups_never_deletes_running_or_restoring(db_sync):
     assert set(statuses) == {"running", "restoring"}
 
 
+# --- recover_stalled_backup_runs (per-kind threshold, FC #739) -------
+
+
+def test_stall_sweep_flips_db_fast_but_spares_running_images(db_sync):
+    from backend.app.tasks.maintenance import recover_stalled_backup_runs
+
+    now = datetime.now(UTC)
+    # A db backup stuck 50 min → past the 40-min db window → flipped to error.
+    db_id = _seed_backup(
+        db_sync, kind="db", status="running",
+        started_at=now - timedelta(minutes=50),
+    )
+    # An images backup running 50 min is still well under the 7h window → spared.
+    img_id = _seed_backup(
+        db_sync, kind="images", status="running",
+        started_at=now - timedelta(minutes=50),
+    )
+    db_sync.commit()
+
+    recover_stalled_backup_runs.apply().get()
+
+    db_status = db_sync.execute(
+        select(BackupRun.status).where(BackupRun.id == db_id)
+    ).scalar_one()
+    img_status = db_sync.execute(
+        select(BackupRun.status).where(BackupRun.id == img_id)
+    ).scalar_one()
+    assert db_status == "error"
+    assert img_status == "running"
+
+
 # --- backup_db_nightly ----------------------------------------------