feat(ansible): runner robustness — cancel, concurrency, structured results, retention

Closes #550 (all four):

- Cancellation: track live subprocesses; POST /ansible/runs/<id>/cancel
  (operator) SIGTERMs then SIGKILLs after a grace; new 'cancelled' status
  (+ migration 0019, ALTER TYPE in autocommit). Queued runs cancel cleanly
  before launch. Cancel button on run detail.
- Concurrency: global semaphore (ansible.max_concurrent_runs, default 3,
  Settings→Ansible) caps simultaneous runs; excess show 'queued' (new status)
  until a slot frees. Semaphore bound lazily per running loop.
- Structured results: parse PLAY RECAP into per-host ok/changed/unreachable/
  failed/skipped + capture failed-task lines, stored in new results JSON
  column (migration 0020); rendered as a host-summary table on run detail.
  Keeps live streaming (no json-callback swap).
- Retention: full output written to a persistent log artifact
  (/data/ansible/runs/<id>.log, env-overridable) beyond the 1 MB DB cap and
  across restarts; in-memory replay buffer bounded + GC'd after completion;
  Download-log route. Boot reconciliation now also sweeps stale 'queued'.

Unit tests for recap parsing + cancel flagging. Status colors updated across
run list / detail / schedules.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

steward/templates/ansible/index.html

@@ -27,7 +27,7 @@
     </thead>
     <tbody>
     {% for run in runs %}
-    {% set status_color = {"running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %}
+    {% set status_color = {"queued": "var(--text-dim)", "running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)", "cancelled": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %}
     <tr>
         <td style="font-size:0.9rem;">{{ run.playbook_path }}</td>
         <td style="color:var(--text-muted);font-size:0.9rem;">{{ run.source_name }}</td>

steward/templates/ansible/run_detail.html

@@ -6,9 +6,17 @@
 <div style="display:flex;align-items:center;gap:1rem;margin-bottom:1.5rem;">
     <a href="/ansible/" class="btn btn-ghost btn-sm">← Runs</a>
     <h1 class="page-title" style="margin-bottom:0;">Run Detail</h1>
+    <div style="margin-left:auto;display:flex;gap:0.5rem;">
+        {% if run.status.value in ("running", "queued") and session.user_role in ("operator", "admin") %}
+        <form method="post" action="/ansible/runs/{{ run.id }}/cancel" onsubmit="return confirm('Cancel this run?');">
+            <button type="submit" class="btn btn-sm btn-danger">Cancel run</button>
+        </form>
+        {% endif %}
+        <a href="/ansible/runs/{{ run.id }}/log" class="btn btn-sm btn-ghost">Download log</a>
+    </div>
 </div>
 
-{% set status_color = {"running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %}
+{% set status_color = {"queued": "var(--text-dim)", "running": "var(--yellow)", "success": "var(--green)", "failed": "var(--red)", "interrupted": "var(--orange)", "cancelled": "var(--orange)"}.get(run.status.value, "var(--text-muted)") %}
 <div class="card">
     <div style="display:grid;grid-template-columns:auto 1fr;gap:0.4rem 1rem;font-size:0.9rem;margin-bottom:1rem;">
         <span style="color:var(--text-muted);">ID</span><span style="color:var(--text-dim);font-family:monospace;">{{ run.id }}</span>
@@ -38,11 +46,41 @@
     </div>
 </div>
 
+{% if run.results and run.results.hosts %}
+<div class="card">
+    <div class="section-title" style="margin-bottom:0.6rem;">Host summary</div>
+    <table class="table">
+        <thead><tr>
+            <th>Host</th>
+            <th style="text-align:center;">ok</th><th style="text-align:center;">changed</th>
+            <th style="text-align:center;">unreachable</th><th style="text-align:center;">failed</th>
+            <th style="text-align:center;">skipped</th>
+        </tr></thead>
+        <tbody>
+        {% for host, c in run.results.hosts.items() %}
+        <tr>
+            <td style="font-family:ui-monospace,monospace;">{{ host }}</td>
+            <td style="text-align:center;">{{ c.ok }}</td>
+            <td style="text-align:center;color:{% if c.changed %}var(--yellow){% else %}var(--text-dim){% endif %};">{{ c.changed }}</td>
+            <td style="text-align:center;color:{% if c.unreachable %}var(--red){% else %}var(--text-dim){% endif %};">{{ c.unreachable }}</td>
+            <td style="text-align:center;color:{% if c.failed %}var(--red){% else %}var(--text-dim){% endif %};">{{ c.failed }}</td>
+            <td style="text-align:center;color:var(--text-dim);">{{ c.skipped }}</td>
+        </tr>
+        {% endfor %}
+        </tbody>
+    </table>
+    {% if run.results.failures %}
+    <div class="section-title" style="margin:1rem 0 0.4rem;">Failures</div>
+    <pre style="margin:0;padding:0.75rem;background:var(--bg);border-radius:4px;font-size:0.75rem;color:#ff9090;max-height:240px;overflow:auto;white-space:pre-wrap;">{{ run.results.failures | join("\n") }}</pre>
+    {% endif %}
+</div>
+{% endif %}
+
 <div class="card" style="padding:0;">
     <div style="padding:0.75rem 1rem;background:var(--bg-elevated);border-bottom:1px solid var(--border);">
         <span style="color:var(--text-muted);font-size:0.85rem;">Output</span>
     </div>
-    {% if run.status.value == "running" %}
+    {% if run.status.value in ("running", "queued") %}
     <pre id="live-output"
          style="margin:0;padding:1rem;background:var(--bg);font-size:0.78rem;color:var(--green);max-height:600px;overflow-y:auto;white-space:pre-wrap;">{{ run.output or "" }}</pre>
     <div id="run-done-status" style="padding:0.5rem 1rem;font-size:0.85rem;color:var(--text-muted);">

steward/templates/ansible/schedules.html

@@ -42,7 +42,7 @@
       {% if s.last_error %}
         <span style="color:var(--red);" title="{{ s.last_error }}">launch error</span>
       {% elif row.last_run %}
-        {% set sc = {"running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %}
+        {% set sc = {"queued":"var(--text-dim)","running":"var(--yellow)","success":"var(--green)","failed":"var(--red)","interrupted":"var(--orange)","cancelled":"var(--orange)"}.get(row.last_run.status.value, "var(--text-muted)") %}
         <a href="/ansible/runs/{{ row.last_run.id }}" style="color:{{ sc }};font-weight:600;">{{ row.last_run.status.value }}</a>
         <span style="color:var(--text-dim);">· {{ s.last_run_at.strftime("%m-%d %H:%M") }}</span>
       {% else %}

steward/templates/settings/ansible.html

@@ -62,6 +62,13 @@
           Enforce SSH host-key checking <span style="color:var(--text-muted);font-size:0.78rem;">(off by default for homelab convenience)</span>
         </label>
       </div>
+      <div class="form-group">
+        <label>Max concurrent runs
+          <span style="color:var(--text-muted);font-size:0.78rem;font-weight:normal;">(extra runs queue; applied on restart)</span>
+        </label>
+        <input type="number" name="max_concurrent_runs" min="1" max="50"
+               value="{{ max_concurrent_runs }}" style="width:7rem;">
+      </div>
       <button type="submit" class="btn">Save credentials</button>
     </form>
   </div>