feat(agent): in-UI log console + a real styling pass on the control page
- logbuf.py: bounded in-memory log ring buffer + a logging.Handler on the root logger; GET /logs serves it; the control page polls it into a console pane — so runs are monitorable without `docker logs`. worker now logs autoscale moves (one line per change, with jobs/s + util + VRAM) and job failures (job + image + reason); detectors already log load/disable. - Restyled the whole control page: a proper dark layout with a header + live connection pill, cards (Control / Status / Logs), a styled Auto switch + worker stepper, status tiles, separate GPU-util and VRAM meters, and the log console. No longer feels like an afterthought; all the existing control hooks are preserved. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ttrj5P7upUTueSfoJcxEqa
This commit is contained in:
@@ -9,6 +9,7 @@ That's the dial the operator turns to trade desktop responsiveness for speed.
|
||||
Stop (or shrinking the pool) RELEASES a slot's still-leased jobs immediately so
|
||||
orphaned work is re-picked at once rather than waiting out the lease.
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import numpy as np
|
||||
@@ -58,6 +59,8 @@ UTIL_HI = 96 # GPU util% considered saturated
|
||||
TPUT_MARGIN = 0.10 # a step up must beat the baseline by this to "help"
|
||||
REPROBE_TICKS = 5 # ticks to hold after settling before re-probing up
|
||||
|
||||
log = logging.getLogger("fc_agent.worker")
|
||||
|
||||
|
||||
class _Slot:
|
||||
"""One worker loop. `inflight` = jobs leased but not yet processed, so a
|
||||
@@ -230,6 +233,7 @@ class Worker:
|
||||
dt = max(1e-3, now - prev_t)
|
||||
tput = (self.processed - prev_p) / dt
|
||||
prev_p, prev_t = self.processed, now
|
||||
t0 = self._target
|
||||
|
||||
g = gpumod.read_gpu() or {}
|
||||
mt = g.get("mem_total_mb") or 0
|
||||
@@ -260,6 +264,12 @@ class Worker:
|
||||
else:
|
||||
base_tput = None # settled → re-probe next cycle
|
||||
|
||||
if self._target != t0:
|
||||
log.info(
|
||||
"autoscale: %d→%d workers (%.2f jobs/s · util %d%% · vram %d%%)",
|
||||
t0, self._target, tput, util, round(vram * 100),
|
||||
)
|
||||
|
||||
def _ensure_embedder(self, model_name: str):
|
||||
if self._embedder is not None:
|
||||
return self._embedder
|
||||
@@ -400,15 +410,21 @@ class Worker:
|
||||
# effort; no-ops if the server is still down, then the server's
|
||||
# orphan-recovery reclaims it) and signal the loop to wait.
|
||||
self._bump(transient=1)
|
||||
log.info("curator unreachable — released job %s, backing off",
|
||||
job.get("job_id"))
|
||||
self.client.release([job["job_id"]])
|
||||
return False
|
||||
# A job-specific HTTP fault (404 image gone, 400) → fail it so it
|
||||
# doesn't re-lease forever.
|
||||
self._bump(errors=1)
|
||||
log.warning("job %s (image %s) failed: %s",
|
||||
job.get("job_id"), job.get("image_id"), str(exc)[:200])
|
||||
self.client.fail(job["job_id"], str(exc)[:500])
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 — a genuine job fault: report it
|
||||
self._bump(errors=1)
|
||||
log.warning("job %s (image %s) failed: %s",
|
||||
job.get("job_id"), job.get("image_id"), str(exc)[:200])
|
||||
self.client.fail(job["job_id"], str(exc)[:500])
|
||||
return True
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user