feat(docker): persist pushed container logs into docker_logs [M79 step 3]
CI / lint (push) Successful in 2s
CI / unit (push) Successful in 41s
CI / integration (push) Successful in 2m25s
CI / publish (push) Successful in 8s

Ingest side of container logs. The host_agent ingest route extracts
sample["docker_logs"] as time-series batches (append every line, each carrying
its own Docker ts with recorded_at as fallback) and hands them to the docker
capability alongside the existing container/swarm/disk data — still under the
begin_nested SAVEPOINT so a logs failure can't sink host metrics.

- ingest.py: _persist_logs + pure _log_rows(batches, host_id) that shapes/filters
  records (drops the _steward truncation marker, malformed + lineless records,
  normalises unknown stream → stdout); persist_host_docker gains a logs= param
- routes.py: accumulate docker_log_batches, add to the guard + capability call
- unit test for _log_rows shaping/filtering; integration push→store→query test

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01CAGR73DUowdVFVvYzLXC5C
This commit is contained in:
2026-07-19 18:46:31 -04:00
parent 45565b2c01
commit a8de3570fe
4 changed files with 126 additions and 4 deletions
+38
View File
@@ -172,6 +172,44 @@ def test_persist_scopes_containers_by_host(app):
assert enrich == ("healthy", 2, "web", 1000) # enrichment round-trips
@_NEEDS_DB
def test_persist_logs_stores_lines(app):
"""Pushed container logs land in docker_logs, host-scoped, one row per line,
ordered by their own ts; the agent's _steward truncation marker is dropped."""
from sqlalchemy import text
from steward.models.hosts import Host
persist = _persist_fn(app)
now = datetime.now(timezone.utc)
log_batches = [(now, [
{"container": "web", "stream": "stdout",
"ts": "2026-01-01T00:00:01+00:00", "line": "started"},
{"container": "web", "stream": "stderr",
"ts": "2026-01-01T00:00:02+00:00", "line": "warn: x"},
{"container": "_steward", "stream": "stderr", "ts": None, "line": "truncated"},
])]
async def _go():
async with app.db_sessionmaker() as s:
async with s.begin():
await s.execute(text("DELETE FROM docker_logs"))
h = Host(id=str(uuid.uuid4()), name="loghost", address="10.0.0.9")
s.add(h)
await s.flush()
hid = h.id
# snapshots empty; logs passed as the 6th positional arg.
await persist(s, h, [], None, None, log_batches)
rows = (await s.execute(text(
"SELECT container_name, stream, line FROM docker_logs "
"WHERE host_id = :h ORDER BY ts"), {"h": hid})).all()
return [tuple(r) for r in rows]
assert asyncio.run(_go()) == [
("web", "stdout", "started"),
("web", "stderr", "warn: x"),
]
@_NEEDS_DB
def test_large_memory_values_persist_as_bigint(app):
"""A container using >2^31 bytes of RAM must persist. Regression: mem_usage_bytes
@@ -65,3 +65,35 @@ def test_no_event_when_unchanged():
old = {"web": {"status": "running", "health": "healthy", "oom_killed": False, "exit_code": None}}
events = _derive_events(old, [_c("web", "running", health="healthy")])
assert events == []
# ── container-log row shaping (m79, pure) ─────────────────────────────────────
def test_log_rows_shapes_and_filters():
from datetime import datetime, timezone
from plugins.docker.ingest import _log_rows
rec_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
batches = [(rec_at, [
{"container": "web", "stream": "stdout",
"ts": "2026-01-01T00:00:01+00:00", "line": "hello"},
{"container": "web", "stream": "stderr", "ts": None, "line": "no-ts"},
{"container": "_steward", "stream": "stderr",
"ts": None, "line": "truncated"}, # advisory marker → dropped
{"container": "web", "stream": "weird", "ts": None, "line": "bad-stream"},
{"container": "", "line": "x"}, # no name → dropped
{"container": "web", "ts": None, "line": None}, # no line → dropped
"not-a-dict", # malformed → dropped
])]
rows = list(_log_rows(batches, "host-1"))
assert [r["line"] for r in rows] == ["hello", "no-ts", "bad-stream"]
assert rows[0]["host_id"] == "host-1" and rows[0]["container_name"] == "web"
assert rows[1]["ts"] == rec_at # None ts → recorded_at fallback
assert rows[2]["stream"] == "stdout" # unknown stream normalised
def test_log_rows_skips_non_list_records():
from datetime import datetime, timezone
from plugins.docker.ingest import _log_rows
rec_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
assert list(_log_rows([(rec_at, None)], "h")) == []