diff --git a/plugins/docker/migrations/versions/docker_008_bigint_mem.py b/plugins/docker/migrations/versions/docker_008_bigint_mem.py new file mode 100644 index 0000000..b11b9ad --- /dev/null +++ b/plugins/docker/migrations/versions/docker_008_bigint_mem.py @@ -0,0 +1,33 @@ +"""Widen memory byte columns to BIGINT + +docker_metrics.mem_usage_bytes and docker_containers.mem_usage_bytes / +mem_limit_bytes were int4, which overflows for any container using >2.1 GB +(asyncpg: "value out of int32 range"), failing the whole ingest batch. Widen to +BIGINT — a safe in-place int4→int8 promotion, no data rewrite. + +Revision ID: docker_008_bigint_mem +Revises: docker_007_disk_usage +Create Date: 2026-06-20 +""" +from typing import Sequence, Union +from alembic import op +import sqlalchemy as sa + +revision: str = "docker_008_bigint_mem" +down_revision: Union[str, None] = "docker_007_disk_usage" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.BigInteger()) + op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.BigInteger()) + op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.BigInteger()) + + +def downgrade() -> None: + # Narrowing back to int4 will error if any stored value exceeds 2^31 — that's + # the very condition this migration fixed, so downgrade is best-effort. + op.alter_column("docker_containers", "mem_limit_bytes", type_=sa.Integer()) + op.alter_column("docker_containers", "mem_usage_bytes", type_=sa.Integer()) + op.alter_column("docker_metrics", "mem_usage_bytes", type_=sa.Integer()) diff --git a/plugins/docker/models.py b/plugins/docker/models.py index 04cb377..60f625d 100644 --- a/plugins/docker/models.py +++ b/plugins/docker/models.py @@ -29,8 +29,10 @@ class DockerContainer(Base): status: Mapped[str] = mapped_column(String(32), nullable=False, default="unknown") # running | stopped | paused | exited | dead cpu_pct: Mapped[float | None] = mapped_column(Float, nullable=True) - mem_usage_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) - mem_limit_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) + # BigInteger: a container's memory usage/limit routinely exceeds 2^31 bytes + # (~2.1 GB), which overflows int4 and fails the insert. + mem_usage_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + mem_limit_bytes: Mapped[int | None] = mapped_column(BigInteger, nullable=True) mem_pct: Mapped[float | None] = mapped_column(Float, nullable=True) restart_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) ports_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]") @@ -78,7 +80,8 @@ class DockerMetric(Base): ) cpu_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) mem_pct: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) - mem_usage_bytes: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # BigInteger: per-container memory usage exceeds 2^31 bytes (~2.1 GB) routinely. + mem_usage_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) # Per-container history lookups filter on (host_id, container_name) then sort # by time — a composite index keeps the rows() sparkline queries cheap. diff --git a/tests/integration/test_docker.py b/tests/integration/test_docker.py index 722e9ef..fa1fcac 100644 --- a/tests/integration/test_docker.py +++ b/tests/integration/test_docker.py @@ -151,6 +151,46 @@ def test_persist_scopes_containers_by_host(app): @_NEEDS_DB +def test_large_memory_values_persist_as_bigint(app): + """A container using >2^31 bytes of RAM must persist. Regression: mem_usage_bytes + / mem_limit_bytes were int4 and overflowed (asyncpg 'value out of int32 range'), + failing the whole ingest batch for any host with a >2.1 GB container.""" + from sqlalchemy import text + from steward.models.hosts import Host + + persist = _persist_fn(app) + now = datetime.now(timezone.utc) + big_usage = 8_185_077_760 # ~8.18 GB — well past int4 max (2_147_483_647) + big_limit = 17_179_869_184 # 16 GB + snapshot = [(now, [{ + "name": "bigmem", "container_id": "deadbeef", "image": "postgres", + "status": "running", "cpu_pct": 1.0, "mem_pct": 50.0, + "mem_usage_bytes": big_usage, "mem_limit_bytes": big_limit, + "ports": [], "started_at": None, + }])] + + async def _go(): + async with app.db_sessionmaker() as s: + async with s.begin(): + await s.execute(text("DELETE FROM docker_metrics")) + await s.execute(text("DELETE FROM docker_containers")) + h = Host(id=str(uuid.uuid4()), name="bigmemhost", address="10.0.0.9") + s.add(h) + await s.flush() + await persist(s, h, snapshot) + row = (await s.execute(text( + "SELECT mem_usage_bytes, mem_limit_bytes FROM docker_containers " + "WHERE name = 'bigmem'"))).first() + metric = (await s.execute(text( + "SELECT mem_usage_bytes FROM docker_metrics " + "WHERE container_name = 'bigmem'"))).scalar() + return row, metric + + row, metric = asyncio.run(_go()) + assert row == (big_usage, big_limit) # current-state row round-trips + assert metric == big_usage # time-series row round-trips + + def test_lifecycle_events_derived_across_snapshots(app): from sqlalchemy import text from steward.models.hosts import Host