bvandeusenandClaude Opus 5.5
ad4f7d9232
Measured on dev (385a579), agent posts cut to five samples still dropped:
flush=8.8s for 990 rows, 6.6s for 680, 8.4s for 1000 — about 110 rows/s,
~9 ms a row, with rules and evaluation near zero. One random read per
row fits a random uuid4 primary key whose index no longer fits in memory:
every new row lands on a random page of it. (Inferred from the rate; the
table and index sizes have not been read.)
- steward.core.ids.time_ordered_id: a UUIDv7-layout string (48-bit ms
timestamp, then random), same 36-char shape as the stored uuid4 ids, so
no migration. Python's uuid7() arrives in 3.14; the image runs 3.13.
- Used for plugin_metrics and its hourly rollup, monitor_results,
audit_events, the docker metrics/hourly/events/logs tables, and the
traefik and unifi time series.
- MAX_SAMPLES_PER_INGEST 5 → 2 (~400 rows) for now: it fits the agent's
10 s even at the old rate, so hosts land data whichever way the new rate
comes out. Raise it once the new rate is measured.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
183 lines
10 KiB
Python
183 lines
10 KiB
Python
# plugins/unifi/models.py
|
|
from __future__ import annotations
|
|
from datetime import datetime, timezone
|
|
from sqlalchemy import Boolean, DateTime, Float, Integer, String, Text
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
from steward.models.base import Base
|
|
from steward.core.ids import time_ordered_id
|
|
|
|
|
|
# ── Time-series (one row per scrape) ──────────────────────────────────────────
|
|
|
|
class UnifiWanStat(Base):
|
|
"""WAN interface health — one row per scrape."""
|
|
__tablename__ = "unifi_wan_stats"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
is_up: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
wan_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
rx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
tx_bytes_rate: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
|
|
|
|
class UnifiClientSnapshot(Base):
|
|
"""Aggregated client counts + top talkers — one row per scrape."""
|
|
__tablename__ = "unifi_client_snapshots"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
total_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
wireless_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
wired_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
guest_clients: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
top_clients_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
|
# JSON: [{"mac","hostname","ip","type","tx_rate","rx_rate","signal","essid"}]
|
|
|
|
|
|
class UnifiWlanStat(Base):
|
|
"""Per-SSID/radio stats — one row per scrape per SSID+radio combination."""
|
|
__tablename__ = "unifi_wlan_stats"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
ssid: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
radio: Mapped[str | None] = mapped_column(String(8), nullable=True) # ng=2.4GHz, na=5GHz, 6e=6GHz
|
|
num_sta: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
tx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
rx_bytes: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
channel: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
satisfaction: Mapped[int | None] = mapped_column(Integer, nullable=True) # 0-100
|
|
|
|
|
|
class UnifiDpiSnapshot(Base):
|
|
"""DPI application/category traffic snapshot — one row per scrape (JSON blob)."""
|
|
__tablename__ = "unifi_dpi_snapshots"
|
|
|
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=time_ordered_id)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
# JSON: [{"cat_id", "cat_name", "rx_bytes", "tx_bytes"}] aggregated across all clients
|
|
by_category_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
|
# JSON: [{"mac", "hostname", "top_cats": [{"cat_name", "bytes"}]}]
|
|
by_client_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")
|
|
|
|
|
|
# ── Upserted inventory (keyed by UniFi ID or MAC) ─────────────────────────────
|
|
|
|
class UnifiDevice(Base):
|
|
"""Network infrastructure devices — upserted by MAC each scrape."""
|
|
__tablename__ = "unifi_devices"
|
|
|
|
mac: Mapped[str] = mapped_column(String(32), primary_key=True)
|
|
name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
model: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
device_type: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
|
ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
state: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
uptime_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
|
|
|
|
class UnifiKnownClient(Base):
|
|
"""All historically seen network clients — upserted by MAC."""
|
|
__tablename__ = "unifi_known_clients"
|
|
|
|
mac: Mapped[str] = mapped_column(String(32), primary_key=True)
|
|
hostname: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
name: Mapped[str | None] = mapped_column(String(255), nullable=True) # user-set alias
|
|
ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
mac_vendor: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
|
note: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
is_blocked: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
first_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
last_seen: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
|
|
|
|
class UnifiEvent(Base):
|
|
"""Site event log — upserted by UniFi event ID to avoid duplicates."""
|
|
__tablename__ = "unifi_events"
|
|
|
|
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
event_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
event_key: Mapped[str] = mapped_column(String(64), nullable=False) # e.g. EVT_WU_Connected
|
|
category: Mapped[str] = mapped_column(String(16), nullable=False) # wlan, wired, ap, ids, wan, system
|
|
message: Mapped[str] = mapped_column(Text, nullable=False)
|
|
source_mac: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
|
source_ip: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
details_json: Mapped[str] = mapped_column(Text, nullable=False, default="{}")
|
|
|
|
|
|
class UnifiAlarm(Base):
|
|
"""Active alarms — upserted by UniFi alarm ID."""
|
|
__tablename__ = "unifi_alarms"
|
|
|
|
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
alarm_time: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
|
alarm_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
message: Mapped[str] = mapped_column(Text, nullable=False)
|
|
archived: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
|
|
|
|
class UnifiSpeedtestResult(Base):
|
|
"""WAN speed test results — upserted by run timestamp."""
|
|
__tablename__ = "unifi_speedtest_results"
|
|
|
|
run_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), primary_key=True)
|
|
download_mbps: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
upload_mbps: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
latency_ms: Mapped[float | None] = mapped_column(Float, nullable=True)
|
|
server_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
|
|
|
|
# ── Config inventory (upserted by UniFi ID) ───────────────────────────────────
|
|
|
|
class UnifiNetwork(Base):
|
|
"""VLAN/network configurations — upserted by UniFi ID."""
|
|
__tablename__ = "unifi_networks"
|
|
|
|
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
purpose: Mapped[str | None] = mapped_column(String(32), nullable=True) # corporate, guest, vlan-only
|
|
vlan: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
ip_subnet: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
|
dhcp_enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
is_guest: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
|
|
|
|
class UnifiPortForward(Base):
|
|
"""Port forwarding rules — upserted by UniFi ID."""
|
|
__tablename__ = "unifi_port_forwards"
|
|
|
|
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
name: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
proto: Mapped[str] = mapped_column(String(8), nullable=False, default="tcp")
|
|
dst_port: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
fwd_ip: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
fwd_port: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
src_filter: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|
|
|
|
|
|
class UnifiFwRule(Base):
|
|
"""Firewall rules — upserted by UniFi ID."""
|
|
__tablename__ = "unifi_fw_rules"
|
|
|
|
unifi_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
|
ruleset: Mapped[str] = mapped_column(String(16), nullable=False) # WAN_IN, LAN_IN, etc.
|
|
rule_index: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
|
action: Mapped[str] = mapped_column(String(16), nullable=False) # accept, drop, reject
|
|
protocol: Mapped[str | None] = mapped_column(String(16), nullable=True)
|
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
|
src_address: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
dst_address: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
|
scraped_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=lambda: datetime.now(timezone.utc))
|