From 9f26258c8e3ef559e1d4403e1635155d46ace4ea Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 12 Oct 2026 06:55:24 -0400 Subject: [PATCH] feat(core): one Prometheus text parser, with types; Traefik reads through it (#5667) steward/core/prometheus.py parses the 0.0.4 exposition into families keyed by name, each with its declared TYPE, so a scraper can tell a counter from a gauge. A histogram or summary owns its _bucket/_sum/_count samples. Label values are unescaped, and commas or braces inside quotes stay part of the value. NaN and +/-Inf are read, timestamps are ignored, and a malformed line is skipped. Traefik's parse_prometheus is now a thin reshaping of it. Co-Authored-By: Claude Opus 5.5 --- plugins/traefik/scraper.py | 34 ++----- steward/core/prometheus.py | 174 ++++++++++++++++++++++++++++++++++ tests/core/test_prometheus.py | 117 +++++++++++++++++++++++ 3 files changed, 298 insertions(+), 27 deletions(-) create mode 100644 steward/core/prometheus.py create mode 100644 tests/core/test_prometheus.py diff --git a/plugins/traefik/scraper.py b/plugins/traefik/scraper.py index 87cbdc5..7af2538 100644 --- a/plugins/traefik/scraper.py +++ b/plugins/traefik/scraper.py @@ -18,6 +18,8 @@ from datetime import datetime, timezone import httpx +from steward.core import prometheus + # ────────────────────────────────────────────────────────────────────────────── # Prometheus text format parser @@ -26,36 +28,14 @@ import httpx # Parsed sample: {metric_name: {frozenset(label_items): float}} ParsedMetrics = dict[str, dict[frozenset, float]] -_SAMPLE_RE = re.compile( - r'^([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+' - r'([+-]?(?:\d+\.?\d*|\.\d+)(?:[eE][+-]?\d+)?|[+-]?Inf|NaN)' -) -_LABEL_RE = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="([^"\\]*(?:\\.[^"\\]*)*)"') - def parse_prometheus(text: str) -> ParsedMetrics: - """Parse Prometheus text exposition format into nested dicts. - - Returns: {metric_name: {frozenset({(label, value), ...}): float}} - """ + """The exposition keyed by sample name, then by its label set — the shape + the per-router arithmetic below reads. Parsing is steward.core.prometheus.""" metrics: ParsedMetrics = {} - for line in text.splitlines(): - line = line.strip() - if not line or line.startswith("#"): - continue - m = _SAMPLE_RE.match(line) - if not m: - continue - name, labels_str, value_str = m.groups() - try: - value = float(value_str) - except ValueError: - continue - labels = frozenset( - (lm.group(1), lm.group(2)) - for lm in _LABEL_RE.finditer(labels_str or "") - ) - metrics.setdefault(name, {})[labels] = value + for family in prometheus.parse(text).values(): + for sample in family.samples: + metrics.setdefault(sample.name, {})[frozenset(sample.labels)] = sample.value return metrics diff --git a/steward/core/prometheus.py b/steward/core/prometheus.py new file mode 100644 index 0000000..6ba860f --- /dev/null +++ b/steward/core/prometheus.py @@ -0,0 +1,174 @@ +"""Prometheus text exposition format (0.0.4): one parser for every scraper. + +The Traefik plugin and the Prometheus plugin both read this format. parse() +returns families keyed by name, each carrying its declared type, so a caller +can tell a counter (store a rate) from a gauge (store the value) without +guessing from the name. + +Samples are grouped under the family their TYPE line declares: a histogram +"x" owns x_bucket, x_sum and x_count; a summary owns x, x_sum and x_count. +A sample with no TYPE line is its own untyped family. Timestamps are read +past and ignored; a line that doesn't parse is skipped, never fatal. +""" +from __future__ import annotations + +import math +import re +from dataclasses import dataclass, field + +_NAME = re.compile(r"[a-zA-Z_:][a-zA-Z0-9_:]*") +_LABEL_NAME = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*") + +TYPES = ("counter", "gauge", "histogram", "summary", "untyped") + +# Suffixes a histogram or summary's samples carry beyond the family name. +_CHILD_SUFFIXES = { + "counter": ("_total",), # OpenMetrics declares a counter without it + "histogram": ("_bucket", "_sum", "_count"), + "summary": ("_sum", "_count"), +} + +_ESCAPES = {"\\": "\\", '"': '"', "n": "\n"} + + +@dataclass(frozen=True) +class Sample: + name: str + labels: tuple[tuple[str, str], ...] # in the order the line gave them + value: float + + def label_dict(self) -> dict[str, str]: + return dict(self.labels) + + +@dataclass +class Family: + name: str + type: str = "untyped" + help: str = "" + samples: list[Sample] = field(default_factory=list) + + +def parse(text: str) -> dict[str, Family]: + """{family name: Family}, in the order families first appear.""" + families: dict[str, Family] = {} + for raw in text.splitlines(): + line = raw.strip() + if not line: + continue + if line.startswith("#"): + _read_comment(line, families) + continue + sample = parse_sample(line) + if sample is None: + continue + fam = families.get(_owner(sample.name, families)) + if fam is None: + fam = families[sample.name] = Family(sample.name) + fam.samples.append(sample) + return families + + +def _read_comment(line: str, families: dict[str, Family]) -> None: + parts = line[1:].strip().split(None, 2) + if len(parts) < 2 or parts[0] not in ("TYPE", "HELP"): + return + name = parts[1] + if not _NAME.fullmatch(name): + return + fam = families.setdefault(name, Family(name)) + rest = parts[2].strip() if len(parts) > 2 else "" + if parts[0] == "TYPE": + fam.type = rest if rest in TYPES else "untyped" + else: + fam.help = rest.replace("\\n", "\n").replace("\\\\", "\\") + + +def _owner(sample_name: str, families: dict[str, Family]) -> str: + """The declared family a sample belongs to: its own name, or the + histogram/summary whose name it extends.""" + if sample_name in families: + return sample_name + for kind, suffixes in _CHILD_SUFFIXES.items(): + for suffix in suffixes: + if sample_name.endswith(suffix): + base = sample_name[: -len(suffix)] + fam = families.get(base) + if fam is not None and fam.type == kind: + return base + return sample_name + + +def parse_sample(line: str) -> Sample | None: + """One sample line, or None when it isn't one.""" + m = _NAME.match(line) + if not m: + return None + name, pos = m.group(), m.end() + labels: tuple[tuple[str, str], ...] = () + if pos < len(line) and line[pos] == "{": + read = _read_labels(line, pos + 1) + if read is None: + return None + labels, pos = read + rest = line[pos:].split() + if not rest or (pos < len(line) and not line[pos].isspace()): + return None + value = parse_value(rest[0]) + if value is None: + return None + return Sample(name, labels, value) + + +def _read_labels(line: str, pos: int) -> tuple[tuple[tuple[str, str], ...], int] | None: + """Labels from just after "{" to just after "}", values unescaped. + Commas and braces inside a quoted value are part of the value.""" + out: list[tuple[str, str]] = [] + n = len(line) + while True: + while pos < n and line[pos] in " \t,": + pos += 1 + if pos >= n: + return None + if line[pos] == "}": + return tuple(out), pos + 1 + m = _LABEL_NAME.match(line, pos) + if not m: + return None + key, pos = m.group(), m.end() + while pos < n and line[pos] in " \t": + pos += 1 + if line[pos:pos + 2] != '="': + return None + pos += 2 + chars: list[str] = [] + while True: + if pos >= n: + return None + c = line[pos] + if c == "\\" and pos + 1 < n: + nxt = line[pos + 1] + chars.append(_ESCAPES.get(nxt, "\\" + nxt)) + pos += 2 + continue + if c == '"': + pos += 1 + break + chars.append(c) + pos += 1 + out.append((key, "".join(chars))) + + +def parse_value(text: str) -> float | None: + """A sample value: a float, NaN, +Inf or -Inf (Go's spellings too).""" + lowered = text.lower() + if lowered in ("+inf", "inf"): + return math.inf + if lowered == "-inf": + return -math.inf + if lowered == "nan": + return math.nan + try: + return float(text) + except ValueError: + return None diff --git a/tests/core/test_prometheus.py b/tests/core/test_prometheus.py new file mode 100644 index 0000000..e7cf72b --- /dev/null +++ b/tests/core/test_prometheus.py @@ -0,0 +1,117 @@ +"""The Prometheus text parser every scraper shares (steward/core/prometheus.py).""" +import math + +from steward.core import prometheus + +# Shaped like Fabled Archives' /metrics: HELP and TYPE lines, info and state +# gauges with several labels, counters, and an untyped stray. +ARCHIVES = """\ +# HELP fabledarchives_build_info Build of the running server. +# TYPE fabledarchives_build_info gauge +fabledarchives_build_info{version="2026.10.11.1458",channel="dev",commit="c49b470"} 1 +# TYPE fabledarchives_backup_target_state gauge +fabledarchives_backup_target_state{target="Steward",target_id="3",state="fresh"} 1 +fabledarchives_backup_target_state{target="Steward",target_id="3",state="late"} 0 +# TYPE fabledarchives_http_requests_total counter +fabledarchives_http_requests_total{route="GET /api/files",status="200"} 1027 +fabledarchives_http_requests_total{route="GET /api/files",status="404"} 3 +stray_metric 4.5 +""" + + +def test_families_carry_their_declared_type(): + fams = prometheus.parse(ARCHIVES) + assert fams["fabledarchives_build_info"].type == "gauge" + assert fams["fabledarchives_build_info"].help == "Build of the running server." + assert fams["fabledarchives_http_requests_total"].type == "counter" + assert fams["stray_metric"].type == "untyped" + assert list(fams) == [ + "fabledarchives_build_info", + "fabledarchives_backup_target_state", + "fabledarchives_http_requests_total", + "stray_metric", + ] + + +def test_labels_keep_their_order_and_values(): + state = prometheus.parse(ARCHIVES)["fabledarchives_backup_target_state"] + assert [s.labels for s in state.samples] == [ + (("target", "Steward"), ("target_id", "3"), ("state", "fresh")), + (("target", "Steward"), ("target_id", "3"), ("state", "late")), + ] + assert [s.value for s in state.samples] == [1.0, 0.0] + reqs = prometheus.parse(ARCHIVES)["fabledarchives_http_requests_total"] + assert reqs.samples[0].label_dict() == {"route": "GET /api/files", "status": "200"} + + +def test_escaped_label_values_and_separators_inside_quotes(): + text = 'm{a="say \\"hi\\"",b="back\\\\slash",c="two\\nlines",d="x,y}z"} 2\n' + (sample,) = prometheus.parse(text)["m"].samples + assert sample.label_dict() == { + "a": 'say "hi"', "b": "back\\slash", "c": "two\nlines", "d": "x,y}z", + } + + +def test_histogram_and_summary_samples_belong_to_their_family(): + text = """\ +# TYPE req_seconds histogram +req_seconds_bucket{le="0.1"} 3 +req_seconds_bucket{le="+Inf"} 5 +req_seconds_sum 0.9 +req_seconds_count 5 +# TYPE rpc summary +rpc{quantile="0.5"} 0.2 +rpc_sum 7 +rpc_count 30 +""" + fams = prometheus.parse(text) + assert set(fams) == {"req_seconds", "rpc"} + assert [s.name for s in fams["req_seconds"].samples] == [ + "req_seconds_bucket", "req_seconds_bucket", "req_seconds_sum", "req_seconds_count", + ] + assert math.isinf(fams["req_seconds"].samples[1].value) + assert [s.name for s in fams["rpc"].samples] == ["rpc", "rpc_sum", "rpc_count"] + + +def test_an_openmetrics_counter_owns_its_total_sample(): + fams = prometheus.parse("# TYPE jobs counter\njobs_total 12\n") + assert list(fams) == ["jobs"] + assert fams["jobs"].type == "counter" + assert fams["jobs"].samples[0].name == "jobs_total" + + +def test_special_values_and_timestamps(): + text = "a NaN\nb +Inf\nc -Inf\nd 1.5e3 1700000000000\n" + fams = prometheus.parse(text) + assert math.isnan(fams["a"].samples[0].value) + assert fams["b"].samples[0].value == math.inf + assert fams["c"].samples[0].value == -math.inf + assert fams["d"].samples[0].value == 1500.0 + + +def test_malformed_lines_are_skipped_not_fatal(): + text = """\ +good 1 +bad-name 2 +unterminated{a="x 3 +nolabelvalue{a=} 4 +novalue +spaced{a="1"}5 +# TYPE weird nonsense +weird 6 +good_too{a="b"} 7 +""" + fams = prometheus.parse(text) + assert set(fams) == {"good", "weird", "good_too"} + assert fams["weird"].type == "untyped" + assert fams["good_too"].samples[0].value == 7.0 + + +def test_traefik_reads_through_the_shared_parser(): + from plugins.traefik.scraper import parse_prometheus + + out = parse_prometheus( + 'traefik_service_requests_total{service="web@docker",code="200"} 10\n' + ) + key = frozenset({("service", "web@docker"), ("code", "200")}) + assert out == {"traefik_service_requests_total": {key: 10.0}}