From 36cc0622cb76631eef6fe29ebd58288a333e8950 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 25 May 2026 20:33:44 -0400 Subject: [PATCH] =?UTF-8?q?fix(importer):=20sanitize=20PostAttachment.ext?= =?UTF-8?q?=20to=20skip=20mangled=20gallery-dl=20URL-encoded=20basenames?= =?UTF-8?q?=20(varchar(32)=20overrun)=20=E2=80=94=20Co-Authored-By:=20Clau?= =?UTF-8?q?de=20Opus=204.7=20(1M=20context)=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/app/services/importer.py | 27 ++++++++++++++++++++++++++- tests/test_importer_attachments.py | 22 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/backend/app/services/importer.py b/backend/app/services/importer.py index f9eb62b..cf4c0c4 100644 --- a/backend/app/services/importer.py +++ b/backend/app/services/importer.py @@ -71,6 +71,31 @@ def is_video(path: Path) -> bool: return path.suffix.lower() in VIDEO_EXTS +def _safe_ext(path: Path) -> str: + """Conservatively extract a file extension for PostAttachment.ext + (varchar(32)). + + gallery-dl produces some filenames with URL-encoded query-string + artifacts embedded into the basename (e.g. + `79507046_media_..._https___www.patreon.com_media-u_Z0FBQUFBQm5q...`). + `Path.suffix` finds the LAST dot and returns everything after, which + in those cases yields a 50+ char "extension" of mostly base64-ish + junk. That blows the column. Operator-flagged 2026-05-25. + + Real extensions are short and alphanumeric. We accept anything ≤ 16 + chars where every post-dot character is alphanumeric; anything else + means the input wasn't a real extension and we return the empty + string. ext is nullable-ish (empty string still satisfies NOT NULL) + and consumers should treat "" as "no known extension". + """ + suffix = path.suffix.lower() + if not suffix or len(suffix) > 16: + return "" + if not all(c.isalnum() for c in suffix[1:]): + return "" + return suffix + + def _mime_for(path: Path) -> str: suffix = path.suffix.lower() image_mimes = { @@ -209,7 +234,7 @@ class Importer: sha256=sha, path=stored, original_filename=source.name, - ext=source.suffix.lower(), + ext=_safe_ext(source), mime=_mime_for(source), size_bytes=source.stat().st_size, )) diff --git a/tests/test_importer_attachments.py b/tests/test_importer_attachments.py index 06b6083..16ec52e 100644 --- a/tests/test_importer_attachments.py +++ b/tests/test_importer_attachments.py @@ -59,3 +59,25 @@ def test_json_sidecar_is_not_attached(importer, import_layout): select(func.count()).select_from(PostAttachment) ).scalar_one() assert n == 0 + + +def test_mangled_filename_extension_is_sanitized(importer, import_layout): + """gallery-dl sometimes URL-encodes a query string into the basename + (`...https___www.patreon.com_media-u_Z0F...`). Python's Path.suffix + returns 50+ chars of base64-ish junk for those, which blows the + PostAttachment.ext varchar(32) column. Operator-flagged 2026-05-25. + The importer should record an empty ext rather than crash.""" + import_root, _ = import_layout + f = ( + import_root / "Alice" + / "79507046_media_https___www.patreon.com_media-u_Z0FBQUFBQm5q" + ) + f.parent.mkdir(parents=True, exist_ok=True) + f.write_bytes(b"binary blob") + r = importer.import_one(f) + assert r.status == "attached" + att = importer.session.execute(select(PostAttachment)).scalar_one() + # Junk "extension" -> stored as empty string (not the 50-char garbage). + assert att.ext == "" + # original_filename is Text-typed so the full name survives intact. + assert att.original_filename.endswith("_Z0FBQUFBQm5q")