Near-duplicate dedup rebuilt on three gates, backup credential leak closed, library consolidation started #255

Merged
bvandeusen merged 8 commits from dev into main 2026-09-21 11:11:39 -04:00
2 changed files with 46 additions and 2 deletions
Showing only changes of commit 6915cbbbe1 - Show all commits
+23 -2
View File
@@ -24,6 +24,25 @@ from pathlib import Path
_BACKUPS_DIRNAME = "_backups"
# Excluded from the images tarball, and each for its own reason (#4233, #4234):
#
# _backups — the archive would otherwise contain every previous archive.
# This is not hypothetical: the 2026-05-23/24 runs, taken before
# this exclude existed, grew 43G -> 107G -> ... -> 2123G as each
# swallowed its predecessors, and cost 4.3T of the images
# filesystem until they were reclaimed on 2026-09-21.
# _quarantine — holds files deliberately pulled OUT of the library.
# secrets — `credential_key.b64`, the key that decrypts the stored
# Patreon/SubscribeStar session credentials.
# cookies — those session cookies themselves.
#
# The last two are the ones worth stating plainly: an images tarball is a media
# archive, and a media archive that carries the key to the operator's accounts
# is a credential leak wearing a backup's name. Encryption at rest buys nothing
# when the key rides along in the same file. A restore therefore does NOT
# re-establish credentials — you sign in again, which is the correct outcome.
_IMAGES_EXCLUDED_DIRNAMES = ("_backups", "_quarantine", "secrets", "cookies")
# Subprocess-level guardrails BEYOND the Celery soft_time_limit. The Celery
# soft limit signals the Python process; subprocess.Popen in a blocking syscall
# ignores that signal, so these bound the worst case directly. Each sits just
@@ -173,8 +192,10 @@ def backup_images(
[
"tar", "--zstd", "-cf", str(tar_path),
"-C", str(images_root.parent), images_root.name,
f"--exclude={images_root.name}/_backups",
f"--exclude={images_root.name}/_quarantine",
*(
f"--exclude={images_root.name}/{name}"
for name in _IMAGES_EXCLUDED_DIRNAMES
),
],
_IMAGES_SUBPROCESS_TIMEOUT_S,
)
+23
View File
@@ -134,6 +134,29 @@ def test_backup_images_excludes_backups_and_quarantine(tmp_path, fake_subprocess
assert any("_quarantine" in e for e in excludes)
def test_backup_images_excludes_credentials(tmp_path, fake_subprocess):
"""#4234: the images tarball carried `secrets/credential_key.b64` — the key
that decrypts the stored session cookies — and `cookies/` itself. A media
archive must not be a credential leak; encryption at rest is worth nothing
if the key travels with the data."""
backup_service.backup_images(images_root=tmp_path)
excludes = [a for a in fake_subprocess[0] if a.startswith("--exclude=")]
assert any(e.endswith("/secrets") for e in excludes)
assert any(e.endswith("/cookies") for e in excludes)
def test_backup_images_excludes_are_root_relative(tmp_path, fake_subprocess):
"""tar matches --exclude against the archived path, which is prefixed with
the root's own directory name (`-C <parent> <name>`). A bare `secrets`
would also match an ARTIST folder called secrets; the prefix is what keeps
the exclusion to the top level."""
backup_service.backup_images(images_root=tmp_path)
excludes = [a for a in fake_subprocess[0] if a.startswith("--exclude=")]
assert excludes and all(
e.startswith(f"--exclude={tmp_path.name}/") for e in excludes
)
# --- restore_db ------------------------------------------------------