fix: dead external links are retried sparingly, not three times a day
CI and images / lint (push) Successful in 1s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 20s
CI and images / extension-test (push) Successful in 23s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m20s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
CI and images / lint (push) Successful in 1s
CI and images / extension-version (push) Successful in 2s
CI and images / frontend-build (push) Successful in 20s
CI and images / extension-test (push) Successful in 23s
CI and images / backend-lint-and-test (push) Successful in 33s
CI and images / integration (push) Successful in 2m20s
CI and images / sign-extension (push) Successful in 2s
CI and images / build-agent (push) Successful in 5s
CI and images / build-web (push) Successful in 2m1s
CI and images / smoke-web (push) Successful in 49s
CI and images / promote (push) Successful in 1s
CI and images / refresh-dev (push) Skipped
CI and images / unattended-failure (push) Skipped
The daily recovery reset every dead link to a fresh three-attempt budget, so a link that can never succeed (an archive too big to import inside the fetch's hard limit, a file made private) was downloaded up to three times a day, every day. Recovery now gives a dead link one more attempt once it has been dead for 2**recoveries days (1, 2, 4, 8, 16), and stops after five. A recovered link that fails again goes straight back to dead. Retention prunes only links recovery has given up on, 30 days after their last attempt. A manual refetch of a post restores the full budgets. Migration 0118 adds external_link.recoveries. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -402,3 +402,74 @@ def test_sweep_recovers_a_link_whose_worker_died(db_sync, monkeypatch):
|
||||
row = db_sync.get(ExternalLink, lid)
|
||||
assert (row.status, row.attempts) == ("failed", 1)
|
||||
assert db_sync.get(ExternalLink, fresh_id).status == "downloading"
|
||||
|
||||
|
||||
def test_recovery_gives_a_dead_link_one_more_attempt_at_growing_intervals(db_sync, monkeypatch):
|
||||
"""A dead link is retried once after 2**recoveries days, not given a fresh
|
||||
three-attempt budget, so a link that always fails isn't downloaded daily."""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
now = datetime.now(UTC)
|
||||
_, due = _seed(db_sync, host="mega", status="dead")
|
||||
_, waiting = _seed(db_sync, host="gdrive", status="dead")
|
||||
_, spent = _seed(db_sync, host="dropbox", status="dead")
|
||||
due.attempts = waiting.attempts = spent.attempts = ext.DEAD_LETTER_THRESHOLD
|
||||
due.recoveries, due.completed_at = 2, now - timedelta(days=4, hours=1)
|
||||
waiting.recoveries, waiting.completed_at = 2, now - timedelta(days=3)
|
||||
spent.recoveries, spent.completed_at = ext._MAX_RECOVERIES, now - timedelta(days=90)
|
||||
db_sync.commit()
|
||||
due_id, waiting_id, spent_id = due.id, waiting.id, spent.id
|
||||
|
||||
enqueued = []
|
||||
monkeypatch.setattr(ext.fetch_external_link, "delay", lambda lid: enqueued.append(lid))
|
||||
out = ext.recover_external_links()
|
||||
|
||||
assert out["revived"] == 1
|
||||
assert enqueued == [due_id]
|
||||
db_sync.expire_all()
|
||||
row = db_sync.get(ExternalLink, due_id)
|
||||
assert (row.status, row.attempts, row.recoveries) == (
|
||||
"failed", ext.DEAD_LETTER_THRESHOLD - 1, 3,
|
||||
)
|
||||
assert db_sync.get(ExternalLink, waiting_id).status == "dead"
|
||||
assert db_sync.get(ExternalLink, spent_id).status == "dead"
|
||||
|
||||
|
||||
def test_a_recovered_link_that_fails_again_goes_straight_back_to_dead(
|
||||
db_sync, tmp_path, monkeypatch,
|
||||
):
|
||||
_, link = _seed(db_sync, host="mega", status="dead")
|
||||
link.attempts = ext.DEAD_LETTER_THRESHOLD
|
||||
db_sync.commit()
|
||||
link_id = link.id
|
||||
monkeypatch.setattr(ext, "IMAGES_ROOT", tmp_path)
|
||||
monkeypatch.setattr(ext, "_redis", lambda: _FakeRedis())
|
||||
monkeypatch.setattr(ext, "fetch_external", lambda *a, **k: FetchResult(error="too big"))
|
||||
monkeypatch.setattr(ext.fetch_external_link, "delay", ext.fetch_external_link)
|
||||
|
||||
ext.recover_external_links()
|
||||
|
||||
db_sync.expire_all()
|
||||
row = db_sync.get(ExternalLink, link_id)
|
||||
assert (row.status, row.recoveries) == ("dead", 1)
|
||||
|
||||
|
||||
def test_prune_removes_only_links_recovery_has_given_up_on(db_sync):
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
old = datetime.now(UTC) - timedelta(days=ext._RETENTION_DAYS + 1)
|
||||
_, spent = _seed(db_sync, host="mega", status="dead")
|
||||
_, recovering = _seed(db_sync, host="gdrive", status="dead")
|
||||
_, recent = _seed(db_sync, host="dropbox", status="dead")
|
||||
spent.recoveries, spent.completed_at = ext._MAX_RECOVERIES, old
|
||||
recovering.recoveries, recovering.completed_at = 1, old
|
||||
recent.recoveries = ext._MAX_RECOVERIES
|
||||
recent.completed_at = datetime.now(UTC) - timedelta(days=1)
|
||||
db_sync.commit()
|
||||
spent_id, recovering_id, recent_id = spent.id, recovering.id, recent.id
|
||||
|
||||
assert ext.prune_external_links() == {"pruned": 1}
|
||||
db_sync.expire_all()
|
||||
assert db_sync.get(ExternalLink, spent_id) is None
|
||||
assert db_sync.get(ExternalLink, recovering_id) is not None
|
||||
assert db_sync.get(ExternalLink, recent_id) is not None
|
||||
|
||||
Reference in New Issue
Block a user