fix(retention): add cleanup sweeps + CalDAV orphan reconciliation
CI & Build / TypeScript typecheck (push) Successful in 37s
CI & Build / Python lint (push) Successful in 2s
CI & Build / Python tests (push) Successful in 42s
CI & Build / Build & push image (push) Successful in 1m29s

Drift-audit Group 4 (retention / unbounded growth):

- CalDAV pull now reconciles deletions: a previously-synced event whose
  caldav_uid no longer appears remotely within the synced window is
  soft-deleted (one batch_id per run, restorable), so a remote delete
  propagates locally instead of orphaning forever. Guarded on a non-empty
  fetch so a spurious empty result can't wipe every local copy. Also wrap
  the blocking fetch in a 120s wait_for and log run duration.
- Notifications: hourly loop now purges read notifications older than 30d
  (unread kept). Table no longer grows without bound.
- Auth tokens: new daily sweep deletes password-reset / invitation tokens
  whose validity window ended >7d ago; wired via start_auth_token_retention_loop
  in app startup. Both tables previously only flipped used=True, never pruned.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-02 18:55:23 -04:00
parent 5fe0fd126d
commit c363a5a6df
4 changed files with 116 additions and 7 deletions
+2
View File
@@ -148,12 +148,14 @@ def create_app() -> Quart:
async def startup(): async def startup():
import asyncio import asyncio
from fabledassistant.services.auth import start_auth_token_retention_loop
from fabledassistant.services.embeddings import backfill_note_embeddings from fabledassistant.services.embeddings import backfill_note_embeddings
from fabledassistant.services.logging import start_log_retention_loop from fabledassistant.services.logging import start_log_retention_loop
from fabledassistant.services.notifications import start_notification_loop from fabledassistant.services.notifications import start_notification_loop
start_log_retention_loop() start_log_retention_loop()
start_notification_loop() start_notification_loop()
start_auth_token_retention_loop()
# Backfill embeddings for any notes that don't have one. Runs in the # Backfill embeddings for any notes that don't have one. Runs in the
# background so it never blocks the server from accepting requests. # background so it never blocks the server from accepting requests.
+43
View File
@@ -356,3 +356,46 @@ async def revoke_invitation(invitation_id: int) -> bool:
await session.commit() await session.commit()
logger.info("Invitation %d revoked", invitation_id) logger.info("Invitation %d revoked", invitation_id)
return True return True
# ── Token retention ─────────────────────────────────────────────────────
# Password-reset and invitation tokens are only ever flipped used=True and are
# never pruned, so on a long-lived instance both tables grow without bound.
# A daily sweep deletes any token whose validity window ended over grace_days
# ago (covers both used and naturally-expired rows once they're cold).
_auth_retention_task = None
async def purge_expired_auth_tokens(grace_days: int = 7) -> int:
"""Delete password-reset / invitation tokens that expired > grace_days ago."""
from sqlalchemy import delete
cutoff = datetime.now(timezone.utc) - timedelta(days=grace_days)
removed = 0
async with async_session() as session:
for model in (PasswordResetToken, InvitationToken):
result = await session.execute(
delete(model).where(model.expires_at < cutoff)
)
removed += result.rowcount or 0
await session.commit()
return removed
async def _auth_token_retention_loop() -> None:
import asyncio
while True:
await asyncio.sleep(86400) # daily
try:
removed = await purge_expired_auth_tokens()
if removed:
logger.info("Auth token retention: deleted %d expired token(s)", removed)
except Exception:
logger.exception("Error in auth token retention cleanup")
def start_auth_token_retention_loop() -> None:
global _auth_retention_task
import asyncio
if _auth_retention_task is None or _auth_retention_task.done():
_auth_retention_task = asyncio.create_task(_auth_token_retention_loop())
+43 -7
View File
@@ -11,7 +11,7 @@ import uuid
from datetime import datetime, timedelta, timezone from datetime import datetime, timedelta, timezone
from typing import Any from typing import Any
from sqlalchemy import select from sqlalchemy import select, update
from fabledassistant.models import async_session from fabledassistant.models import async_session
from fabledassistant.models.event import Event from fabledassistant.models.event import Event
@@ -20,6 +20,9 @@ logger = logging.getLogger(__name__)
_SYNC_PAST_DAYS = 30 _SYNC_PAST_DAYS = 30
_SYNC_FUTURE_DAYS = 180 _SYNC_FUTURE_DAYS = 180
# Wall-clock cap on the blocking CalDAV fetch so a hung/slow server can't
# wedge the hourly sweep indefinitely.
_SYNC_TIMEOUT_SECONDS = 120
def _parse_dt(val: Any) -> datetime | None: def _parse_dt(val: Any) -> datetime | None:
@@ -116,16 +119,24 @@ async def sync_user_events(user_id: int) -> dict:
config = await get_caldav_config(user_id) config = await get_caldav_config(user_id)
started = datetime.now(timezone.utc)
range_start = started - timedelta(days=_SYNC_PAST_DAYS)
range_end = started + timedelta(days=_SYNC_FUTURE_DAYS)
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
try: try:
remote_events: list[dict] = await loop.run_in_executor( remote_events: list[dict] = await asyncio.wait_for(
None, _sync_one_user, config, user_id loop.run_in_executor(None, _sync_one_user, config, user_id),
timeout=_SYNC_TIMEOUT_SECONDS,
) )
except asyncio.TimeoutError:
logger.warning("CalDAV pull sync timed out for user %d after %ds", user_id, _SYNC_TIMEOUT_SECONDS)
return {"error": "CalDAV fetch timed out"}
except Exception: except Exception:
logger.warning("CalDAV pull sync failed for user %d", user_id, exc_info=True) logger.warning("CalDAV pull sync failed for user %d", user_id, exc_info=True)
return {"error": "CalDAV fetch failed"} return {"error": "CalDAV fetch failed"}
created = updated = unchanged = skipped = 0 created = updated = unchanged = skipped = deleted = 0
async with async_session() as session: async with async_session() as session:
for ev in remote_events: for ev in remote_events:
@@ -185,13 +196,38 @@ async def sync_user_events(user_id: int) -> dict:
else: else:
unchanged += 1 unchanged += 1
# Reconcile deletions: a previously-synced event (has a caldav_uid)
# that no longer appears remotely within the synced window is
# soft-deleted, so a delete on the remote propagates locally instead
# of orphaning forever. Guarded on a non-empty fetch so a spurious
# empty result can't wipe every local copy.
if remote_events:
remote_uids = {e["caldav_uid"] for e in remote_events}
orphan_batch = str(uuid.uuid4())
orphan_res = await session.execute(
update(Event)
.where(
Event.user_id == user_id,
Event.caldav_uid.isnot(None),
Event.caldav_uid.notin_(remote_uids),
Event.deleted_at.is_(None),
Event.start_dt >= range_start,
Event.start_dt <= range_end,
)
.values(deleted_at=datetime.now(timezone.utc), deleted_batch_id=orphan_batch)
)
deleted = orphan_res.rowcount or 0
await session.commit() await session.commit()
elapsed = (datetime.now(timezone.utc) - started).total_seconds()
logger.info( logger.info(
"CalDAV sync user %d: %d created, %d updated, %d unchanged, %d skipped (trashed)", "CalDAV sync user %d: %d created, %d updated, %d unchanged, %d skipped (trashed), "
user_id, created, updated, unchanged, skipped, "%d deleted (orphaned) in %.1fs",
user_id, created, updated, unchanged, skipped, deleted, elapsed,
) )
return {"created": created, "updated": updated, "unchanged": unchanged, "skipped": skipped} return {"created": created, "updated": updated, "unchanged": unchanged,
"skipped": skipped, "deleted": deleted}
async def sync_all_users() -> None: async def sync_all_users() -> None:
@@ -236,6 +236,28 @@ async def check_due_tasks() -> None:
logger.exception("Failed to send task reminder for user %d", user_id) logger.exception("Failed to send task reminder for user %d", user_id)
# Read notifications are kept this long before the hourly sweep deletes them;
# unread are kept regardless. Without a sweep the table grows without bound.
_NOTIFICATION_RETENTION_DAYS = 30
async def purge_old_read_notifications(retention_days: int = _NOTIFICATION_RETENTION_DAYS) -> int:
"""Delete already-read in-app notifications older than retention_days."""
from datetime import timedelta
from sqlalchemy import delete
from fabledassistant.models.notification import Notification
cutoff = datetime.now(timezone.utc) - timedelta(days=retention_days)
async with async_session() as session:
result = await session.execute(
delete(Notification).where(
Notification.read_at.isnot(None),
Notification.read_at < cutoff,
)
)
await session.commit()
return result.rowcount or 0
async def _notification_loop() -> None: async def _notification_loop() -> None:
while True: while True:
await asyncio.sleep(3600) # hourly await asyncio.sleep(3600) # hourly
@@ -243,6 +265,12 @@ async def _notification_loop() -> None:
await check_due_tasks() await check_due_tasks()
except Exception: except Exception:
logger.exception("Error in notification loop") logger.exception("Error in notification loop")
try:
removed = await purge_old_read_notifications()
if removed:
logger.info("Notification retention: deleted %d read notification(s)", removed)
except Exception:
logger.exception("Error in notification retention cleanup")
def start_notification_loop() -> None: def start_notification_loop() -> None: