Expire trash after 30 days, and make the deadline something you can see

Trash had no end. A note sat in /trash until someone emptied it by hand, and
its attachment BYTES sat on disk the whole time — the pile-up the operator
asked about. Nothing purged; there was no scheduler at all.

Retention is server-owned: `trash_retention_days` (default 30, 0 = keep
forever) in the settings registry, so it lands in admin Settings with no
migration and takes effect without a restart. A background sweep started in
before_serving does the work. Clients learn about a purge the way they learn
about any deletion — as a tombstone on the delta feed.

An auto-purge nobody can see coming is data loss on a timer, so the window is
now visible: /api/config publishes it, notes carry `deleted_at`, Trash leads
with the policy, and each card counts down. The countdown rounds DOWN — saying
"1 day left" for a note with ten minutes on the clock is the one error here
that actually costs someone a note.

Three things this turned up on the way:

- `DELETE /api/notes/<id>` hard-deleted the row, leaving no tombstone at all.
  A permanent delete in the web UI never reached a linked device, which would
  keep its copy forever and push it back on the next edit. It now purges
  through the same path as everything else.
- The purge left `note_revisions` and `note_link_previews` behind. A revision
  holds the full body, so the text of a "permanently deleted" note was still
  sitting in the database.
- `deleted_at` now SURVIVES a purge instead of being cleared. It's still true,
  and it means every query that says "not trashed" excludes tombstones for
  free — without it a content-less row reads as a perfectly normal active note
  and shows up on the board as a blank card.

Desktop keeps its own clock only when there's nobody else to keep one: the
sweep runs at startup on an UNLINKED device and refuses otherwise. A linked
client that expired notes on its own schedule could destroy something the
server was deliberately keeping, then push that delete upstream. Local policy
must never outrank the server's — so it also adopts the server's window for
the countdown rather than showing its offline default.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SreJkbxB4gx8pPsu8QbLPi

src/thoughtsync/app.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import asyncio
 import mimetypes
 import os
 import secrets
+from contextlib import suppress
 from datetime import timedelta
 
 from quart import Quart, has_request_context, jsonify, request, send_from_directory
@@ -15,6 +17,7 @@ from .db import session_scope
 from .graph import bp as graph_bp
 from .labels import bp as labels_bp
 from .notes import bp as notes_bp
+from .retention import run_sweeper
 from .saved_filters import bp as saved_filters_bp
 from .settings import get_public_config, get_setting, load_or_create_secret_key
 from .settings_api import bp as settings_bp
@@ -80,6 +83,21 @@ def create_app() -> Quart:
                 app.config["PERMANENT_SESSION_LIFETIME"] = timedelta(days=days)
             except (ValueError, TypeError, KeyError):
                 pass
+        # Expire old trash in the background (retention.py). One task per process is
+        # correct because the image serves with a single hypercorn worker (Dockerfile);
+        # if that ever gains `--workers`, this needs a lock so N workers don't each
+        # sweep. Duplicate sweeps would be harmless but wasteful — a purged row is
+        # skipped by `purged_at IS NULL` — so this is about load, not correctness.
+        app.config["TRASH_SWEEPER"] = asyncio.create_task(run_sweeper())
+
+    @app.after_serving
+    async def _shutdown() -> None:
+        task = app.config.get("TRASH_SWEEPER")
+        if task is not None:
+            task.cancel()
+            # Await the cancellation so shutdown doesn't race a sweep mid-transaction.
+            with suppress(asyncio.CancelledError):
+                await task
 
     @app.get("/api/health")
     async def health():

src/thoughtsync/models/note.py

@@ -83,6 +83,9 @@ class Note(Base):
             "pinned": self.pinned,
             "archived": self.archived,
             "trashed": self.deleted_at is not None,
+            # WHEN it was trashed, not just that it was: clients count the retention
+            # window from here to show how long a note has left before it's purged.
+            "deleted_at": iso(self.deleted_at),
             "remind_at": iso(self.remind_at),
             "recurrence": self.recurrence,
             "created_at": iso(self.created_at),

src/thoughtsync/notes/__init__.py

@@ -36,6 +36,7 @@ from ..models.note_link import NoteLink
 from ..models.note_link_preview import NoteLinkPreview
 from ..models.note_revision import NoteRevision
 from ..responses import json_error, not_found, parse_uuid
+from ..retention import purge_note
 from ..settings import get_setting
 from ..unfurl import UnfurlError, unfurl
 from ._bp import bp
@@ -971,6 +972,10 @@ async def delete_note(note_id: str):
             return not_found()
         if note.deleted_at is None:
             return json_error("note must be trashed before permanent delete", 409)
-        await db.delete(note)
+        # A tombstone, not a dropped row. Deleting the row outright would leave the
+        # server with no record the note ever existed, so a linked device that was
+        # offline at the time would keep its copy forever — and push it back the
+        # next time it was edited. The delete has to be something clients can LEARN.
+        await purge_note(db, note)
         await db.commit()
         return jsonify({"ok": True})

src/thoughtsync/notes/helpers.py

@@ -45,20 +45,33 @@ def parse_list_items(raw: object) -> list[str]:
 
 
 def apply_filter(stmt, filter_name: str):
-    """Narrow a notes query to one board view."""
+    """Narrow a notes query to one board view.
+
+    Every branch excludes purge tombstones — content-less rows kept only so the sync
+    feed can tell offline clients a note is gone (see `retention.purge_note`). The
+    active/archived branches get that for free from `deleted_at IS NULL`, since a
+    tombstone keeps the timestamp; Trash is the one view that has to say so.
+    """
     if filter_name == "archived":
         return stmt.where(Note.deleted_at.is_(None), Note.archived.is_(True))
     if filter_name == "trash":
-        return stmt.where(Note.deleted_at.is_not(None))
+        return stmt.where(Note.deleted_at.is_not(None), Note.purged_at.is_(None))
     return stmt.where(Note.deleted_at.is_(None), Note.archived.is_(False))
 
 
 async def _get_owned(db, note_id: str) -> Note | None:
-    """Fetch a note the current user OWNS (mutations are owner-only in M1/M2)."""
+    """Fetch a note the current user OWNS (mutations are owner-only in M1/M2).
+
+    A purged note reads as absent: the REST API must treat it as gone, so opening,
+    editing or restoring one 404s. The sync push path looks rows up directly rather
+    than through here, which is what still lets a client re-create an id it owns.
+    """
     nid = parse_uuid(note_id)
     if nid is None:
         return None
-    return await db.scalar(select(Note).where(Note.id == nid, Note.owner_id == g.user_id))
+    return await db.scalar(
+        select(Note).where(Note.id == nid, Note.owner_id == g.user_id, Note.purged_at.is_(None))
+    )
 
 
 def _escape_like(s: str) -> str:

src/thoughtsync/retention.py

@@ -0,0 +1,170 @@
+"""Trash retention — what "permanently deleted" means, and when it happens by itself.
+
+Two things live here, deliberately together:
+
+**`purge_note`** — the single definition of destroying a note. Three callers reach
+permanent deletion by different routes (the user's Delete forever in the web UI,
+a client's `op=delete` over sync, and the sweeper below), and if each had its own
+idea of what to tear down they would drift — one would forget the files, another
+the revision history, and "permanently deleted" would quietly mean three different
+things depending on how you got there.
+
+**The sweeper** — trash that nobody empties is not free: a trashed note keeps its
+attachment BYTES on disk for as long as it sits there. So trash expires. The window
+is the `trash_retention_days` setting (default 30, `0` = keep forever), re-read on
+every pass so a change in admin Settings takes effect without a restart.
+
+A purged note is not a deleted ROW — it's a content-less tombstone. That's what lets
+an offline client that reappears next month learn the note is gone instead of
+faithfully resurrecting it on the next push.
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timedelta, timezone
+
+from sqlalchemy import delete as sa_delete
+from sqlalchemy import select
+
+from .config import Config
+from .db import session_scope
+from .models.label import NoteLabel
+from .models.note import Note
+from .models.note_attachment import NoteAttachment
+from .models.note_item import NoteItem
+from .models.note_link import NoteLink
+from .models.note_link_preview import NoteLinkPreview
+from .models.note_revision import NoteRevision
+from .settings import get_setting
+
+logger = logging.getLogger(__name__)
+
+# How often the sweeper wakes. Retention is measured in days, so anything under
+# "a few times a day" buys nothing but load — a note trashed at 09:00 expiring at
+# 14:00 rather than 09:00 thirty days later is not a difference anyone can feel.
+SWEEP_INTERVAL_SECONDS = 6 * 60 * 60
+
+# Let the app finish booting (migrations, first requests) before the first sweep.
+SWEEP_STARTUP_DELAY_SECONDS = 60
+
+# Rows purged per transaction. A long-neglected install could have thousands of
+# expired notes on the first sweep; committing in batches keeps that from becoming
+# one enormous transaction holding locks while it deletes files.
+SWEEP_BATCH = 200
+
+
+def expired_before(now: datetime, retention_days: int) -> datetime | None:
+    """The cutoff: trash older than this has expired. `None` = retention is off.
+
+    Kept separate from the query so the window arithmetic — including the two ways
+    to say "never" (0 and negative, the latter reachable by typing a stray minus in
+    Settings) — is testable without a database.
+    """
+    if retention_days <= 0:
+        return None
+    return now - timedelta(days=retention_days)
+
+
+async def purge_note(db, note: Note, edited_at: datetime | None = None) -> None:
+    """Turn a note into a content-less tombstone: delete its children (and the
+    attachment files on disk), clear its content, stamp `purged_at`.
+
+    The row survives on purpose — offline clients read it off the delta feed and
+    learn the note is gone. Everything that carries the note's CONTENT goes, and
+    that includes history: a revision row holds the full body, so leaving revisions
+    behind would mean the text of a "permanently deleted" note is still on the
+    server, recoverable by anyone who can read the table.
+    """
+    atts = (await db.scalars(select(NoteAttachment).where(NoteAttachment.note_id == note.id))).all()
+    for a in atts:
+        try:
+            (Config.media_root() / a.path).unlink(missing_ok=True)
+        except OSError:
+            # A missing or unreadable file must not strand the row: the DB record is
+            # what the user asked us to destroy, and a failed unlink leaving it in
+            # place would make the note reappear whole on the next sweep.
+            logger.warning("couldn't remove attachment file %s during purge", a.path, exc_info=True)
+    await db.execute(sa_delete(NoteAttachment).where(NoteAttachment.note_id == note.id))
+    await db.execute(sa_delete(NoteItem).where(NoteItem.note_id == note.id))
+    await db.execute(sa_delete(NoteLabel).where(NoteLabel.note_id == note.id))
+    await db.execute(sa_delete(NoteLink).where(NoteLink.source_id == note.id))
+    await db.execute(sa_delete(NoteLinkPreview).where(NoteLinkPreview.note_id == note.id))
+    await db.execute(sa_delete(NoteRevision).where(NoteRevision.note_id == note.id))
+    note.title = None
+    note.body = ""
+    note.display_title = ""
+    # `deleted_at` deliberately SURVIVES. It's still true — that is when the note was
+    # deleted — and keeping it means every ordinary query, present and future, that
+    # says "not trashed" (`deleted_at IS NULL`) excludes tombstones for free. Clearing
+    # it would leave a content-less row looking like a perfectly normal active note,
+    # and it would surface on the board as a blank card. Only the Trash view, which
+    # asks for `deleted_at IS NOT NULL`, has to name `purged_at` explicitly.
+    note.remind_at = None
+    note.purged_at = datetime.now(timezone.utc)
+    if edited_at is not None:
+        note.updated_at = edited_at
+
+
+async def sweep_expired_trash(db, retention_days: int, *, now: datetime | None = None) -> int:
+    """Purge every note whose trash has expired. Returns how many were purged.
+
+    Runs across ALL owners — it's a server-wide policy, not a per-user action, and
+    the sweeper has no session to scope it by (rule 47 is about honoring the ACL on
+    user-initiated reads, not about exempting rows from server maintenance).
+    """
+    cutoff = expired_before(now or datetime.now(timezone.utc), retention_days)
+    if cutoff is None:
+        return 0
+    total = 0
+    while True:
+        expired = (
+            await db.scalars(
+                select(Note)
+                .where(
+                    Note.deleted_at.is_not(None),
+                    Note.deleted_at < cutoff,
+                    # Already a tombstone. Without this the purge would re-run on
+                    # every sweep forever, bumping sync_revision each time and
+                    # handing clients an endless stream of "news" about one note.
+                    Note.purged_at.is_(None),
+                )
+                .order_by(Note.deleted_at)
+                .limit(SWEEP_BATCH)
+            )
+        ).all()
+        if not expired:
+            return total
+        for note in expired:
+            await purge_note(db, note)
+        await db.commit()
+        total += len(expired)
+
+
+async def sweep_once() -> int:
+    """One sweep against the live retention setting, in its own session."""
+    async with session_scope() as db:
+        try:
+            days = int(await get_setting(db, "trash_retention_days"))
+        except (KeyError, TypeError, ValueError):
+            return 0
+        return await sweep_expired_trash(db, days)
+
+
+async def run_sweeper() -> None:
+    """The background loop. Started in `before_serving`, cancelled on shutdown.
+
+    A sweep failure (DB blip, unreadable media directory) must never take the loop
+    down with it — the next pass simply finds the same expired rows and tries again.
+    """
+    await asyncio.sleep(SWEEP_STARTUP_DELAY_SECONDS)
+    while True:
+        try:
+            purged = await sweep_once()
+            if purged:
+                logger.info("trash retention: purged %d expired note(s)", purged)
+        except asyncio.CancelledError:
+            raise
+        except Exception:
+            logger.exception("trash retention sweep failed; will retry next pass")
+        await asyncio.sleep(SWEEP_INTERVAL_SECONDS)

src/thoughtsync/settings.py

@@ -43,6 +43,15 @@ REGISTRY: list[SettingDef] = [
         "How long a signed-in session stays valid before another login is required.",
         "Access",
     ),
+    SettingDef(
+        "trash_retention_days",
+        "int",
+        30,
+        "Trash retention (days)",
+        "How long a note stays in Trash before it's permanently deleted, freeing its "
+        "attachments from disk. Set to 0 to keep trashed notes until they're deleted by hand.",
+        "Notes",
+    ),
     SettingDef(
         "max_attachment_mb",
         "int",
@@ -117,11 +126,16 @@ async def get_setting(db, key: str) -> Any:
 
 
 async def get_public_config(db) -> dict:
-    """Non-sensitive settings the unauthenticated login/register screen needs."""
+    """Non-sensitive settings every client reads — the login/register screen before
+    sign-in, and the app itself afterwards. Nothing here is owner-scoped."""
     return {
         "site_name": await get_setting(db, "site_name"),
         "allow_registration": await get_setting(db, "allow_registration"),
         "enable_url_unfurl": await get_setting(db, "enable_url_unfurl"),
+        # Server policy, not user data: clients need it to say how long a note has
+        # left in Trash. A native client also reads it BEFORE linking, which is why
+        # it belongs on the unauthenticated config rather than behind login.
+        "trash_retention_days": await get_setting(db, "trash_retention_days"),
     }
 
 

src/thoughtsync/sync.py

@@ -20,14 +20,11 @@ from sqlalchemy import func, select
 
 from .auth import login_required
 from .common import iso, parse_dt
-from .config import Config
 from .db import session_scope
 from .labeling import reconcile_manual_labels, resolve_owned_label_ids
 from .models.label import Label, NoteLabel
 from .models.note import Note
-from .models.note_attachment import NoteAttachment
 from .models.note_item import NoteItem
-from .models.note_link import NoteLink
 from .models.note_revision import NoteRevision
 from .notes import (
     _reconcile_tags,
@@ -38,6 +35,7 @@ from .notes import (
     normalize_color,
     normalize_recurrence,
 )
+from .retention import purge_note
 from .serialize import serialize_label_sync
 
 bp = Blueprint("sync", __name__, url_prefix="/api/sync")
@@ -245,29 +243,6 @@ async def _apply_note_manual_labels(db, note: Note, ch: dict) -> None:
     await reconcile_manual_labels(db, note, owned)
 
 
-async def _purge_note(db, note: Note, edited_at: datetime | None) -> None:
-    """Turn a note into a content-less tombstone: delete children (+ attachment files),
-    clear content, set purged_at. Kept so offline clients learn it's gone."""
-    atts = (await db.scalars(select(NoteAttachment).where(NoteAttachment.note_id == note.id))).all()
-    for a in atts:
-        try:
-            (Config.media_root() / a.path).unlink(missing_ok=True)
-        except OSError:
-            pass
-    await db.execute(sa_delete(NoteAttachment).where(NoteAttachment.note_id == note.id))
-    await db.execute(sa_delete(NoteItem).where(NoteItem.note_id == note.id))
-    await db.execute(sa_delete(NoteLabel).where(NoteLabel.note_id == note.id))
-    await db.execute(sa_delete(NoteLink).where(NoteLink.source_id == note.id))
-    note.title = None
-    note.body = ""
-    note.display_title = ""
-    note.deleted_at = None
-    note.remind_at = None
-    note.purged_at = datetime.now(timezone.utc)
-    if edited_at is not None:
-        note.updated_at = edited_at
-
-
 async def _apply_note(db, ch: dict) -> dict:
     raw_id = ch.get("id")
     try:
@@ -290,7 +265,7 @@ async def _apply_note(db, ch: dict) -> dict:
             return {"id": str(nid), "entity": "note", "status": "noop"}
         if not client_wins(edited_at, note.updated_at):
             return {"id": str(nid), "entity": "note", "status": "kept", "sync_revision": note.sync_revision}
-        await _purge_note(db, note, edited_at)
+        await purge_note(db, note, edited_at)
         await db.flush()
         await db.refresh(note, ["sync_revision"])
         return {"id": str(nid), "entity": "note", "status": "applied", "sync_revision": note.sync_revision}