feat: a tick keeps looking back 30 days, so an EDITED post is reached (4386)

Operator, 2026-09-23, on a Floppystack post: "this post has been updated as
he implements hot fixes — any chance we have a way to scan for or see updated
posts so we can update ours to match and pull the new attachments and
pictures etc."

The download half already worked: extract_media reads the media list off the
LIVE feed response every walk, so a newly attached hotfix build is a ledger
key we have never seen. Only REACHING the post was missing — a tick stopped
after 20 contiguous already-have-it items, and a post edited three days after
publication sits well below twenty. Not a bug in the early-out; a count
cannot express "recent".

The early-out now needs BOTH conditions: the run of seen items AND a post
published before the horizon. Strictly a widening — window 0 is exactly the
old behaviour, and no window can make a tick stop EARLIER than it used to, so
a source paused for months still walks its whole unseen backlog. The horizon
is a floor on how far to look, never a ceiling.

Inside the window the post-record gate is bypassed too (write_post_record
revisit=True): the body is re-read from the feed response already in hand, so
a revisit costs zero requests, and a body that comes back empty writes
NOTHING rather than blanking one a detail-fetch had filled. Revisits are kept
out of the #862 body-drift canary's sample for the same reason — an empty
revisit is healthy, and counting it would walk the alarm toward firing on
good ticks.

The run summary names what changed ("3 post(s) updated (5 new file(s))") with
a line per post; the ask was to SEE updated posts, not only to end up with
their bytes.

download_revisit_days is a settings row, not a constant (rule 25) — how long
a creator keeps editing is a property of the creator. Default 30, 0 turns it
off. Migration 0108.

Also corrects two stale docstrings: both clients described post_meta as
feeding an Ingester.preview that no longer calls it. It had no consumer at
all until this change.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01LVjrnpQjRgHdvq95rASoiR

backend/app/services/download_backends.py

@@ -24,6 +24,7 @@ import asyncio
 from pathlib import Path
 
 from .gallery_dl import DownloadResult, ErrorType
+from .ingest_core import DEFAULT_REVISIT_DAYS
 from .patreon_ingester import PatreonIngester
 from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
 from .platforms import known_platform_keys
@@ -83,6 +84,7 @@ async def run_download(
     mode: str | None,
     gdl,
     sync_session_factory,
+    revisit_days: int = DEFAULT_REVISIT_DAYS,
 ) -> tuple[DownloadResult, str | None]:
     """Uniform download across backends — the download counterpart to
     `verify_source_credential`, so this module is the ONE place that knows how
@@ -106,7 +108,7 @@ async def run_download(
         ), None
     if uses_native_ingester(platform):
         return await _run_native_ingester(
-            ctx, source_config, mode, gdl, sync_session_factory
+            ctx, source_config, mode, gdl, sync_session_factory, revisit_days
         )
     result = await gdl.download(
         url=ctx["url"],
@@ -146,6 +148,7 @@ def _campaign_resolution_error(platform: str, url: str) -> str:
 
 async def _run_native_ingester(
     ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
+    revisit_days: int = DEFAULT_REVISIT_DAYS,
 ) -> tuple[DownloadResult, str | None]:
     """Run the native ingester for a native platform in a worker thread (sync
     requests/subprocess). Patreon resolves a campaign id from the vanity URL;
@@ -210,6 +213,9 @@ async def _run_native_ingester(
             mode=mode,
             resume_cursor=source_config.resume_cursor,
             time_budget_seconds=source_config.timeout,
+            # How far back a tick keeps looking for EDITED posts. The ingester
+            # applies it to ticks only; a backfill ignores it.
+            revisit_days=revisit_days,
             posts_base=int(overrides.get("_backfill_posts", 0)),
             # plan #709: live progress writes to this running event mid-walk.
             event_id=ctx.get("event_id"),

backend/app/services/download_service.py

@@ -39,6 +39,7 @@ from .gallery_dl import (
     walk_completed,
 )
 from .importer import Importer
+from .ingest_core import DEFAULT_REVISIT_DAYS
 from .platforms import auth_type_for
 from .scheduler_service import set_platform_cooldown
 
@@ -60,6 +61,7 @@ class DownloadService:
         importer: Importer,
         cred_service: CredentialService,
         sync_session_factory=None,
+        revisit_days: int = DEFAULT_REVISIT_DAYS,
     ):
         self.async_session = async_session
         self.sync_session = sync_session
@@ -71,6 +73,12 @@ class DownloadService:
         # the multi-minute walk — see PatreonIngester). Only the patreon branch
         # of phase 2 uses it; gallery-dl sources leave it None.
         self.sync_session_factory = sync_session_factory
+        # ImportSettings.download_revisit_days — how far back a tick keeps
+        # looking for EDITED posts (ingest_core.DEFAULT_REVISIT_DAYS). Passed in
+        # rather than read here because the task already loads the settings row
+        # for rate_limit/validate_files, and a second load on every download
+        # would be the same row twice for one number.
+        self.revisit_days = revisit_days
 
     async def download_source(self, source_id: int) -> int:
         """Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
@@ -178,6 +186,7 @@ class DownloadService:
         return await run_download(
             ctx=ctx, source_config=source_config, skip_value=skip_value, mode=mode,
             gdl=self.gdl, sync_session_factory=self.sync_session_factory,
+            revisit_days=self.revisit_days,
         )
 
     async def _phase1_setup(self, source_id: int) -> dict[str, Any]:

backend/app/services/ingest_core.py

@@ -31,6 +31,7 @@ import json
 import logging
 import time
 from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
 
 from sqlalchemy import delete, func, select, text
 from sqlalchemy.dialects.postgresql import insert as pg_insert
@@ -51,6 +52,33 @@ log = logging.getLogger(__name__)
 # per-file HEADs. Headroom against paywalled/undownloadable items interleaving.
 _TICK_SEEN_THRESHOLD = 20
 
+# How far back a tick keeps looking even once everything is already-have-it —
+# the REVISIT WINDOW. Operator, 2026-09-23, holding up a Floppystack post:
+# *"this post has been updated as he implements hot fixes — any chance we have a
+# way to scan for or see updated posts so we can update ours to match and pull
+# the new attachments and pictures etc."*
+#
+# A creator who edits a three-day-old post to append a hotfix build was
+# structurally unreachable: that post sits twenty-odd already-seen items down
+# the feed, so the count early-out above fired before the walk ever got to it.
+# Not a bug in the early-out — a COUNT cannot express "recent".
+#
+# So the early-out now needs BOTH conditions: the run of already-seen items AND
+# a post published before the horizon. Strictly a widening. Two properties this
+# shape has and a plain "walk the last N days" would not:
+#
+#   * window 0 is exactly the old behaviour, so the feature has an off switch
+#     that costs nothing to reason about;
+#   * no window can make a tick stop EARLIER than it used to. A source paused
+#     for months has an unseen backlog stretching well past any horizon, and
+#     the walk still runs to the end of it — the horizon is a FLOOR on how far
+#     to look, never a ceiling.
+#
+# The live value is `ImportSettings.download_revisit_days` (rule 25 — an
+# operator tuning how far back their creators edit should not need a redeploy).
+# This is the fallback for a caller that passes none.
+DEFAULT_REVISIT_DAYS = 30
+
 # plan #705 #7: after this many failed download/validate attempts a media is
 # "dead-lettered" and skipped on routine tick/backfill walks (recovery still
 # re-attempts it). Stops a permanently-broken media re-erroring forever.
@@ -78,6 +106,28 @@ _LIVE_PROGRESS_INTERVAL = 5.0
 _CANARY_MIN_SAMPLE = 30
 
 
+def _parse_published(raw: object) -> datetime | None:
+    """An ISO-8601 post date from either native client, as aware UTC.
+
+    Patreon's `published_at` is tz-aware with a `Z` or `+00:00` offset;
+    SubscribeStar's is NAIVE (`_parse_ss_datetime` renders a parsed local
+    timestamp with no zone). A naive value is read as UTC — the alternative is
+    discarding it, and a post whose date we refuse to read is a post the revisit
+    window can never reach.
+
+    Anything unparseable returns None, which reads downstream as "not provably
+    recent" and leaves the walk on its pre-revisit behaviour. Never raises: a
+    date we cannot read must not fail a walk that is otherwise working.
+    """
+    if not isinstance(raw, str) or not raw.strip():
+        return None
+    try:
+        parsed = datetime.fromisoformat(raw.strip().replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
+
+
 class Ingester:
     """Generic native-ingest orchestration. Subclass with a platform adapter
     (see the module docstring) — or construct directly with the keyword seams."""
@@ -132,11 +182,17 @@ class Ingester:
         resume_cursor: str | None = None,
         time_budget_seconds: float = 870.0,
         seen_threshold: int = _TICK_SEEN_THRESHOLD,
+        revisit_days: int = DEFAULT_REVISIT_DAYS,
         posts_base: int = 0,
         event_id: int | None = None,
     ) -> DownloadResult:
         """Walk + download for one source, returning a gallery-dl-shaped result.
 
+        `revisit_days` is the tick's revisit window (see DEFAULT_REVISIT_DAYS):
+        inside it a tick neither early-outs nor trusts the post-record gate, so
+        a post edited after we first captured it is re-read and its new
+        attachments downloaded. 0 turns the window off.
+
         `mode` is "tick" | "backfill" | "recovery" | "recapture". Recovery
         bypasses the tier-1 seen-ledger AND the dead-letter ledger (tier-2 disk
         still skips kept files). Recapture (#830) is the cheap "re-grab post
@@ -179,6 +235,21 @@ class Ingester:
         # no media download, no post-record stub. Absent on stub/not-yet-migrated
         # clients → nothing is ever treated as gated.
         post_is_gated = getattr(self.client, "post_is_gated", None)
+        # The revisit window (see DEFAULT_REVISIT_DAYS). `post_meta` is an
+        # existing client seam — both native clients already implement it, for
+        # a preview sample whose caller has since gone, so this needed no new
+        # contract, only a live consumer for one. Absent seam, an unreadable
+        # date or a window of 0 → `horizon` never matches and the walk behaves
+        # exactly as it did before 2026-09-23.
+        #
+        # The window applies to TICKS only. A backfill is gated on purpose
+        # (capture each post once) and `recapture` mode already exists for the
+        # operator-driven "re-read every body" pass; a horizon there would be a
+        # third overlapping answer to a question that has two.
+        post_meta = getattr(self.client, "post_meta", None)
+        horizon: datetime | None = None
+        if mode == "tick" and revisit_days > 0 and post_meta is not None:
+            horizon = datetime.now(UTC) - timedelta(days=revisit_days)
         start = time.monotonic()
         last_live = start  # plan #709: last live-progress write timestamp
         log_lines: list[str] = []
@@ -210,6 +281,12 @@ class Ingester:
         # absolute across chunks instead of an inflating sum. posts_processed
         # stays the gross per-chunk count used for the run summary.
         chunk_new_posts = 0
+        # Posts inside the revisit window that we had already captured, and the
+        # media those revisits turned up. Reported in the run summary — the
+        # operator's ask was to SEE the updated posts, not only to end up with
+        # their files ("so we can update ours to match").
+        revisited = 0
+        revisit_downloads = 0
         consecutive_seen = 0
         emitted_cursor: str | None = None
         reached_bottom = False
@@ -322,6 +399,20 @@ class Ingester:
                 # resume_cursor None, so everything counts.
                 if not (resume_cursor and page_cursor == resume_cursor):
                     chunk_new_posts += 1
+                # Inside the revisit window? Computed per post rather than
+                # "stop once one post is old" because the feed is only MOSTLY
+                # date-ordered — a pinned or re-pinned post can sit above older
+                # ones, and one such post must not end the walk.
+                in_window = False
+                if horizon is not None:
+                    published = _parse_published((post_meta(post) or {}).get("date"))
+                    in_window = published is not None and published >= horizon
+                # Set by the post-record block below when this post was already
+                # captured on an earlier walk. Stays False when the platform has
+                # no post-record seam, so the revisit accounting simply reports
+                # nothing rather than guessing.
+                post_already_recorded = False
+                downloaded_before = downloaded
                 # Tier-gated post (#874): the account can't fully view it, so
                 # Patreon serves only blurred locked-preview media. Skip it
                 # ENTIRELY — no media download AND no post-record stub (operator
@@ -353,11 +444,31 @@ class Ingester:
                             set() if recapture_records
                             else self._seen_keys(source_id, [pkey])
                         )
-                        if pkey not in already:
-                            rec = write_post_record(post, artist_slug)
-                            posts_recorded += 1
-                            if rec.body_chars:
-                                posts_with_body += 1
+                        post_already_recorded = pkey in already
+                        # A post inside the revisit window is re-read even
+                        # though the gate has it: that gate's whole job is to
+                        # stop us paying for a post twice, and an EDITED post is
+                        # not the same post. `revisit=True` keeps the cost at
+                        # zero requests — the downloader re-reads the body from
+                        # the feed response already in hand and declines to
+                        # write at all if that body came back empty, so a
+                        # detail-fetched body is never overwritten by a blank.
+                        if not post_already_recorded or in_window:
+                            rec = write_post_record(
+                                post, artist_slug, revisit=post_already_recorded,
+                            )
+                            if not post_already_recorded:
+                                # FIRST captures only feed the #862 body canary.
+                                # A revisit legitimately comes back empty — a
+                                # post whose body only ever arrived from the
+                                # detail endpoint has none in the feed, and the
+                                # downloader declines to write it. Counting
+                                # those into the sample would walk the canary
+                                # toward firing on healthy ticks, which is the
+                                # one thing a drift alarm must never do.
+                                posts_recorded += 1
+                                if rec.body_chars:
+                                    posts_with_body += 1
                             if rec.path is not None:
                                 post_records.append(str(rec.path))
                                 self._mark_seen(source_id, [(pkey, ppid)])
@@ -367,7 +478,8 @@ class Ingester:
                             # a 0-char body is the "why is this one empty" answer.
                             log_lines.append(
                                 f"  post {ppid} [{rec.post_type or '?'}] "
-                                f"body: {rec.body_chars} chars"
+                                + ("re-read, " if post_already_recorded else "")
+                                + f"body: {rec.body_chars} chars"
                                 + ("" if rec.body_chars else " — EMPTY")
                                 + (f" — {rec.title}" if rec.title else "")
                             )
@@ -451,7 +563,15 @@ class Ingester:
                         to_fail.append((key, media_item.post_id, outcome.error or "error"))
                         # An error neither advances nor resets the run-of-seen.
 
-                    if mode == "tick" and consecutive_seen >= seen_threshold:
+                    # `not in_window` is the revisit window's half of the
+                    # early-out: a run of already-seen items is only permission
+                    # to stop once the walk is BELOW the horizon. Both halves,
+                    # never either alone — see DEFAULT_REVISIT_DAYS.
+                    if (
+                        mode == "tick"
+                        and not in_window
+                        and consecutive_seen >= seen_threshold
+                    ):
                         early_out = True
                         break
 
@@ -465,6 +585,21 @@ class Ingester:
                 if to_fail:
                     self._record_failures(source_id, to_fail)
 
+                # An already-captured post that yielded NEW media is an edited
+                # post — the operator's Floppystack case, and the one thing in
+                # this walk worth naming individually in the run log. The media
+                # half needed no new detection: `extract_media` reads the media
+                # list off the live feed response, so a hotfix build appended
+                # last night is simply a ledger key we have never seen.
+                new_here = downloaded - downloaded_before
+                if post_already_recorded and new_here:
+                    revisited += 1
+                    revisit_downloads += new_here
+                    log_lines.append(
+                        f"  post {post.get('id')} — updated: "
+                        f"{new_here} new file(s)"
+                    )
+
                 # plan #709: time-throttled live progress to the running event so
                 # the Downloads view ticks ~every 5s, independent of page size.
                 now = time.monotonic()
@@ -527,6 +662,12 @@ class Ingester:
             # visible in the Raw stdout (e.g. "bodies 3/180" reads as off).
             + (f", bodies {posts_with_body}/{posts_recorded}" if posts_recorded else "")
             + (f", {gated_skipped} gated-skipped" if gated_skipped else "")
+            # Only when it happened: on a quiet tick this is 0 and saying so
+            # every run would bury the times it is not.
+            + (
+                f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
+                if revisited else ""
+            )
             + (", reached end" if reached_bottom else "")
             + (", time-boxed" if budget_hit else "")
         )

backend/app/services/patreon_client.py

@@ -483,8 +483,14 @@ class PatreonClient:
 
     @staticmethod
     def post_meta(post: dict) -> dict:
-        """Title + published date for a post — for the preview sample (plan #708
-        B4). Part of the client contract `ingest_core.Ingester.preview` calls."""
+        """Title + published date for a post. Part of the client contract.
+
+        Written for a preview sample (plan #708 B4) whose caller has since gone;
+        as of 2026-09-23 its consumer is the core's REVISIT WINDOW, which needs
+        a post's date to know whether a tick is still inside it. Both native
+        clients answer in the same shape — an ISO-8601 string under `date`, or
+        None — so the core reads a date without knowing the platform.
+        """
         attrs = post.get("attributes") or {}
         title = attrs.get("title")
         published = attrs.get("published_at")

backend/app/services/patreon_downloader.py

@@ -340,7 +340,7 @@ class PatreonDownloader(BaseNativeDownloader):
 
     def _write_sidecar_data(
         self, post: dict, sidecar_path: Path, *, source_url: str | None = None,
-        minimal: bool = False,
+        minimal: bool = False, detail_fetch: bool = True,
     ) -> Path:
         """Serialize the post's metadata to `sidecar_path`. The post-only record
         (`write_post_record`) writes the FULL post (body/title/date/url); the
@@ -364,7 +364,13 @@ class PatreonDownloader(BaseNativeDownloader):
         # dict — so a multi-image post fetches detail at most once, the post-record
         # body-length read reuses it, and a fully-seen post (no fresh download → no
         # sidecar write) never pays the extra GET.
-        if (not content or not content.strip()) and self._content_fetcher:
+        # `detail_fetch=False` on a REVISIT (a post inside the tick's revisit
+        # window that we already captured): re-read the body from the feed
+        # response we are holding and pay nothing. Without this a 30-day window
+        # would buy one detail GET per body-less post per tick, forever — a
+        # per-creator cost that grows with how prolific they are, to re-fetch a
+        # body we already stored.
+        if (not content or not content.strip()) and self._content_fetcher and detail_fetch:
             fetched = self._content_fetcher(str(post.get("id") or ""))
             if fetched:
                 content = fetched
@@ -386,7 +392,9 @@ class PatreonDownloader(BaseNativeDownloader):
         sidecar_path.write_text(json.dumps(data, indent=2))
         return sidecar_path
 
-    def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
+    def write_post_record(
+        self, post: dict, artist_slug: str, *, revisit: bool = False,
+    ) -> PostRecordOutcome:
         """Write a post-ONLY sidecar (no media file) for a media-less post, so
         the importer can still upsert the Post + its body — text posts often hold
         the only copy of an external <a href> link. Named `_post.json`: the
@@ -397,6 +405,18 @@ class PatreonDownloader(BaseNativeDownloader):
         Returns a PostRecordOutcome (path None when the post has no id) carrying
         the captured body's shape — post_type + final char count — so the engine
         can log per-post handling without re-reading the post itself.
+
+        `revisit=True` is the tick re-reading a post it already captured
+        (ingest_core's revisit window, #...). Two differences, both about not
+        making an update cost more than it is worth:
+
+        * no detail-fetch — the body comes from the feed response already in
+          hand, so a revisit costs zero requests;
+        * a body that comes back empty writes NOTHING and returns `path=None`.
+          On a first capture an empty body is the truth about the post; on a
+          revisit it usually just means this post's body only ever came from
+          the detail endpoint we just declined to call, and writing it would
+          blank a stored body to say something we never learned.
         """
         attrs = post.get("attributes") or {}
         title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
@@ -406,9 +426,17 @@ class PatreonDownloader(BaseNativeDownloader):
             return PostRecordOutcome(
                 path=None, post_type=post_type, title=title, body_chars=0,
             )
+        if revisit:
+            feed_body = post_body_html(attrs)
+            if not (isinstance(feed_body, str) and feed_body.strip()):
+                return PostRecordOutcome(
+                    path=None, post_type=post_type, title=title, body_chars=0,
+                )
         post_dir = self.images_root / artist_slug / "patreon" / post_dir_name(post)
         post_dir.mkdir(parents=True, exist_ok=True)
-        path = self._write_sidecar_data(post, post_dir / "_post.json")
+        path = self._write_sidecar_data(
+            post, post_dir / "_post.json", detail_fetch=not revisit,
+        )
         # _write_sidecar_data has by now memoized any detail-fetched body onto
         # post["attributes"]["content"], so re-read it for the FINAL char count.
         body = (post.get("attributes") or {}).get("content")

backend/app/services/subscribestar_client.py

@@ -742,8 +742,15 @@ class SubscribeStarClient:
 
     @staticmethod
     def post_meta(post: dict) -> dict:
-        """Title + date for the preview sample. Title is synthesized from the body
-        (SubscribeStar has no title field)."""
+        """Title + date. Title is None — SubscribeStar has no title field, and
+        the importer synthesizes one from the body.
+
+        `date` is the contract the core's REVISIT WINDOW reads (2026-09-23):
+        ISO-8601 or None. NAIVE here, because `_parse_ss_datetime` renders a
+        parsed local timestamp with no zone; the core reads a naive date as UTC
+        rather than discarding it, since a date it refuses to read is a post
+        the window can never reach.
+        """
         attrs = post.get("attributes") or {}
         return {"title": None, "date": attrs.get("published_at")}
 

backend/app/services/subscribestar_downloader.py

@@ -184,10 +184,20 @@ class SubscribeStarDownloader(BaseNativeDownloader):
         sidecar_path.write_text(json.dumps(data, indent=2))
         return sidecar_path
 
-    def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
+    def write_post_record(
+        self, post: dict, artist_slug: str, *, revisit: bool = False,
+    ) -> PostRecordOutcome:
         """Write the post-first `_post.json` (body/links/metadata) — the sole
         writer of the post record on the native path. SubscribeStar's body is
-        already in the feed HTML, so no detail-fetch is needed."""
+        already in the feed HTML, so no detail-fetch is needed.
+
+        `revisit=True` is the tick re-reading a post it already captured
+        (ingest_core's revisit window). The no-detail-fetch half of that
+        contract is free here — there is no detail endpoint — but the
+        don't-blank-a-stored-body half still applies: a chunk that parsed with
+        no content must not overwrite a body we already have. Same guarantee as
+        the Patreon downloader, for the same reason, so a walk behaves the same
+        on both platforms."""
         attrs = post.get("attributes") or {}
         title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
         post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
@@ -196,6 +206,12 @@ class SubscribeStarDownloader(BaseNativeDownloader):
             return PostRecordOutcome(
                 path=None, post_type=post_type, title=title, body_chars=0,
             )
+        if revisit:
+            body = attrs.get("content")
+            if not (isinstance(body, str) and body.strip()):
+                return PostRecordOutcome(
+                    path=None, post_type=post_type, title=title, body_chars=0,
+                )
         post_dir = self.images_root / artist_slug / "subscribestar" / post_dir_name(post)
         post_dir.mkdir(parents=True, exist_ok=True)
         path = self._write_sidecar_data(post, post_dir / "_post.json")