Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
93e37681b7 | ||
|
|
80ef9bce48 | ||
|
|
3898ce7be4 | ||
|
|
64ca858574 | ||
|
|
91be9df671 | ||
|
|
412edec028 | ||
|
|
937421485d | ||
|
|
9d0c0b7da8 | ||
|
|
43b778aa04 | ||
|
|
9cbdb70e13 | ||
|
|
8e4d252ae4 | ||
|
|
bd06794647 | ||
|
|
fdd3e01f56 | ||
|
|
f575cfb93b | ||
|
|
c82fb308b6 | ||
|
|
717b601c81 | ||
|
|
cfa4fb4084 | ||
|
|
2aa2002f22 | ||
|
|
66ff671f09 | ||
|
|
19aece1fc4 | ||
|
|
c9089b1d03 | ||
|
|
644d538bab | ||
|
|
ff35da4743 | ||
|
|
2f66de2928 | ||
|
|
8cf8d2ca4d | ||
|
|
94e7d20792 | ||
|
|
fb605af959 | ||
|
|
4c56cf121f | ||
|
|
b1d58bc3b8 | ||
|
|
9564d073b9 | ||
|
|
65386f02a0 | ||
|
|
f87a06a6bd | ||
|
|
5d284aae9f | ||
|
|
af7b5c95e9 | ||
|
|
667b05f14e | ||
|
|
8de7ccd07d | ||
|
|
d65f0b2091 |
@@ -242,20 +242,32 @@ jobs:
|
||||
id: tag
|
||||
run: |
|
||||
# Three trigger shapes:
|
||||
# refs/tags/v… → tag-push: publish ONLY the immutable version
|
||||
# tag (e.g. :v26.05.26.5). Don't touch :latest;
|
||||
# that already got published by the main-push
|
||||
# build for the merge commit.
|
||||
# refs/heads/main → push to main (incl. PR merge commits):
|
||||
# publish :main + :latest (floating).
|
||||
# refs/tags/v… → tag-push: opt-in milestone label (vYY.MM.DD,
|
||||
# no `.N` per family release-posture rule).
|
||||
# Publish ONLY the immutable version tag;
|
||||
# don't touch :latest (the main-push build
|
||||
# for the merge commit already did that).
|
||||
# refs/heads/main → push to main: publish :main + :latest
|
||||
# (floating) AND :c-<short_sha> (immutable
|
||||
# per-commit rollback substrate, per family
|
||||
# release-posture rule "Tags are milestones,
|
||||
# not gates — commit-SHA images are the
|
||||
# rollback unit"). Rollback to any commit
|
||||
# becomes `docker pull …:c-<sha>` without a
|
||||
# release ceremony.
|
||||
# anything else → safety net; shouldn't fire given the `on:`
|
||||
# config above (dev was dropped). Tag :dev to
|
||||
# surface the unexpected run in the registry.
|
||||
# config above. Tag :dev to surface the
|
||||
# unexpected run in the registry.
|
||||
# POSIX-safe substring (the runner shell is dash/BusyBox sh, not
|
||||
# bash — `${var:0:7}` errors with "Bad substitution"; cut works
|
||||
# everywhere). Operator-flagged 2026-06-01 after first :c-<sha>
|
||||
# main-push build failed at this step.
|
||||
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:main,git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
@@ -286,13 +298,19 @@ jobs:
|
||||
id: tag
|
||||
run: |
|
||||
# Mirrors build-web's three-shape logic (tag-push / main-push /
|
||||
# safety-net dev). The -ml image follows the same release cadence
|
||||
# as the web image.
|
||||
# safety-net dev) including the per-commit :c-<short_sha> tag
|
||||
# on main-push per the family release-posture rule. The -ml
|
||||
# image follows the same release cadence as the web image.
|
||||
# POSIX-safe substring (the runner shell is dash/BusyBox sh, not
|
||||
# bash — `${var:0:7}` errors with "Bad substitution"; cut works
|
||||
# everywhere). Operator-flagged 2026-06-01 after first :c-<sha>
|
||||
# main-push build failed at this step.
|
||||
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:main,git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:dev" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""drop artist + copyright ml thresholds; lower general default to 0.50
|
||||
|
||||
Revision ID: 0029
|
||||
Revises: 0028
|
||||
Create Date: 2026-06-01
|
||||
|
||||
Operator-flagged 2026-06-01: the view modal's Suggestions panel hides
|
||||
most general-category predictions because the default threshold is
|
||||
0.95. Lowering the default to 0.50 (matches character) so general
|
||||
suggestions surface more aggressively; the value remains tunable in
|
||||
Settings → ML.
|
||||
|
||||
Same change retires two ML suggestion categories whose Tag.kind
|
||||
surfaces are unused:
|
||||
|
||||
- `artist`: retired in FC-2d-vii-c — artist identity is acquisition-
|
||||
derived (image_record.artist_id), never ML-inferred. The threshold
|
||||
column was a leftover from before that retirement.
|
||||
- `copyright`: retired 2026-06-01 — the app uses `fandom` for the
|
||||
franchise/copyright concept (per TagsView.vue's doc comment); no
|
||||
Tag rows of kind=copyright exist, and the threshold column never
|
||||
fed anything user-visible.
|
||||
|
||||
Both columns are dropped from ml_settings; the existing row's
|
||||
suggestion_threshold_general value is bumped from 0.95 to 0.50 iff
|
||||
it's still at the old default, so deployed installs pick up the new
|
||||
UX without overriding any operator tuning.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0029"
|
||||
down_revision: Union[str, None] = "0028"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Bump the general threshold for installs still at the old default.
|
||||
op.execute(text(
|
||||
"UPDATE ml_settings "
|
||||
"SET suggestion_threshold_general = 0.50 "
|
||||
"WHERE id = 1 AND suggestion_threshold_general = 0.95"
|
||||
))
|
||||
op.drop_column("ml_settings", "suggestion_threshold_artist")
|
||||
op.drop_column("ml_settings", "suggestion_threshold_copyright")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Restore the columns with their prior defaults. The bump from
|
||||
# 0.95 → 0.50 isn't reversible without remembering whether the
|
||||
# operator had explicitly set 0.95 (unlikely — that was just the
|
||||
# default) so we leave the current general value as-is.
|
||||
from sqlalchemy import Column, Float
|
||||
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
Column(
|
||||
"suggestion_threshold_artist",
|
||||
Float, nullable=False, server_default="0.30",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
Column(
|
||||
"suggestion_threshold_copyright",
|
||||
Float, nullable=False, server_default="0.50",
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,145 @@
|
||||
"""nullable post.source_id + denormalized post.artist_id; retire sidecar synthetics
|
||||
|
||||
Revision ID: 0030
|
||||
Revises: 0029
|
||||
Create Date: 2026-06-01
|
||||
|
||||
Operator-asked 2026-06-01 after the Dymkens orphan investigation: the
|
||||
sidecar synthetic Source pattern (`sidecar:<platform>:<slug>` rows
|
||||
with enabled=false) was technically correct but misled the operator
|
||||
into thinking they had phantom subscriptions. The synthetics existed
|
||||
solely to satisfy `Post.source_id NOT NULL` for filesystem-imported
|
||||
content with no real subscription.
|
||||
|
||||
This migration makes the data model honest:
|
||||
|
||||
1. **Post gets a denormalized `artist_id` column** so artist filters
|
||||
work without traversing `Post → Source.artist_id`. Backfilled from
|
||||
the existing Source linkage, then NOT NULL'd.
|
||||
2. **`Post.source_id` becomes nullable**, FK ondelete `CASCADE` → `SET
|
||||
NULL`. Deleting a Source detaches its Posts instead of destroying
|
||||
imported content (semantically: subscription ends, archive stays).
|
||||
3. **`ImageProvenance.source_id` becomes nullable** with the same FK
|
||||
semantic change.
|
||||
4. **Sidecar synthetic Sources are deleted** — first NULL out the
|
||||
FKs from Post + ImageProvenance pointing at them (so the implicit
|
||||
CASCADE doesn't fire), then delete. DownloadEvent FK is unchanged
|
||||
(still CASCADE'd, NOT NULL'd) — synthetics have `enabled=false`
|
||||
so no events exist for them.
|
||||
|
||||
Uniqueness handling: the existing `uq_post_source_external_id`
|
||||
(source_id, external_post_id) keeps working for source-bound Posts
|
||||
(Postgres treats NULL != NULL so NULL-source rows aren't deduped by
|
||||
it). A second partial unique index covers the NULL-source case on
|
||||
(artist_id, external_post_id) so filesystem-imported posts still
|
||||
dedupe within an artist.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy import text
|
||||
|
||||
revision: str = "0030"
|
||||
down_revision: Union[str, None] = "0029"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
|
||||
# Step 1: add Post.artist_id, initially nullable for backfill.
|
||||
# FK naming follows the Base.metadata naming_convention
|
||||
# (fk_<table>_<column>_<referred_table>) — alembic 0001 set this up.
|
||||
op.add_column(
|
||||
"post",
|
||||
sa.Column("artist_id", sa.Integer, nullable=True),
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_post_artist_id_artist", "post", "artist",
|
||||
["artist_id"], ["id"], ondelete="CASCADE",
|
||||
)
|
||||
|
||||
# Step 2: backfill from Source.artist_id (every existing Post has a
|
||||
# Source today, so every row gets populated).
|
||||
conn.execute(text("""
|
||||
UPDATE post p
|
||||
SET artist_id = s.artist_id
|
||||
FROM source s
|
||||
WHERE p.source_id = s.id AND p.artist_id IS NULL
|
||||
"""))
|
||||
|
||||
# Sanity: count any remaining NULLs. Should be zero pre-this-migration.
|
||||
remaining = conn.execute(text(
|
||||
"SELECT COUNT(*) FROM post WHERE artist_id IS NULL"
|
||||
)).scalar_one()
|
||||
if remaining:
|
||||
raise RuntimeError(
|
||||
f"alembic 0030: {remaining} post rows have no resolvable "
|
||||
f"artist_id after backfill. Investigate before continuing."
|
||||
)
|
||||
|
||||
# Step 3: enforce NOT NULL + add index for artist-filter queries.
|
||||
op.alter_column("post", "artist_id", nullable=False)
|
||||
op.create_index("ix_post_artist_id", "post", ["artist_id"])
|
||||
|
||||
# Step 4: relax post.source_id + flip FK to SET NULL. The original FK
|
||||
# name from alembic 0001 is `fk_post_source_id_source` per the
|
||||
# NAMING_CONVENTION in models/base.py.
|
||||
op.alter_column("post", "source_id", nullable=True)
|
||||
op.drop_constraint("fk_post_source_id_source", "post", type_="foreignkey")
|
||||
op.create_foreign_key(
|
||||
"fk_post_source_id_source", "post", "source",
|
||||
["source_id"], ["id"], ondelete="SET NULL",
|
||||
)
|
||||
|
||||
# Step 5: relax image_provenance.source_id + flip FK to SET NULL.
|
||||
op.alter_column("image_provenance", "source_id", nullable=True)
|
||||
op.drop_constraint(
|
||||
"fk_image_provenance_source_id_source", "image_provenance",
|
||||
type_="foreignkey",
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_image_provenance_source_id_source", "image_provenance", "source",
|
||||
["source_id"], ["id"], ondelete="SET NULL",
|
||||
)
|
||||
|
||||
# Step 6: partial unique index on (artist_id, external_post_id) for
|
||||
# NULL-source Posts. The existing uq_post_source_external_id keeps
|
||||
# guarding source-bound rows; NULL-source rows now dedupe within
|
||||
# an artist.
|
||||
op.execute(
|
||||
"CREATE UNIQUE INDEX uq_post_artist_external_id_null_source "
|
||||
"ON post (artist_id, external_post_id) "
|
||||
"WHERE source_id IS NULL"
|
||||
)
|
||||
|
||||
# Step 7: retire sidecar synthetic Sources. NULL out the references
|
||||
# FIRST (the new FK is SET NULL so CASCADE wouldn't fire anyway, but
|
||||
# being explicit makes the intent clear). Then delete the synthetic
|
||||
# source rows. Any DownloadEvent rows under synthetics CASCADE-die
|
||||
# with the source — synthetics have enabled=false so there shouldn't
|
||||
# be any in practice.
|
||||
conn.execute(text("""
|
||||
UPDATE post
|
||||
SET source_id = NULL
|
||||
WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%')
|
||||
"""))
|
||||
conn.execute(text("""
|
||||
UPDATE image_provenance
|
||||
SET source_id = NULL
|
||||
WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%')
|
||||
"""))
|
||||
deleted = conn.execute(text(
|
||||
"DELETE FROM source WHERE url LIKE 'sidecar:%' RETURNING id"
|
||||
)).rowcount
|
||||
print(f"alembic 0030: deleted {deleted} sidecar synthetic source rows")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Lossy migration — the deleted sidecar synthetics can't be
|
||||
# restored from the orphan post.source_id / image_provenance.source_id
|
||||
# values, and the partial unique index encodes a constraint that
|
||||
# NULL-source Posts may now exist. No safe downgrade.
|
||||
pass
|
||||
@@ -0,0 +1,45 @@
|
||||
"""source.backfill_runs_remaining: sticky deep-scan mode
|
||||
|
||||
Revision ID: 0031
|
||||
Revises: 0030
|
||||
Create Date: 2026-06-01
|
||||
|
||||
Tick vs backfill mode for subscription downloads. When
|
||||
`backfill_runs_remaining > 0`, the next N download runs use
|
||||
`skip: True` + 30-min timeout (walk full history). When 0, runs use
|
||||
`skip: "exit:20"` + 14.5-min timeout (catch-up mode, exits early once
|
||||
20 contiguous archived items are seen).
|
||||
|
||||
Operator-flagged 2026-06-01 (Knuxy run #38887): a creator with ~550
|
||||
archived posts saturates the 870s catch-up timeout even when there is
|
||||
no new content, because gallery-dl's default `skip: True` keeps walking.
|
||||
Tick mode short-circuits that; backfill mode is the explicit opt-in for
|
||||
deep history scans.
|
||||
|
||||
Default 0 (all existing subscriptions start in tick mode).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0031"
|
||||
down_revision: Union[str, None] = "0030"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"source",
|
||||
sa.Column(
|
||||
"backfill_runs_remaining",
|
||||
sa.Integer,
|
||||
nullable=False,
|
||||
server_default="0",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source", "backfill_runs_remaining")
|
||||
@@ -57,6 +57,24 @@ def _sha256(path: Path) -> str:
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
@extension_bp.route("/probe", methods=["GET"])
|
||||
async def probe_source():
|
||||
"""Read-only resolution of a creator-page URL: tells the extension
|
||||
whether this URL is already a Source, is for an Artist that exists
|
||||
but with a different URL, is brand new, or doesn't match any known
|
||||
platform pattern. Drives the content-script chip's color/copy
|
||||
BEFORE the operator clicks, so the button can show 'already added'
|
||||
without requiring an add-attempt."""
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return _bad("invalid_body", detail="url query parameter is required")
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
result = await ExtensionService(session).probe(url)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@extension_bp.route("/quick-add-source", methods=["POST"])
|
||||
async def quick_add_source():
|
||||
body = await request.get_json(silent=True)
|
||||
|
||||
@@ -9,9 +9,7 @@ ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
||||
|
||||
|
||||
_EDITABLE = (
|
||||
"suggestion_threshold_artist",
|
||||
"suggestion_threshold_character",
|
||||
"suggestion_threshold_copyright",
|
||||
"suggestion_threshold_general",
|
||||
"centroid_similarity_threshold",
|
||||
"min_reference_images",
|
||||
@@ -28,9 +26,7 @@ async def get_settings():
|
||||
).scalar_one()
|
||||
return jsonify(
|
||||
{
|
||||
"suggestion_threshold_artist": s.suggestion_threshold_artist,
|
||||
"suggestion_threshold_character": s.suggestion_threshold_character,
|
||||
"suggestion_threshold_copyright": s.suggestion_threshold_copyright,
|
||||
"suggestion_threshold_general": s.suggestion_threshold_general,
|
||||
"centroid_similarity_threshold": s.centroid_similarity_threshold,
|
||||
"min_reference_images": s.min_reference_images,
|
||||
|
||||
@@ -120,6 +120,31 @@ async def delete_source(source_id: int):
|
||||
return "", 204
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
||||
async def set_backfill(source_id: int):
|
||||
"""Plan #544: arm a source for backfill mode for the next N download
|
||||
runs. Body: `{"runs": int}` (1..10, default 3). Returns the updated
|
||||
source dict. While backfill_runs_remaining > 0, downloads use
|
||||
gallery-dl's full-walk config (skip: True + 30-min timeout) instead
|
||||
of the catch-up default (skip: "exit:20" + 14.5-min timeout)."""
|
||||
payload = await request.get_json(silent=True) or {}
|
||||
runs = payload.get("runs", 3)
|
||||
try:
|
||||
runs = int(runs)
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_runs", detail="runs must be an integer")
|
||||
async with get_session() as session:
|
||||
try:
|
||||
record = await SourceService(session).set_backfill_runs(
|
||||
source_id, runs,
|
||||
)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except ValueError as exc:
|
||||
return _bad("invalid_runs", detail=str(exc))
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
||||
async def check_source(source_id: int):
|
||||
"""FC-3c: enqueue a download for this source.
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""Thumbnail admin API: backfill trigger."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from quart import Blueprint, jsonify
|
||||
|
||||
thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
||||
@@ -7,7 +9,20 @@ thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
||||
|
||||
@thumbnails_bp.route("/backfill", methods=["POST"])
|
||||
async def trigger_backfill():
|
||||
from ..tasks.thumbnail import backfill_thumbnails
|
||||
"""Run the backfill scan synchronously, return the counts. The actual
|
||||
thumbnail generation work is still off-loaded to the thumbnail Celery
|
||||
queue via `generate_thumbnail.delay()` per missing row — so this
|
||||
handler is fast even on a 100k-image library (a scan is just SELECT
|
||||
id, thumbnail_path + a file.stat() per row, no heavy work).
|
||||
|
||||
r = backfill_thumbnails.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
Operator-flagged 2026-06-01: the previous fire-and-forget shape
|
||||
returned `{celery_task_id}` only, so the admin UI had no idea whether
|
||||
backfill found 0 or 5000 candidates — \"found nothing\" was
|
||||
indistinguishable from \"the worker isn't picking up the task.\""""
|
||||
from ..tasks.thumbnail import _run_backfill_scan
|
||||
|
||||
# Sync scan inside an executor so we don't block the event loop.
|
||||
counts = await asyncio.get_running_loop().run_in_executor(
|
||||
None, _run_backfill_scan,
|
||||
)
|
||||
return jsonify(counts), 200
|
||||
|
||||
@@ -34,8 +34,12 @@ class ImageProvenance(Base):
|
||||
post_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
# Nullable since alembic 0030 — provenance rows for filesystem-imported
|
||||
# content with no subscription have NULL source_id. FK ondelete SET
|
||||
# NULL so deleting a Source detaches its provenance rows instead of
|
||||
# destroying the linkage between image and post.
|
||||
source_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
captured_at: Mapped[datetime] = mapped_column(
|
||||
|
||||
@@ -15,17 +15,14 @@ class MLSettings(Base):
|
||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
suggestion_threshold_artist: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.30
|
||||
)
|
||||
suggestion_threshold_character: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
)
|
||||
suggestion_threshold_copyright: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
)
|
||||
# Default lowered 0.95 → 0.50 on 2026-06-01 — operator-flagged that
|
||||
# 0.95 hid most general suggestions. Operator-tunable via Settings →
|
||||
# ML if too noisy.
|
||||
suggestion_threshold_general: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.95
|
||||
Float, nullable=False, default=0.50
|
||||
)
|
||||
centroid_similarity_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.55
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
"""Post — provenance anchor for content downloaded from a Source.
|
||||
"""Post — provenance anchor for one creator post (may contain many images).
|
||||
|
||||
A Post is one creator post; it may contain many images/videos.
|
||||
`source_id` is nullable since alembic 0030 — filesystem-imported posts
|
||||
with no live subscription have NULL source_id. `artist_id` is the
|
||||
denormalized always-present link to the creator (added in 0030 so
|
||||
artist-filter queries don't depend on the Source detour).
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
@@ -14,12 +17,25 @@ from .base import Base
|
||||
class Post(Base):
|
||||
__tablename__ = "post"
|
||||
__table_args__ = (
|
||||
# Source-bound dedup. Postgres treats NULL != NULL so rows
|
||||
# with source_id IS NULL aren't deduped by this constraint;
|
||||
# the partial unique index `uq_post_artist_external_id_null_source`
|
||||
# (created in alembic 0030) covers that case via
|
||||
# (artist_id, external_post_id).
|
||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
source_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
# Denormalized; always equals source.artist_id when source_id is set
|
||||
# (the importer is responsible for keeping them consistent on insert).
|
||||
# Filter queries (artist detail, artist-scoped posts feed) use this
|
||||
# directly instead of joining through Source.
|
||||
artist_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("artist.id", ondelete="CASCADE"),
|
||||
nullable=False, index=True,
|
||||
)
|
||||
external_post_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
post_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
@@ -29,4 +29,13 @@ class Source(Base):
|
||||
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
# alembic 0031: sticky deep-scan budget. When > 0, the next N download
|
||||
# runs use gallery-dl's full-walk config (skip: True + 1800s timeout);
|
||||
# when 0, runs use tick mode (skip: "exit:20" + 870s, exits early once
|
||||
# 20 contiguous archived items are seen). Auto-decrements per run, with
|
||||
# an auto-reset to 0 on clean exit + zero downloads (queue drained).
|
||||
backfill_runs_remaining: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0, server_default="0",
|
||||
)
|
||||
|
||||
artist = relationship("Artist", back_populates="sources")
|
||||
|
||||
@@ -58,13 +58,17 @@ class ArtistService:
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
# Posts under this artist that have at least one image attached.
|
||||
# Use Post.artist_id (alembic 0030) for the artist filter; keep
|
||||
# the ImageProvenance JOIN so date bounds reflect only image-
|
||||
# bearing posts (matches the original semantic). NULL-source
|
||||
# posts now surface too.
|
||||
date_row = (
|
||||
await self.session.execute(
|
||||
select(func.min(Post.post_date), func.max(Post.post_date))
|
||||
.select_from(Post)
|
||||
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
||||
.join(Source, Source.id == ImageProvenance.source_id)
|
||||
.where(Source.artist_id == aid)
|
||||
.where(Post.artist_id == aid)
|
||||
)
|
||||
).first()
|
||||
dmin, dmax = date_row if date_row else (None, None)
|
||||
@@ -98,14 +102,14 @@ class ArtistService:
|
||||
)
|
||||
).all()
|
||||
|
||||
# Same Post.artist_id direct filter — counts NULL-source posts too.
|
||||
month = func.date_trunc("month", Post.post_date).label("m")
|
||||
activity = (
|
||||
await self.session.execute(
|
||||
select(month, func.count(func.distinct(ImageProvenance.image_record_id)))
|
||||
.select_from(Post)
|
||||
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
||||
.join(Source, Source.id == ImageProvenance.source_id)
|
||||
.where(and_(Source.artist_id == aid, Post.post_date.isnot(None)))
|
||||
.where(and_(Post.artist_id == aid, Post.post_date.isnot(None)))
|
||||
.group_by(month)
|
||||
.order_by(month)
|
||||
)
|
||||
@@ -114,9 +118,7 @@ class ArtistService:
|
||||
post_count = (
|
||||
await self.session.execute(
|
||||
select(func.count(func.distinct(Post.id)))
|
||||
.select_from(Post)
|
||||
.join(Source, Source.id == Post.source_id)
|
||||
.where(Source.artist_id == aid)
|
||||
.where(Post.artist_id == aid)
|
||||
)
|
||||
).scalar_one()
|
||||
|
||||
|
||||
@@ -354,18 +354,35 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
||||
Returns:
|
||||
dry_run=True: {"count": N, "sample_names": [first 50]}
|
||||
dry_run=False: {"deleted": N, "sample_names": [first 50]}
|
||||
|
||||
Implementation note: the previous SELECT-ids → DELETE-WHERE-IN
|
||||
pattern was vulnerable to the psycopg 65535-parameter ceiling on
|
||||
libraries with tag explosions. The live delete now runs a single
|
||||
DELETE with the same NOT-IN predicate find_unused_tags uses, so
|
||||
the row count scales without binding every id as a parameter.
|
||||
Audit 2026-06-02.
|
||||
"""
|
||||
unused = find_unused_tags(session)
|
||||
sample = [t.name for t in unused[:50]]
|
||||
sample_rows = find_unused_tags(session, limit=50)
|
||||
sample = [t.name for t in sample_rows]
|
||||
used_via_image_tag = select(image_tag.c.tag_id).distinct()
|
||||
used_via_series = select(SeriesPage.series_tag_id).where(
|
||||
SeriesPage.series_tag_id.is_not(None)
|
||||
).distinct()
|
||||
if dry_run:
|
||||
return {"count": len(unused), "sample_names": sample}
|
||||
ids = [t.id for t in unused]
|
||||
if ids:
|
||||
session.execute(
|
||||
Tag.__table__.delete().where(Tag.id.in_(ids))
|
||||
)
|
||||
session.commit()
|
||||
return {"deleted": len(ids), "sample_names": sample}
|
||||
count = session.execute(
|
||||
select(func.count())
|
||||
.select_from(Tag)
|
||||
.where(Tag.id.not_in(used_via_image_tag))
|
||||
.where(Tag.id.not_in(used_via_series))
|
||||
).scalar_one()
|
||||
return {"count": count, "sample_names": sample}
|
||||
result = session.execute(
|
||||
Tag.__table__.delete()
|
||||
.where(Tag.id.not_in(used_via_image_tag))
|
||||
.where(Tag.id.not_in(used_via_series))
|
||||
)
|
||||
session.commit()
|
||||
return {"deleted": result.rowcount or 0, "sample_names": sample}
|
||||
|
||||
|
||||
# Legacy tags FC no longer uses, in two shapes:
|
||||
|
||||
@@ -25,7 +25,14 @@ from sqlalchemy.orm import joinedload
|
||||
|
||||
from ..models import Artist, DownloadEvent, Source
|
||||
from .credential_service import CredentialService
|
||||
from .gallery_dl import GalleryDLService, SourceConfig
|
||||
from .gallery_dl import (
|
||||
BACKFILL_SKIP_VALUE,
|
||||
BACKFILL_TIMEOUT_SECONDS,
|
||||
TICK_SKIP_VALUE,
|
||||
ErrorType,
|
||||
GalleryDLService,
|
||||
SourceConfig,
|
||||
)
|
||||
from .importer import Importer
|
||||
from .patreon_resolver import resolve_campaign_id
|
||||
from .scheduler_service import set_platform_cooldown
|
||||
@@ -84,6 +91,18 @@ class DownloadService:
|
||||
ctx = setup
|
||||
|
||||
source_config = SourceConfig.from_dict(ctx["config_overrides"] or {})
|
||||
# alembic 0031 / plan #544: derive skip_value + timeout from the
|
||||
# source's backfill_runs_remaining counter. When > 0, walk the full
|
||||
# post history (skip: True + 1800s); when 0, exit gallery-dl after
|
||||
# 20 contiguous archived items (skip: "exit:20" + the default
|
||||
# 870s). Operator sets backfill via POST /api/sources/{id}/backfill.
|
||||
backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
|
||||
if backfill_remaining > 0:
|
||||
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
||||
source_config.timeout = BACKFILL_TIMEOUT_SECONDS
|
||||
else:
|
||||
skip_value = TICK_SKIP_VALUE
|
||||
|
||||
effective_url = _effective_url(
|
||||
ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
|
||||
)
|
||||
@@ -95,6 +114,7 @@ class DownloadService:
|
||||
source_config=source_config,
|
||||
cookies_path=ctx["cookies_path"],
|
||||
auth_token=ctx["auth_token"],
|
||||
skip_value=skip_value,
|
||||
)
|
||||
|
||||
resolved_campaign_id: str | None = None
|
||||
@@ -121,6 +141,7 @@ class DownloadService:
|
||||
source_config=source_config,
|
||||
cookies_path=ctx["cookies_path"],
|
||||
auth_token=ctx["auth_token"],
|
||||
skip_value=skip_value,
|
||||
)
|
||||
|
||||
return await self._phase3_persist(
|
||||
@@ -185,6 +206,7 @@ class DownloadService:
|
||||
"config_overrides": dict(source.config_overrides or {}),
|
||||
"cookies_path": cookies_path,
|
||||
"auth_token": auth_token,
|
||||
"backfill_runs_remaining": source.backfill_runs_remaining or 0,
|
||||
}
|
||||
|
||||
async def _phase3_persist(
|
||||
@@ -238,6 +260,40 @@ class DownloadService:
|
||||
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
# Enqueue thumbnail + ML for newly-attached images, matching
|
||||
# the filesystem-import path (tasks/import_file.py:228-239).
|
||||
# Importer.attach_in_place deliberately skips inline thumb
|
||||
# generation to keep the import queue moving; the calling
|
||||
# task is responsible for the enqueue. Operator-flagged
|
||||
# 2026-06-01: without this, every downloaded image stayed
|
||||
# at thumbnail_path=NULL until a periodic backfill swept
|
||||
# it up, surfacing as broken-thumbnail tiles in the gallery
|
||||
# for hours after a download landed. Lazy import to avoid
|
||||
# circular-import risk between this service and the
|
||||
# tasks/* modules that import it.
|
||||
from ..tasks.ml import tag_and_embed
|
||||
from ..tasks.thumbnail import generate_thumbnail
|
||||
ids = list(result.member_image_ids)
|
||||
if result.image_id is not None and result.image_id not in ids:
|
||||
ids.append(result.image_id)
|
||||
for img_id in ids:
|
||||
generate_thumbnail.delay(img_id)
|
||||
tag_and_embed.delay(img_id)
|
||||
elif result.status == "attached":
|
||||
# Non-media or extracted archive captured as PostAttachment
|
||||
# (FC-2d-iii). The canonical copy lives in the attachments
|
||||
# store; the original download path is now redundant —
|
||||
# mirror duplicate_hash cleanup so we don't keep two copies.
|
||||
# Operator-flagged 2026-06-02 (Lustria OST zip).
|
||||
import_summary["attached"] += 1
|
||||
try:
|
||||
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
|
||||
"duplicate_hash", "duplicate_phash",
|
||||
):
|
||||
@@ -246,6 +302,12 @@ class DownloadService:
|
||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||
except OSError:
|
||||
pass
|
||||
elif result.status == "skipped":
|
||||
# Soft skip (too_small, too_transparent, invalid_image) —
|
||||
# the file just didn't qualify, not a download/ingest
|
||||
# failure. Don't flag the run as error; the file stays
|
||||
# on disk for operator inspection.
|
||||
import_summary["skipped"] += 1
|
||||
else:
|
||||
import_summary["errors"] += 1
|
||||
|
||||
@@ -259,12 +321,21 @@ class DownloadService:
|
||||
run_stats["quarantined_count"] = dl_result.files_quarantined
|
||||
stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr)
|
||||
|
||||
status = "ok" if (dl_result.success and import_summary["errors"] == 0) else "error"
|
||||
# Plan #544: PARTIAL means the run downloaded ≥1 file but the
|
||||
# subprocess didn't finish in budget (typically wall-clock timeout
|
||||
# mid-walk). Real work happened; the next tick continues via
|
||||
# gallery-dl's archive. NOT a failure for status purposes.
|
||||
if dl_result.success and import_summary["errors"] == 0:
|
||||
status = "ok"
|
||||
elif dl_result.error_type == ErrorType.PARTIAL and import_summary["errors"] == 0:
|
||||
status = "ok"
|
||||
else:
|
||||
status = "error"
|
||||
ev.status = status
|
||||
ev.finished_at = datetime.now(UTC)
|
||||
ev.files_count = import_summary["attached"]
|
||||
ev.bytes_downloaded = bytes_downloaded
|
||||
ev.error = dl_result.error_message if not dl_result.success else None
|
||||
ev.error = dl_result.error_message if status == "error" else None
|
||||
ev.metadata_ = {
|
||||
"run_stats": run_stats,
|
||||
"error_type": dl_result.error_type.value if dl_result.error_type else None,
|
||||
@@ -279,6 +350,19 @@ class DownloadService:
|
||||
source_id=ctx["source_id"], status=status, error_message=ev.error,
|
||||
error_type=dl_result.error_type.value if dl_result.error_type else None,
|
||||
)
|
||||
# Plan #544: backfill lifecycle — auto-complete when a clean
|
||||
# backfill run drained the queue (gallery-dl exited 0 + zero files
|
||||
# downloaded means there was nothing to fetch); otherwise decrement
|
||||
# the counter. Next tick falls back to tick mode once it hits 0.
|
||||
backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
|
||||
if backfill_remaining > 0:
|
||||
src = (await self.async_session.execute(
|
||||
select(Source).where(Source.id == ctx["source_id"])
|
||||
)).scalar_one()
|
||||
if dl_result.return_code == 0 and dl_result.files_downloaded == 0:
|
||||
src.backfill_runs_remaining = 0
|
||||
else:
|
||||
src.backfill_runs_remaining = max(0, backfill_remaining - 1)
|
||||
await self.async_session.commit()
|
||||
return event_id
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Artist, Source
|
||||
from ..utils.slug import slugify
|
||||
from .source_service import NEW_SOURCE_BACKFILL_RUNS
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
@@ -86,6 +87,67 @@ class ExtensionService:
|
||||
"created_artist": created_artist,
|
||||
}
|
||||
|
||||
async def probe(self, url: str) -> dict:
|
||||
"""Read-only resolution of a creator-page URL against the FC DB.
|
||||
Returns one of:
|
||||
- {state: 'unknown_platform'} — URL didn't match any
|
||||
platform's strict artist-page pattern
|
||||
- {state: 'new', platform, slug} — would create both
|
||||
artist and source on quick-add
|
||||
- {state: 'artist_match', platform, slug, artist}
|
||||
— artist exists, this
|
||||
exact URL isn't a Source yet (collapses the sidecar-synthetic
|
||||
case too — the synthetic anchor counts as an existing artist
|
||||
row but not as a pollable Source for this URL)
|
||||
- {state: 'source_match', platform, slug, artist, source}
|
||||
— exact (artist, platform,
|
||||
url) Source already exists
|
||||
|
||||
Side-effect-free: two SELECTs at most.
|
||||
"""
|
||||
try:
|
||||
platform, raw_slug = self._derive(url)
|
||||
except (UnknownPlatformError, InvalidUrlError):
|
||||
return {"state": "unknown_platform"}
|
||||
|
||||
slug = slugify(raw_slug)
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return {"state": "new", "platform": platform, "slug": slug}
|
||||
|
||||
artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist.id,
|
||||
Source.platform == platform,
|
||||
Source.url == url,
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
return {
|
||||
"state": "artist_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
}
|
||||
|
||||
return {
|
||||
"state": "source_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
"source": {
|
||||
"id": source.id,
|
||||
"artist_id": source.artist_id,
|
||||
"platform": source.platform,
|
||||
"url": source.url,
|
||||
"enabled": source.enabled,
|
||||
},
|
||||
}
|
||||
|
||||
def _derive(self, url: str) -> tuple[str, str]:
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
raise InvalidUrlError("url is empty")
|
||||
@@ -143,9 +205,16 @@ class ExtensionService:
|
||||
return existing, False
|
||||
sp = await self.session.begin_nested()
|
||||
try:
|
||||
# New subscription sources arm a few backfill runs so the
|
||||
# first ticks walk the full history (otherwise gallery-dl's
|
||||
# exit:20 short-circuits before the archive is built).
|
||||
# Mirrors SourceService.create — without it, Firefox quick-
|
||||
# add on a creator with >20 unsynced posts would surface
|
||||
# as "check failed" with no diagnosis. Audit 2026-06-02.
|
||||
src = Source(
|
||||
artist_id=artist_id, platform=platform,
|
||||
url=url, enabled=True,
|
||||
backfill_runs_remaining=NEW_SOURCE_BACKFILL_RUNS,
|
||||
)
|
||||
self.session.add(src)
|
||||
await self.session.flush()
|
||||
|
||||
@@ -39,9 +39,40 @@ class ErrorType(StrEnum):
|
||||
HTTP_ERROR = "http_error"
|
||||
UNSUPPORTED_URL = "unsupported_url"
|
||||
VALIDATION_FAILED = "validation_failed"
|
||||
# Run made real progress (downloaded ≥1 file) but did not finish in the
|
||||
# subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
|
||||
# mapping classifies this as "ok" because the next tick continues.
|
||||
PARTIAL = "partial"
|
||||
UNKNOWN_ERROR = "unknown_error"
|
||||
|
||||
|
||||
# Tick mode (routine cron polls): skip ≤20 contiguous already-archived
|
||||
# items, then exit gallery-dl. Established subscription with zero new
|
||||
# content exits in ~30s of HEAD requests instead of walking to the bottom
|
||||
# of the post history (which can be hours for prolific creators). 20 (not
|
||||
# 5) is operator-set headroom against any edge case where paywalled or
|
||||
# otherwise-non-downloadable items might interleave with archived ones —
|
||||
# 20 contiguous HEADs is still negligible.
|
||||
TICK_SKIP_VALUE = "exit:20"
|
||||
|
||||
# Backfill mode (operator-triggered deep scan): walk the full history.
|
||||
# Source.backfill_runs_remaining > 0 selects this mode; the longer
|
||||
# timeout below absorbs creators with thousands of posts.
|
||||
#
|
||||
# 30 seconds shy of Celery's hard `time_limit=1200` on download_source
|
||||
# (tasks/download.py:33). subprocess.run MUST raise TimeoutExpired
|
||||
# before Celery SIGKILLs the worker — same rationale as the tick
|
||||
# default at line 74. The audit (2026-06-02) caught this at 1800,
|
||||
# guaranteeing SIGKILL on any backfill that ran to its subprocess
|
||||
# budget: stdout/stderr lost, backfill_runs_remaining never
|
||||
# decrements, recovery sweep stamps generic "stranded" 30 min later.
|
||||
# Recreates the exact Knuxy #38275 failure mode the tick 870s default
|
||||
# was added to prevent. backfill_runs_remaining=3 still gives ~58
|
||||
# minutes of cumulative walk across three runs for prolific creators.
|
||||
BACKFILL_SKIP_VALUE = True
|
||||
BACKFILL_TIMEOUT_SECONDS = 1170
|
||||
|
||||
|
||||
# 30 seconds shy of download_source's Celery soft_time_limit (900s, see
|
||||
# tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before
|
||||
# Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race,
|
||||
@@ -56,12 +87,19 @@ _DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
||||
|
||||
@dataclass
|
||||
class SourceConfig:
|
||||
"""Per-source overrides loaded from Source.config_overrides JSON.
|
||||
|
||||
Note: the gallery-dl `skip` value (tick vs backfill, see TICK_SKIP_VALUE /
|
||||
BACKFILL_SKIP_VALUE) is NOT carried here — it derives from the
|
||||
Source.backfill_runs_remaining column at the download_service layer
|
||||
and is passed to _build_config_for_source as `skip_value`. Same for
|
||||
the per-run subprocess timeout.
|
||||
"""
|
||||
content_types: list[str] = field(default_factory=lambda: ["all"])
|
||||
sleep: float | None = None
|
||||
sleep_request: float | None = None
|
||||
directory_pattern: str | None = None
|
||||
filename_pattern: str | None = None
|
||||
skip_existing: bool = True
|
||||
save_metadata: bool = True
|
||||
timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS
|
||||
|
||||
@@ -73,7 +111,6 @@ class SourceConfig:
|
||||
sleep_request=data.get("sleep_request"),
|
||||
directory_pattern=data.get("directory_pattern"),
|
||||
filename_pattern=data.get("filename_pattern"),
|
||||
skip_existing=data.get("skip_existing", True),
|
||||
save_metadata=data.get("save_metadata", True),
|
||||
timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS),
|
||||
)
|
||||
@@ -231,6 +268,25 @@ class GalleryDLService:
|
||||
"part-directory": str(self._config_dir / "temp"),
|
||||
"retries": 3,
|
||||
"timeout": 120.0,
|
||||
# Forward Patreon as Referer/Origin to yt-dlp when it
|
||||
# fetches video manifests. Operator-flagged 2026-06-01
|
||||
# (DaferQ patreon): video posts hosted on Mux carry a JWT
|
||||
# playback restriction that checks Referer/Origin on every
|
||||
# request — not just the token signature. gallery-dl's
|
||||
# HEAD probe to stream.mux.com returns 200 (the token is
|
||||
# valid), but yt-dlp's actual GET-with-Range to fetch the
|
||||
# m3u8 manifest 403s because yt-dlp sends its own default
|
||||
# Referer, which Mux's policy rejects. Forcing the right
|
||||
# headers fixes the headers-only case; Mux IP-range
|
||||
# restrictions are unfixable from here.
|
||||
"ytdl": {
|
||||
"raw-options": {
|
||||
"http_headers": {
|
||||
"Referer": "https://www.patreon.com/",
|
||||
"Origin": "https://www.patreon.com",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"output": {"progress": True},
|
||||
}
|
||||
@@ -245,7 +301,18 @@ class GalleryDLService:
|
||||
platform: str,
|
||||
source_config: SourceConfig,
|
||||
artist_slug: str,
|
||||
skip_value: bool | str = BACKFILL_SKIP_VALUE,
|
||||
) -> dict:
|
||||
"""`skip_value` controls gallery-dl's archive-walk behavior:
|
||||
- True (BACKFILL_SKIP_VALUE): walk full post history, skipping
|
||||
archived items but continuing past them. Used in backfill mode.
|
||||
- "exit:20" (TICK_SKIP_VALUE): exit gallery-dl after 20
|
||||
contiguous archived items. Used in tick (routine catch-up)
|
||||
mode for fast no-op syncs on creators with deep history.
|
||||
- False: don't skip — redownload everything (not used in FC).
|
||||
The caller (download_service) chooses based on
|
||||
Source.backfill_runs_remaining.
|
||||
"""
|
||||
config = json.loads(json.dumps(self._get_default_config())) # deep copy
|
||||
|
||||
destination = str(self.images_root / artist_slug / platform)
|
||||
@@ -255,7 +322,7 @@ class GalleryDLService:
|
||||
config["extractor"]["sleep"] = source_config.sleep
|
||||
if source_config.sleep_request is not None:
|
||||
config["extractor"]["sleep-request"] = source_config.sleep_request
|
||||
config["extractor"]["skip"] = source_config.skip_existing
|
||||
config["extractor"]["skip"] = skip_value
|
||||
|
||||
if source_config.save_metadata:
|
||||
config["extractor"]["postprocessors"] = [
|
||||
@@ -394,6 +461,22 @@ class GalleryDLService:
|
||||
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
||||
)
|
||||
|
||||
# Partial-success: the subprocess exited non-zero (typically because
|
||||
# the wall-clock timeout fired mid-walk), but it had downloaded ≥1
|
||||
# file by then and no source-level error category fired. The work
|
||||
# the run DID do is real; gallery-dl's archive will pick up where
|
||||
# it left off on the next tick. Mapped to status="ok" downstream
|
||||
# (download_service.py) so this doesn't flag the source as
|
||||
# "needs attention." Operator-flagged 2026-06-01 after a Knuxy
|
||||
# patreon run downloaded hundreds of files then ran red on timeout.
|
||||
files_downloaded = self._count_downloaded_files(stdout)
|
||||
if not has_actual_error and files_downloaded > 0:
|
||||
return (
|
||||
ErrorType.PARTIAL,
|
||||
f"Downloaded {files_downloaded} file{'s' if files_downloaded != 1 else ''}; "
|
||||
"run did not complete in budget — next tick will continue",
|
||||
)
|
||||
|
||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||
|
||||
def _count_downloaded_files(self, stdout: str) -> int:
|
||||
@@ -545,6 +628,7 @@ class GalleryDLService:
|
||||
source_config: SourceConfig | None = None,
|
||||
cookies_path: str | None = None,
|
||||
auth_token: str | None = None,
|
||||
skip_value: bool | str = BACKFILL_SKIP_VALUE,
|
||||
) -> DownloadResult:
|
||||
start_time = time.time()
|
||||
started_at = datetime.now(UTC).isoformat()
|
||||
@@ -552,7 +636,9 @@ class GalleryDLService:
|
||||
if source_config is None:
|
||||
source_config = SourceConfig()
|
||||
|
||||
config = self._build_config_for_source(platform, source_config, artist_slug)
|
||||
config = self._build_config_for_source(
|
||||
platform, source_config, artist_slug, skip_value=skip_value,
|
||||
)
|
||||
|
||||
if cookies_path:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
@@ -769,7 +855,13 @@ class GalleryDLService:
|
||||
),
|
||||
)
|
||||
etype, msg = self._categorize_error(proc.returncode, proc.stdout, proc.stderr)
|
||||
if proc.returncode == 0 or etype == ErrorType.NO_NEW_CONTENT:
|
||||
# TIER_LIMITED proves auth worked — gallery-dl reached the
|
||||
# post, was told it's tier-gated. The download path treats
|
||||
# this as success (line 712); verify must too, or operators
|
||||
# rotate working cookies for no reason. Audit 2026-06-02.
|
||||
if proc.returncode == 0 or etype in (
|
||||
ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED,
|
||||
):
|
||||
return True, "Credentials valid — the feed authenticated."
|
||||
if etype == ErrorType.AUTH_ERROR:
|
||||
return False, msg
|
||||
|
||||
@@ -20,8 +20,9 @@ from datetime import datetime
|
||||
|
||||
from sqlalchemy import Select, and_, exists, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
|
||||
from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag
|
||||
from ..models.tag import image_tag
|
||||
|
||||
CURSOR_SEPARATOR = "|"
|
||||
@@ -133,10 +134,23 @@ def _provenance_clause(post_id, artist_id):
|
||||
ImageProvenance.post_id == post_id,
|
||||
)
|
||||
if artist_id is not None:
|
||||
# Use Post.artist_id (alembic 0030 denormalized column) instead
|
||||
# of joining through ImageProvenance.source_id → Source.artist_id.
|
||||
# The denormalization is the always-present linkage; the source
|
||||
# path now drops NULL-source provenance rows (filesystem-imported
|
||||
# content) which would otherwise vanish from artist-filtered
|
||||
# gallery views.
|
||||
# ALIAS Post: the gallery query outer-joins Post on
|
||||
# ImageRecord.primary_post_id (`_outer_join_primary_post`).
|
||||
# SQLAlchemy would otherwise correlate a bare `Post` reference
|
||||
# in this EXISTS subquery to that outer Post (which is NULL for
|
||||
# images with no primary post), and the filter would silently
|
||||
# match nothing.
|
||||
post_inner = aliased(Post)
|
||||
return exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.source_id == Source.id,
|
||||
Source.artist_id == artist_id,
|
||||
ImageProvenance.post_id == post_inner.id,
|
||||
post_inner.artist_id == artist_id,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
@@ -212,10 +212,10 @@ class Importer:
|
||||
which would lose the surrounding scan's progress — and re-run `stmt`
|
||||
(scalar_one) to return the row the other worker created.
|
||||
|
||||
Centralizes the pattern shared by _find_or_create_source,
|
||||
_source_for_sidecar, and _find_or_create_post. The plain
|
||||
SELECT-then-INSERT version lost races under the 5-min recovery sweep
|
||||
(operator-flagged 2026-05-26)."""
|
||||
Centralizes the pattern shared by _find_or_create_source and
|
||||
_find_or_create_post. The plain SELECT-then-INSERT version lost
|
||||
races under the 5-min recovery sweep (operator-flagged
|
||||
2026-05-26)."""
|
||||
existing = self.session.execute(stmt).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
return existing
|
||||
@@ -258,47 +258,25 @@ class Importer:
|
||||
lambda: Source(artist_id=artist_id, platform=platform, url=url),
|
||||
)
|
||||
|
||||
def _source_for_sidecar(
|
||||
self, *, artist_id: int, platform: str, artist_slug: str,
|
||||
) -> Source:
|
||||
"""Sidecar-import Source resolver. Used by both filesystem imports
|
||||
and gallery-dl downloads (both write sidecar JSON, both flow through
|
||||
_apply_sidecar / _capture_attachment).
|
||||
def _lookup_source_for_sidecar(
|
||||
self, *, artist_id: int, platform: str,
|
||||
) -> Source | None:
|
||||
"""Find the real subscription Source for (artist, platform), or
|
||||
None if no subscription exists.
|
||||
|
||||
Source represents a subscription feed (one per artist+platform — the
|
||||
URL polled by the FC-3 downloader). The filesystem importer used to
|
||||
call _find_or_create_source(url=sd.post_url), creating one Source
|
||||
row per post URL — 100s of junk Sources per artist, all with
|
||||
enabled=True, polluting the artist detail page and tricking the
|
||||
subscription checker into trying to poll patreon post URLs as feeds.
|
||||
Operator-flagged 2026-05-26; consolidated via alembic 0022.
|
||||
|
||||
Resolution order: prefer a real (non-sidecar) Source over a
|
||||
synthetic anchor. When alembic 0022 ran, it may have rewritten
|
||||
per-post Sources into `sidecar:<platform>:<slug>` synthetic
|
||||
anchors. If the operator later added the real subscription, both
|
||||
rows now coexist. A naive `ORDER BY id ASC LIMIT 1` lookup would
|
||||
pick the older synthetic and silently attach every gallery-dl
|
||||
download to the wrong Source — operator-flagged 2026-05-31 after
|
||||
the Subscriptions UI surfaced the phantom anchors. Pick the real
|
||||
one when one exists; fall back to the synthetic; only create a
|
||||
new synthetic when nothing exists for (artist, platform).
|
||||
Pre-alembic-0030 this method would CREATE a synthetic
|
||||
`sidecar:<platform>:<slug>` Source when no real one existed —
|
||||
because `Post.source_id` was NOT NULL and the importer needed
|
||||
something to attach Posts to. Alembic 0030 relaxed both
|
||||
`Post.source_id` and `ImageProvenance.source_id` to nullable, so
|
||||
synthetic anchors are obsolete; the importer now leaves
|
||||
source_id as None when no subscription exists for the (artist,
|
||||
platform). Operator-asked 2026-06-01: synthetic Sources had
|
||||
leaked into the Subscriptions UI as phantom subscriptions and
|
||||
the operator wanted the data model to truthfully say "this
|
||||
content has no live subscription."
|
||||
"""
|
||||
real_stmt = (
|
||||
select(Source)
|
||||
.where(
|
||||
Source.artist_id == artist_id,
|
||||
Source.platform == platform,
|
||||
~Source.url.like("sidecar:%"),
|
||||
)
|
||||
.order_by(Source.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
real = self.session.execute(real_stmt).scalar_one_or_none()
|
||||
if real is not None:
|
||||
return real
|
||||
|
||||
any_stmt = (
|
||||
stmt = (
|
||||
select(Source)
|
||||
.where(
|
||||
Source.artist_id == artist_id,
|
||||
@@ -307,29 +285,37 @@ class Importer:
|
||||
.order_by(Source.id.asc())
|
||||
.limit(1)
|
||||
)
|
||||
return self._get_or_create(
|
||||
any_stmt,
|
||||
lambda: Source(
|
||||
artist_id=artist_id,
|
||||
platform=platform,
|
||||
url=f"sidecar:{platform}:{artist_slug}",
|
||||
enabled=False,
|
||||
),
|
||||
)
|
||||
return self.session.execute(stmt).scalar_one_or_none()
|
||||
|
||||
def _find_or_create_post(
|
||||
self, *, source_id: int, external_post_id: str,
|
||||
self, *, source_id: int | None, external_post_id: str,
|
||||
artist_id: int,
|
||||
) -> Post:
|
||||
"""Race-safe find-or-create on `post` keyed by
|
||||
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
||||
— same savepoint + IntegrityError-recovery pattern."""
|
||||
stmt = select(Post).where(
|
||||
Post.source_id == source_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
"""Race-safe find-or-create on `post`. Keyed by
|
||||
(source_id, external_post_id) when source_id is set — the
|
||||
`uq_post_source_external_id` constraint guards. For NULL-source
|
||||
posts the existence check matches on (artist_id, external_post_id),
|
||||
which the partial unique index `uq_post_artist_external_id_null_source`
|
||||
(alembic 0030) guards. Same savepoint + IntegrityError-recovery
|
||||
pattern as the rest of the helpers."""
|
||||
if source_id is not None:
|
||||
stmt = select(Post).where(
|
||||
Post.source_id == source_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
else:
|
||||
stmt = select(Post).where(
|
||||
Post.source_id.is_(None),
|
||||
Post.artist_id == artist_id,
|
||||
Post.external_post_id == external_post_id,
|
||||
)
|
||||
return self._get_or_create(
|
||||
stmt,
|
||||
lambda: Post(source_id=source_id, external_post_id=external_post_id),
|
||||
lambda: Post(
|
||||
source_id=source_id,
|
||||
artist_id=artist_id,
|
||||
external_post_id=external_post_id,
|
||||
),
|
||||
)
|
||||
|
||||
def import_one(self, source: Path) -> ImportResult:
|
||||
@@ -370,12 +356,14 @@ class Importer:
|
||||
return None
|
||||
sd = parse_sidecar(data)
|
||||
platform = sd.platform or "unknown"
|
||||
src = self._source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform, artist_slug=artist.slug,
|
||||
src = self._lookup_source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
)
|
||||
epid = sd.external_post_id or sc.stem
|
||||
return self._find_or_create_post(
|
||||
source_id=src.id, external_post_id=epid,
|
||||
source_id=src.id if src else None,
|
||||
external_post_id=epid,
|
||||
artist_id=artist.id,
|
||||
)
|
||||
|
||||
def _capture_attachment(
|
||||
@@ -662,16 +650,34 @@ class Importer:
|
||||
them through. The sidecar JSON gallery-dl emits next to each
|
||||
downloaded file is read by `_apply_sidecar` via `find_sidecar`.
|
||||
|
||||
File-type dispatch parity with `import_one` (FC-2d-iii): zips,
|
||||
PDFs, audio etc. become PostAttachments; archives are extracted.
|
||||
Without this dispatch, gallery-dl-downloaded non-media bounced
|
||||
back as `skipped+invalid_image`, which DownloadService counted
|
||||
as an ingest error and flipped otherwise-successful runs to
|
||||
status="error". Operator-flagged 2026-06-02 after a Lustria
|
||||
patreon run with a 94MB OST zip went red despite 21 successful
|
||||
image attaches.
|
||||
|
||||
Caller's responsibilities after this returns:
|
||||
- duplicate_hash / duplicate_phash skip → delete the on-disk file
|
||||
- superseded → file stays where it is (now canonical)
|
||||
- imported → file stays where it is
|
||||
- attached → the file's been copied into the attachments store;
|
||||
caller may delete the on-disk original (mirrors duplicate_hash)
|
||||
- failed → file untouched; caller decides
|
||||
"""
|
||||
if not is_supported(path):
|
||||
if path.suffix.lower() == ".json":
|
||||
return ImportResult(
|
||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||
error=f"unsupported extension {path.suffix}",
|
||||
error="sidecar json is metadata, not content",
|
||||
)
|
||||
if is_archive(path):
|
||||
return self._import_archive(path)
|
||||
if not is_supported(path):
|
||||
post = self._post_for_sidecar(path, artist) if artist else None
|
||||
return self._capture_attachment(
|
||||
path, post=post, artist=artist, resolved=True,
|
||||
)
|
||||
|
||||
# Format / dimension / transparency filters (mirror _import_media).
|
||||
@@ -858,14 +864,15 @@ class Importer:
|
||||
src = explicit_source
|
||||
else:
|
||||
platform = sd.platform or "unknown"
|
||||
src = self._source_for_sidecar(
|
||||
src = self._lookup_source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
artist_slug=artist.slug,
|
||||
)
|
||||
|
||||
epid = sd.external_post_id or sc.stem
|
||||
post = self._find_or_create_post(
|
||||
source_id=src.id, external_post_id=epid,
|
||||
source_id=src.id if src else None,
|
||||
external_post_id=epid,
|
||||
artist_id=artist.id,
|
||||
)
|
||||
if sd.post_url is not None:
|
||||
post.post_url = sd.post_url
|
||||
@@ -901,7 +908,7 @@ class Importer:
|
||||
ImageProvenance(
|
||||
image_record_id=record.id,
|
||||
post_id=post.id,
|
||||
source_id=src.id,
|
||||
source_id=src.id if src else None,
|
||||
captured_metadata=sd.raw,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -48,11 +48,11 @@ class SuggestionService:
|
||||
).scalar_one()
|
||||
|
||||
def _threshold_for(self, s: MLSettings, category: str) -> float:
|
||||
# 'artist' intentionally absent (FC-2d-vii-c) — falls through to
|
||||
# the 1.01 "never surfaces" default like any unsurfaced category.
|
||||
# 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired;
|
||||
# both fall through to the 1.01 "never surfaces" default like any
|
||||
# unsurfaced category.
|
||||
return {
|
||||
"character": s.suggestion_threshold_character,
|
||||
"copyright": s.suggestion_threshold_copyright,
|
||||
"general": s.suggestion_threshold_general,
|
||||
}.get(category, 1.01)
|
||||
|
||||
|
||||
@@ -38,10 +38,13 @@ STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05"))
|
||||
|
||||
# The categories FC-2b surfaces in the UI. Others (meta/rating/year) are
|
||||
# still stored but the suggestion service filters them out.
|
||||
# FC-2d-vii-c: 'artist' retired — artist identity is acquisition-derived
|
||||
# (image_record.artist_id), never ML-inferred. Raw predictions are still
|
||||
# stored at STORE_FLOOR but artist never surfaces.
|
||||
SURFACED_CATEGORIES = {"character", "copyright", "general"}
|
||||
# 'artist' retired in FC-2d-vii-c — artist identity is acquisition-derived
|
||||
# (image_record.artist_id), never ML-inferred. 'copyright' retired
|
||||
# 2026-06-01 — operator doesn't use the copyright tag-kind; fandom is
|
||||
# this app's franchise/series concept (per TagsView.vue's doc comment).
|
||||
# Raw predictions for both categories still get stored at STORE_FLOOR but
|
||||
# don't surface in suggestions.
|
||||
SURFACED_CATEGORIES = {"character", "general"}
|
||||
|
||||
# ImageNet preprocessing constants (per Camie v2 onnx_inference.py).
|
||||
_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
|
||||
|
||||
@@ -96,10 +96,12 @@ def str_field(v) -> str | None:
|
||||
# Shared gallery-dl invocation defaults. Embedded in each platform's
|
||||
# default_config (with platform-specific overrides) so per-platform
|
||||
# choices stay explicit.
|
||||
# Note: the gallery-dl `skip` value (tick "exit:20" vs backfill True) is
|
||||
# NOT here — it's derived from Source.backfill_runs_remaining at download
|
||||
# time. See plan #544 / gallery_dl.TICK_SKIP_VALUE,BACKFILL_SKIP_VALUE.
|
||||
GD_DEFAULTS = {
|
||||
"sleep": 3.0,
|
||||
"sleep_request": 1.5,
|
||||
"skip_existing": True,
|
||||
"save_metadata": True,
|
||||
"timeout": 3600,
|
||||
}
|
||||
|
||||
@@ -69,13 +69,19 @@ class PostFeedService:
|
||||
raise ValueError("direction must be 'older' or 'newer'")
|
||||
|
||||
sort_key = _sort_key()
|
||||
# Artist via the denormalized Post.artist_id (alembic 0030);
|
||||
# Source via LEFT JOIN since post.source_id can now be NULL for
|
||||
# filesystem-imported posts with no live subscription. A
|
||||
# platform= filter implicitly excludes NULL-source posts (they
|
||||
# have no platform); an artist_id= filter still surfaces them
|
||||
# because Post.artist_id is always set.
|
||||
stmt = (
|
||||
select(Post, Artist, Source)
|
||||
.join(Source, Post.source_id == Source.id)
|
||||
.join(Artist, Source.artist_id == Artist.id)
|
||||
.join(Artist, Post.artist_id == Artist.id)
|
||||
.outerjoin(Source, Post.source_id == Source.id)
|
||||
)
|
||||
if artist_id is not None:
|
||||
stmt = stmt.where(Source.artist_id == artist_id)
|
||||
stmt = stmt.where(Post.artist_id == artist_id)
|
||||
if platform is not None:
|
||||
stmt = stmt.where(Source.platform == platform)
|
||||
if cursor:
|
||||
@@ -135,8 +141,8 @@ class PostFeedService:
|
||||
cursor for each end. Returns None if the post doesn't exist."""
|
||||
anchor = (await self.session.execute(
|
||||
select(Post, Artist, Source)
|
||||
.join(Source, Post.source_id == Source.id)
|
||||
.join(Artist, Source.artist_id == Artist.id)
|
||||
.join(Artist, Post.artist_id == Artist.id)
|
||||
.outerjoin(Source, Post.source_id == Source.id)
|
||||
.where(Post.id == post_id)
|
||||
)).one_or_none()
|
||||
if anchor is None:
|
||||
@@ -168,8 +174,8 @@ class PostFeedService:
|
||||
async def get_post(self, post_id: int) -> dict | None:
|
||||
row = (await self.session.execute(
|
||||
select(Post, Artist, Source)
|
||||
.join(Source, Post.source_id == Source.id)
|
||||
.join(Artist, Source.artist_id == Artist.id)
|
||||
.join(Artist, Post.artist_id == Artist.id)
|
||||
.outerjoin(Source, Post.source_id == Source.id)
|
||||
.where(Post.id == post_id)
|
||||
)).one_or_none()
|
||||
if row is None:
|
||||
@@ -260,7 +266,7 @@ class PostFeedService:
|
||||
return out
|
||||
|
||||
def _to_dict(
|
||||
self, post: Post, artist: Artist, source: Source,
|
||||
self, post: Post, artist: Artist, source: Source | None,
|
||||
thumbs_map: dict, atts_map: dict,
|
||||
) -> dict:
|
||||
plain_full = html_to_plain(post.description) if post.description else None
|
||||
@@ -269,6 +275,9 @@ class PostFeedService:
|
||||
else:
|
||||
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
||||
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
||||
# `source` is null for filesystem-imported posts with no live
|
||||
# subscription (alembic 0030). Frontend renders that as a
|
||||
# "filesystem import" affordance instead of a platform chip.
|
||||
return {
|
||||
"id": post.id,
|
||||
"external_post_id": post.external_post_id,
|
||||
@@ -279,7 +288,10 @@ class PostFeedService:
|
||||
"description_plain": description_plain,
|
||||
"description_truncated": truncated,
|
||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||
"source": {"id": source.id, "platform": source.platform},
|
||||
"source": (
|
||||
{"id": source.id, "platform": source.platform}
|
||||
if source is not None else None
|
||||
),
|
||||
"thumbnails": thumbs_entry["thumbs"],
|
||||
"thumbnails_more": thumbs_entry["more"],
|
||||
"attachments": atts_map.get(post.id, []),
|
||||
|
||||
@@ -70,11 +70,15 @@ class ProvenanceService:
|
||||
rec = await self.session.get(ImageRecord, image_id)
|
||||
if rec is None:
|
||||
return None
|
||||
# Artist via Post.artist_id (alembic 0030); Source via LEFT JOIN
|
||||
# since both Post.source_id and ImageProvenance.source_id can be
|
||||
# NULL for filesystem-imported content. Frontend renders source=
|
||||
# null as "filesystem import."
|
||||
stmt = (
|
||||
select(ImageProvenance, Post, Source, Artist)
|
||||
.join(Post, Post.id == ImageProvenance.post_id)
|
||||
.join(Source, Source.id == ImageProvenance.source_id)
|
||||
.join(Artist, Artist.id == Source.artist_id)
|
||||
.join(Artist, Artist.id == Post.artist_id)
|
||||
.outerjoin(Source, Source.id == ImageProvenance.source_id)
|
||||
.where(ImageProvenance.image_record_id == image_id)
|
||||
.order_by(ImageProvenance.captured_at.asc(),
|
||||
ImageProvenance.id.asc())
|
||||
@@ -90,7 +94,7 @@ class ProvenanceService:
|
||||
"captured_at": ip.captured_at.isoformat()
|
||||
if ip.captured_at else None,
|
||||
"post": _post_dict(post),
|
||||
"source": _source_dict(src),
|
||||
"source": _source_dict(src) if src is not None else None,
|
||||
"artist": _artist_dict(art),
|
||||
}
|
||||
for ip, post, src, art in rows
|
||||
@@ -99,10 +103,12 @@ class ProvenanceService:
|
||||
}
|
||||
|
||||
async def for_post(self, post_id: int) -> dict | None:
|
||||
# Same LEFT JOIN to Source — get_post must succeed for a
|
||||
# NULL-source post.
|
||||
stmt = (
|
||||
select(Post, Source, Artist)
|
||||
.join(Source, Source.id == Post.source_id)
|
||||
.join(Artist, Artist.id == Source.artist_id)
|
||||
.join(Artist, Artist.id == Post.artist_id)
|
||||
.outerjoin(Source, Source.id == Post.source_id)
|
||||
.where(Post.id == post_id)
|
||||
)
|
||||
row = (await self.session.execute(stmt)).first()
|
||||
@@ -111,7 +117,7 @@ class ProvenanceService:
|
||||
post, src, art = row
|
||||
return {
|
||||
"post": _post_dict(post),
|
||||
"source": _source_dict(src),
|
||||
"source": _source_dict(src) if src is not None else None,
|
||||
"artist": _artist_dict(art),
|
||||
"attachments": await self._attachments_for_posts([post.id]),
|
||||
}
|
||||
|
||||
@@ -86,7 +86,7 @@ def attempt_refetch(
|
||||
if src is None:
|
||||
return {"status": "no_source"}
|
||||
|
||||
# Remove the bad copy so gallery-dl (skip_existing) re-fetches it on
|
||||
# Remove the bad copy so gallery-dl's archive-skip re-fetches it on
|
||||
# the source re-check instead of skipping the still-present corrupt
|
||||
# file.
|
||||
try:
|
||||
|
||||
@@ -62,6 +62,7 @@ class SourceRecord:
|
||||
check_interval_override: int | None
|
||||
consecutive_failures: int
|
||||
next_check_at: str | None
|
||||
backfill_runs_remaining: int
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
@@ -78,6 +79,7 @@ class SourceRecord:
|
||||
"check_interval_override": self.check_interval_override,
|
||||
"consecutive_failures": self.consecutive_failures,
|
||||
"next_check_at": self.next_check_at,
|
||||
"backfill_runs_remaining": self.backfill_runs_remaining,
|
||||
}
|
||||
|
||||
|
||||
@@ -85,6 +87,11 @@ class SourceRecord:
|
||||
|
||||
_EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"}
|
||||
|
||||
# Plan #544 follow-up: newly created enabled sources pre-arm backfill so
|
||||
# their first N polls walk gallery-dl's full post history with the longer
|
||||
# timeout (matches the manual "Deep scan" button's default).
|
||||
NEW_SOURCE_BACKFILL_RUNS = 3
|
||||
|
||||
|
||||
class SourceService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
@@ -140,6 +147,7 @@ class SourceService:
|
||||
check_interval_override=source.check_interval_override,
|
||||
consecutive_failures=source.consecutive_failures or 0,
|
||||
next_check_at=nxt.isoformat() if nxt else None,
|
||||
backfill_runs_remaining=source.backfill_runs_remaining or 0,
|
||||
)
|
||||
|
||||
async def _row_to_record(self, source: Source) -> SourceRecord:
|
||||
@@ -157,7 +165,7 @@ class SourceService:
|
||||
if artist_id is not None:
|
||||
stmt = stmt.where(Source.artist_id == artist_id)
|
||||
if not include_synthetic:
|
||||
# Filesystem-import sidecar anchors (importer._source_for_sidecar)
|
||||
# Pre-alembic-0030 sidecar synthetic anchors
|
||||
# have url='sidecar:<platform>:<slug>' and exist only to give
|
||||
# imported Posts a NOT-NULL Source FK. They aren't pollable
|
||||
# feeds; the Subscriptions UI used to render them as phantom
|
||||
@@ -201,10 +209,21 @@ class SourceService:
|
||||
select(func.count(Source.id)).where(Source.artist_id == artist_id)
|
||||
)).scalar_one()
|
||||
|
||||
# Plan #544 follow-up: a freshly added subscription has no archive
|
||||
# yet, so the first few polls would walk the full post history in
|
||||
# tick mode and trip exit:20 after ~20 contiguous archive hits —
|
||||
# except there are none yet, so tick mode would walk forever and
|
||||
# blow the wall-clock cap. Pre-arm backfill so the initial syncs
|
||||
# use the longer timeout + skip:True walk. Tick mode resumes once
|
||||
# the budget is spent or the queue drains.
|
||||
# Disabled sources (incl. sidecar synthetics, url='sidecar:...')
|
||||
# are never polled, so leave their counter at 0.
|
||||
backfill_runs = NEW_SOURCE_BACKFILL_RUNS if enabled else 0
|
||||
source = Source(
|
||||
artist_id=artist_id, platform=platform, url=url,
|
||||
enabled=enabled, config_overrides=config_overrides,
|
||||
check_interval_override=check_interval_override,
|
||||
backfill_runs_remaining=backfill_runs,
|
||||
)
|
||||
self.session.add(source)
|
||||
try:
|
||||
@@ -264,6 +283,25 @@ class SourceService:
|
||||
await self.session.commit()
|
||||
return await self._row_to_record(source)
|
||||
|
||||
async def set_backfill_runs(
|
||||
self, source_id: int, runs: int,
|
||||
) -> SourceRecord:
|
||||
"""Plan #544: arm a source for backfill mode. The next `runs`
|
||||
download runs will use gallery-dl's full-walk config (skip: True
|
||||
+ 30-min timeout) instead of the catch-up default. Runs must be
|
||||
1..10 — bigger is rejected to keep the operator from accidentally
|
||||
setting a runaway budget."""
|
||||
if not isinstance(runs, int) or runs < 1 or runs > 10:
|
||||
raise ValueError("runs must be an integer in [1, 10]")
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
raise LookupError(f"source id={source_id} not found")
|
||||
source.backfill_runs_remaining = runs
|
||||
await self.session.commit()
|
||||
return await self._row_to_record(source)
|
||||
|
||||
async def delete(self, source_id: int) -> None:
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
|
||||
@@ -7,7 +7,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
from sqlalchemy import and_, delete, or_, select, update
|
||||
from sqlalchemy import Integer, and_, cast, delete, func, or_, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import (
|
||||
@@ -182,6 +182,11 @@ def recover_interrupted_tasks() -> int:
|
||||
.where(ImportTask.created_at < orphan_cutoff)
|
||||
.values(
|
||||
status="failed",
|
||||
# Without finished_at, cleanup_old_tasks (`WHERE
|
||||
# finished_at < cutoff`) never reaps these rows —
|
||||
# orphan-swept rows would become permanent table
|
||||
# tenants. Audit 2026-06-02.
|
||||
finished_at=now,
|
||||
error=(
|
||||
"orphan pending/queued swept by recover_interrupted_tasks "
|
||||
"(scanner likely crashed mid-enqueue); retry via "
|
||||
@@ -278,6 +283,14 @@ def recover_stalled_task_runs() -> int:
|
||||
f"no completion signal received within {minutes} min"
|
||||
),
|
||||
finished_at=now,
|
||||
# Matches celery_signals.finalize's
|
||||
# int((now - started_at).total_seconds() * 1000)
|
||||
# — sweep-closed rows now carry duration like
|
||||
# normally-finalized rows. Audit 2026-06-02.
|
||||
duration_ms=cast(
|
||||
func.extract("epoch", now - TaskRun.started_at) * 1000,
|
||||
Integer,
|
||||
),
|
||||
)
|
||||
)
|
||||
for w in extra_where:
|
||||
|
||||
@@ -20,14 +20,32 @@ IMAGES_ROOT = Path("/images")
|
||||
THUMB_MAGIC_JPEG = b"\xff\xd8\xff"
|
||||
THUMB_MAGIC_PNG = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
# Minimum file size for a thumbnail to count as valid. Anything smaller
|
||||
# is almost certainly truncated/corrupt — a legitimate 400×400 JPEG@85
|
||||
# bottoms out around 2KB even on a solid-color image; 400×400 PNG starts
|
||||
# around 1KB. 256 bytes is well below any real thumbnail and well above
|
||||
# header-only corrupt files (~8-12 bytes). Operator-flagged 2026-06-01:
|
||||
# header-only corrupt files were silently passing the magic-byte check
|
||||
# and backfill counted them as "ok" — so broken-image tiles in the UI
|
||||
# never got regenerated even after running backfill.
|
||||
MIN_THUMB_BYTES = 256
|
||||
|
||||
|
||||
def _thumb_is_valid(path: Path) -> bool:
|
||||
"""Return True iff `path` exists and starts with a JPEG or PNG magic header.
|
||||
"""Return True iff `path` exists, starts with a JPEG or PNG magic
|
||||
header, AND is at least MIN_THUMB_BYTES on disk.
|
||||
|
||||
The on-disk thumbnail format is set by services/thumbnailer.py — JPEG for
|
||||
opaque sources, PNG for alpha sources. Anything else (missing file, OSError,
|
||||
truncated, wrong magic) is invalid.
|
||||
The on-disk thumbnail format is set by services/thumbnailer.py — JPEG
|
||||
for opaque sources, PNG for alpha sources. Anything else (missing
|
||||
file, OSError, truncated below the size floor, wrong magic) is
|
||||
invalid and gets re-enqueued.
|
||||
"""
|
||||
try:
|
||||
size = path.stat().st_size
|
||||
except OSError:
|
||||
return False
|
||||
if size < MIN_THUMB_BYTES:
|
||||
return False
|
||||
try:
|
||||
with path.open("rb") as f:
|
||||
head = f.read(12)
|
||||
@@ -42,6 +60,59 @@ def _thumb_is_valid(path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _run_backfill_scan() -> dict:
|
||||
"""Synchronous scan logic shared by the Celery task and the API
|
||||
endpoint. Returns {enqueued, ok, regenerated, scanned}.
|
||||
|
||||
Operator-flagged 2026-06-01: the original task was fire-and-forget,
|
||||
so the admin UI couldn't show what backfill actually found —
|
||||
operator saw \"Enqueued.\" with no counts and assumed nothing was
|
||||
happening. Now the API runs this synchronously and returns the
|
||||
real numbers; the periodic Celery task wraps it too."""
|
||||
from sqlalchemy import select, update
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
enqueued = 0
|
||||
ok = 0
|
||||
regenerated = 0
|
||||
scanned = 0
|
||||
last_id = 0
|
||||
with SessionLocal() as session:
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(ImageRecord.id, ImageRecord.thumbnail_path)
|
||||
.where(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(500)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
scanned += len(rows)
|
||||
for image_id, thumb_path in rows:
|
||||
if thumb_path is None:
|
||||
generate_thumbnail.delay(image_id)
|
||||
enqueued += 1
|
||||
elif _thumb_is_valid(Path(thumb_path)):
|
||||
ok += 1
|
||||
else:
|
||||
session.execute(
|
||||
update(ImageRecord)
|
||||
.where(ImageRecord.id == image_id)
|
||||
.values(thumbnail_path=None)
|
||||
)
|
||||
generate_thumbnail.delay(image_id)
|
||||
enqueued += 1
|
||||
regenerated += 1
|
||||
session.commit()
|
||||
last_id = rows[-1][0]
|
||||
return {
|
||||
"scanned": scanned,
|
||||
"enqueued": enqueued,
|
||||
"ok": ok,
|
||||
"regenerated": regenerated,
|
||||
}
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
||||
bind=True,
|
||||
@@ -84,48 +155,15 @@ def backfill_thumbnails(self) -> dict:
|
||||
"""Scan ImageRecord and enqueue generate_thumbnail for rows whose
|
||||
thumbnail is missing, gone from disk, or has wrong magic bytes.
|
||||
|
||||
Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path for
|
||||
rows that point at a missing or corrupt file before enqueueing — keeps
|
||||
the DB self-consistent on partial runs and makes re-runs safe.
|
||||
Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path
|
||||
for rows that point at a missing or corrupt file before enqueueing —
|
||||
keeps the DB self-consistent on partial runs and makes re-runs safe.
|
||||
|
||||
Returns {"enqueued": N, "ok": M, "regenerated": K} where:
|
||||
- enqueued = total generate_thumbnail.delay() calls
|
||||
- ok = rows whose existing thumbnail file is valid (skipped)
|
||||
- regenerated = subset of enqueued that had a non-NULL thumbnail_path
|
||||
cleared (i.e. missing + corrupt)
|
||||
Returns {scanned, enqueued, ok, regenerated} where:
|
||||
- scanned = total rows examined
|
||||
- enqueued = total generate_thumbnail.delay() calls
|
||||
- ok = rows whose existing thumbnail file is valid (skipped)
|
||||
- regenerated = subset of enqueued that had a non-NULL
|
||||
thumbnail_path cleared (i.e. missing + corrupt)
|
||||
"""
|
||||
from sqlalchemy import select, update
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
enqueued = 0
|
||||
ok = 0
|
||||
regenerated = 0
|
||||
last_id = 0
|
||||
with SessionLocal() as session:
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(ImageRecord.id, ImageRecord.thumbnail_path)
|
||||
.where(ImageRecord.id > last_id)
|
||||
.order_by(ImageRecord.id.asc())
|
||||
.limit(500)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
for image_id, thumb_path in rows:
|
||||
if thumb_path is None:
|
||||
generate_thumbnail.delay(image_id)
|
||||
enqueued += 1
|
||||
elif _thumb_is_valid(Path(thumb_path)):
|
||||
ok += 1
|
||||
else:
|
||||
session.execute(
|
||||
update(ImageRecord)
|
||||
.where(ImageRecord.id == image_id)
|
||||
.values(thumbnail_path=None)
|
||||
)
|
||||
generate_thumbnail.delay(image_id)
|
||||
enqueued += 1
|
||||
regenerated += 1
|
||||
session.commit()
|
||||
last_id = rows[-1][0]
|
||||
return {"enqueued": enqueued, "ok": ok, "regenerated": regenerated}
|
||||
return _run_backfill_scan()
|
||||
|
||||
@@ -259,6 +259,29 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'PROBE_SOURCE':
|
||||
try {
|
||||
return await api.probeSource(msg.url);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'OPEN_ARTIST_PAGE': {
|
||||
// apiUrl is configured with the /api suffix (see
|
||||
// options/options.html placeholder); the SPA artist route is
|
||||
// /artist/:slug, served from the same origin. Strip /api so the
|
||||
// browser-level URL hits the Vue router, not the JSON API.
|
||||
const base = (api.baseUrl || '').replace(/\/+$/, '').replace(/\/api$/, '');
|
||||
const slug = encodeURIComponent(msg.slug || '');
|
||||
if (!base || !slug) return { error: 'apiUrl or slug missing' };
|
||||
try {
|
||||
await browser.tabs.create({ url: `${base}/artist/${slug}` });
|
||||
return { success: true };
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
default:
|
||||
return { error: `Unknown message type: ${msg.type}` };
|
||||
}
|
||||
|
||||
@@ -5,11 +5,26 @@
|
||||
background: rgb(20, 23, 26); color: rgb(244, 186, 122);
|
||||
font: 500 14px/1.2 system-ui, sans-serif;
|
||||
box-shadow: 0 4px 16px rgba(0, 0, 0, 0.4); cursor: pointer;
|
||||
transition: transform 100ms ease;
|
||||
transition: transform 100ms ease, background 150ms ease, color 150ms ease;
|
||||
}
|
||||
.fc-add-source-btn:hover { transform: translateY(-1px); }
|
||||
.fc-add-source-btn:disabled { opacity: 0.6; cursor: wait; }
|
||||
|
||||
/* state colors map to the FC palette: parchment-on-slate base,
|
||||
accent-orange for new, sage for already-subscribed, amber-warning for
|
||||
artist-exists-but-source-missing. All readable on the dark base. */
|
||||
.fc-add-source-btn--new {
|
||||
background: rgb(20, 23, 26); color: rgb(244, 186, 122);
|
||||
}
|
||||
.fc-add-source-btn--artist-match {
|
||||
background: rgb(28, 23, 16); color: rgb(255, 200, 120);
|
||||
border: 1px solid rgb(180, 130, 60);
|
||||
}
|
||||
.fc-add-source-btn--source-match {
|
||||
background: rgb(18, 28, 20); color: rgb(140, 220, 160);
|
||||
border: 1px solid rgb(80, 160, 100);
|
||||
}
|
||||
|
||||
.fc-toast {
|
||||
all: revert;
|
||||
position: fixed; bottom: 84px; right: 24px; z-index: 2147483647;
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
if (window.__fc_addsource_injected) return;
|
||||
window.__fc_addsource_injected = true;
|
||||
|
||||
// Cached probe result for the current URL so click-handlers know which
|
||||
// action to dispatch without round-tripping again.
|
||||
let currentProbe = null;
|
||||
|
||||
evaluate();
|
||||
|
||||
const reEval = () => evaluate();
|
||||
@@ -9,38 +13,116 @@
|
||||
const origPush = history.pushState;
|
||||
history.pushState = function () { origPush.apply(this, arguments); reEval(); };
|
||||
|
||||
function evaluate() {
|
||||
const platform = getPlatformFromUrl(window.location.href);
|
||||
const onArtist = platform && isArtistPage(window.location.href, platform);
|
||||
let btn = document.getElementById('fc-add-source-btn');
|
||||
if (onArtist && !btn) injectButton();
|
||||
else if (!onArtist && btn) btn.remove();
|
||||
async function evaluate() {
|
||||
const url = window.location.href;
|
||||
const platform = getPlatformFromUrl(url);
|
||||
const onArtist = platform && isArtistPage(url, platform);
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
if (!onArtist) {
|
||||
if (btn) btn.remove();
|
||||
currentProbe = null;
|
||||
return;
|
||||
}
|
||||
// On artist pages, ask the backend what state the URL is in BEFORE
|
||||
// injecting the button — so the chip can render the right state on
|
||||
// first paint instead of flashing the generic "Add" copy and
|
||||
// updating afterwards.
|
||||
let probe;
|
||||
try {
|
||||
probe = await browser.runtime.sendMessage({ type: 'PROBE_SOURCE', url });
|
||||
} catch (e) {
|
||||
probe = { error: e?.message || 'probe failed' };
|
||||
}
|
||||
currentProbe = probe;
|
||||
if (probe?.state === 'unknown_platform') {
|
||||
if (btn) btn.remove();
|
||||
return;
|
||||
}
|
||||
renderButton(probe);
|
||||
}
|
||||
|
||||
function injectButton() {
|
||||
const btn = document.createElement('button');
|
||||
btn.id = 'fc-add-source-btn';
|
||||
function renderButton(probe) {
|
||||
let btn = document.getElementById('fc-add-source-btn');
|
||||
if (!btn) {
|
||||
btn = document.createElement('button');
|
||||
btn.id = 'fc-add-source-btn';
|
||||
btn.addEventListener('click', onClick);
|
||||
document.body.appendChild(btn);
|
||||
}
|
||||
// Reset state classes so re-renders (SPA navigation) don't stack.
|
||||
btn.className = 'fc-add-source-btn';
|
||||
btn.textContent = '+ Add to FabledCurator';
|
||||
btn.addEventListener('click', onClick);
|
||||
document.body.appendChild(btn);
|
||||
btn.classList.add(`fc-add-source-btn--${stateModifier(probe)}`);
|
||||
btn.textContent = labelFor(probe);
|
||||
btn.disabled = false;
|
||||
}
|
||||
|
||||
function stateModifier(probe) {
|
||||
if (!probe || probe.error) return 'new';
|
||||
return ({
|
||||
source_match: 'source-match',
|
||||
artist_match: 'artist-match',
|
||||
new: 'new',
|
||||
})[probe.state] || 'new';
|
||||
}
|
||||
|
||||
function labelFor(probe) {
|
||||
if (!probe || probe.error) return '+ Add to FabledCurator';
|
||||
const platformName = platformDisplayName(probe.platform);
|
||||
const artistName = probe.artist?.name;
|
||||
switch (probe.state) {
|
||||
case 'source_match':
|
||||
return `✓ In FabledCurator · ${platformName}`;
|
||||
case 'artist_match':
|
||||
return `+ Add ${platformName} source to ${artistName || 'artist'}`;
|
||||
case 'new':
|
||||
default:
|
||||
return '+ Add to FabledCurator';
|
||||
}
|
||||
}
|
||||
|
||||
function platformDisplayName(key) {
|
||||
return PLATFORMS[key]?.name || key || '';
|
||||
}
|
||||
|
||||
async function onClick() {
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
if (!btn) return;
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
const probe = currentProbe;
|
||||
|
||||
if (probe?.state === 'source_match') {
|
||||
btn.textContent = 'Opening…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'OPEN_ARTIST_PAGE',
|
||||
slug: probe.artist?.slug,
|
||||
});
|
||||
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
btn.textContent = 'Adding…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'ADD_AS_SOURCE',
|
||||
url: window.location.href,
|
||||
});
|
||||
if (r.error) {
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
} else {
|
||||
const verb = r.created_source ? 'Added' : 'Already a source for';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
||||
// Re-probe so the chip flips green without waiting for the next
|
||||
// navigation.
|
||||
evaluate();
|
||||
return;
|
||||
}
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
|
||||
@@ -83,6 +83,12 @@ class FabledCuratorAPI {
|
||||
quickAddSource(url) {
|
||||
return this.request('POST', '/extension/quick-add-source', { url });
|
||||
}
|
||||
probeSource(url) {
|
||||
// Read-only existence check. Drives the content-script chip's
|
||||
// color/copy BEFORE the operator clicks Add.
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
return this.request('GET', `/extension/probe?${qs}`);
|
||||
}
|
||||
|
||||
// Connection test = the cheapest read with auth.
|
||||
testConnection() {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "FabledCurator",
|
||||
"version": "1.0.5",
|
||||
"version": "1.0.6",
|
||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||
|
||||
"browser_specific_settings": {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fabledcurator-extension",
|
||||
"version": "1.0.5",
|
||||
"version": "1.0.6",
|
||||
"private": true,
|
||||
"description": "Firefox extension for FabledCurator",
|
||||
"scripts": {
|
||||
|
||||
@@ -48,10 +48,11 @@ function onSearch(q) {
|
||||
if (!query) { results.value = []; return }
|
||||
loading.value = true
|
||||
try {
|
||||
// Scope the autocomplete to the prediction's category where it maps
|
||||
// to a tag kind. 'copyright' has no tag kind; search unscoped there.
|
||||
const kind = ['artist', 'character'].includes(props.category)
|
||||
? props.category : null
|
||||
// Scope the autocomplete to the prediction's category where it
|
||||
// maps to a tag kind. Only 'character' surfaces as both a
|
||||
// suggestion category and a tag kind now ('artist' + 'copyright'
|
||||
// retired); other categories search unscoped.
|
||||
const kind = props.category === 'character' ? 'character' : null
|
||||
const params = { q: query, limit: 20 }
|
||||
if (kind) params.kind = kind
|
||||
results.value = await api.get('/api/tags/autocomplete', { params })
|
||||
|
||||
@@ -92,7 +92,16 @@ let prevBodyOverflow = null
|
||||
// own keystrokes.
|
||||
function onKeyDown(ev) {
|
||||
if (ev.key === 'Escape') {
|
||||
if (isTextEntry(ev.target)) return
|
||||
// Escape closes the modal even from inside a text input — that's
|
||||
// the universal "get me out of here" expectation, and the
|
||||
// autofocused tag-entry field would otherwise trap focus with no
|
||||
// visible escape (operator-flagged 2026-06-01). EXCEPTION: when a
|
||||
// nested Vuetify overlay is open (v-menu autocomplete dropdown,
|
||||
// FandomPicker v-dialog, per-suggestion 3-dot menu), let that
|
||||
// overlay's own Esc handling fire instead of closing the whole
|
||||
// modal mid-interaction. Vuetify marks open overlays with
|
||||
// `.v-overlay--active`.
|
||||
if (document.querySelector('.v-overlay--active')) return
|
||||
ev.preventDefault()
|
||||
emit('close')
|
||||
} else if (ev.key === 'ArrowLeft') {
|
||||
|
||||
@@ -10,17 +10,30 @@
|
||||
density="compact"
|
||||
>{{ state.error }}</v-alert>
|
||||
|
||||
<template v-else>
|
||||
<!-- Cards scroll independently of the section title + attachments
|
||||
below them. Cap at ~2.5 cards visible (operator-asked 2026-06-01:
|
||||
keeps the Tags section anchored below at a consistent position;
|
||||
the half-visible third card hints there's more). -->
|
||||
<div v-else class="fc-prov__cards">
|
||||
<article
|
||||
v-for="e in state.entries" :key="e.provenance_id" class="fc-prov__card"
|
||||
>
|
||||
<div class="fc-prov__head">
|
||||
<span class="fc-prov__platform">{{ e.source.platform }}</span>
|
||||
<!-- Posts with no live subscription have source=null (alembic
|
||||
0030); render an explicit "filesystem import" affordance
|
||||
instead of a platform chip. -->
|
||||
<span class="fc-prov__platform">
|
||||
{{ e.source?.platform ?? 'filesystem import' }}
|
||||
</span>
|
||||
<span v-if="postDate(e)" class="fc-prov__date">{{ postDate(e) }}</span>
|
||||
</div>
|
||||
<div class="fc-prov__post">
|
||||
<button
|
||||
type="button" class="fc-prov__post"
|
||||
:title="`Open ${postTitle(e)} in the posts feed for ${e.artist.name}`"
|
||||
@click="openPost(e.post.id, e.artist.id)"
|
||||
>
|
||||
{{ postTitle(e) }}
|
||||
</div>
|
||||
</button>
|
||||
<div class="fc-prov__meta">
|
||||
<RouterLink :to="`/artist/${e.artist.slug}`">
|
||||
by {{ e.artist.name }}
|
||||
@@ -29,12 +42,8 @@
|
||||
· {{ e.post.attachment_count }} files
|
||||
</span>
|
||||
</div>
|
||||
<div class="fc-prov__actions">
|
||||
<a href="#" @click.prevent="openPost(e.post.id)">
|
||||
View post
|
||||
</a>
|
||||
<div v-if="e.post.description_html" class="fc-prov__actions">
|
||||
<a
|
||||
v-if="e.post.description_html"
|
||||
href="#" @click.prevent="toggleDesc(e.provenance_id)"
|
||||
>{{ expanded[e.provenance_id] ? 'Hide description ▴' : 'Show description ▾' }}</a>
|
||||
</div>
|
||||
@@ -58,7 +67,7 @@
|
||||
</RouterLink>
|
||||
</div>
|
||||
</article>
|
||||
</template>
|
||||
</div>
|
||||
|
||||
<div v-if="attachments.length" class="fc-prov__attach">
|
||||
<h4 class="fc-prov__attach-title">Attachments</h4>
|
||||
@@ -137,10 +146,16 @@ function postTitle(e) {
|
||||
return toPlainText(e.post.title) || `Post ${e.post.external_post_id}`
|
||||
}
|
||||
|
||||
function openPost(postId) {
|
||||
function openPost(postId, artistId) {
|
||||
// Land on the post in the posts feed (in context), not the gallery
|
||||
// image grid. Operator-flagged 2026-05-28.
|
||||
router.push({ path: '/posts', query: { post_id: postId } })
|
||||
// image grid. Scope the feed to this artist so the user lands in
|
||||
// that creator's stream, not the global one — operator-flagged
|
||||
// 2026-06-01. PostsView reads `artist_id` from the query string
|
||||
// (PostsView.vue line ~92) and filters via post_feed_service.
|
||||
router.push({
|
||||
path: '/posts',
|
||||
query: { post_id: postId, artist_id: artistId },
|
||||
})
|
||||
modal.close()
|
||||
}
|
||||
</script>
|
||||
@@ -153,6 +168,18 @@ function openPost(postId) {
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
margin-bottom: 12px;
|
||||
}
|
||||
.fc-prov__cards {
|
||||
/* 2.5 cards-worth at the typical collapsed card height (~108px each
|
||||
incl. 10px gap). Slightly under to ensure the third card's bottom
|
||||
edge is clipped — the visual cue that there's more below. */
|
||||
max-height: 270px;
|
||||
overflow-y: auto;
|
||||
/* Hairline scrollbar that doesn't compete with content. */
|
||||
scrollbar-width: thin;
|
||||
scrollbar-color: rgb(var(--v-theme-surface-light)) transparent;
|
||||
/* Pad-right so the scrollbar gutter doesn't squeeze card borders. */
|
||||
padding-right: 4px;
|
||||
}
|
||||
.fc-prov__card {
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
border-radius: 6px; padding: 10px 12px; margin-bottom: 10px;
|
||||
@@ -163,8 +190,21 @@ function openPost(postId) {
|
||||
text-transform: lowercase;
|
||||
}
|
||||
.fc-prov__post {
|
||||
font-weight: 700; margin: 4px 0;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
/* Clickable title — opens the post in the artist-scoped feed
|
||||
(operator-flagged 2026-06-01: title IS the primary action, the
|
||||
prior "View post" link was redundant). Styled as a button-link:
|
||||
accent color, underline on hover, focus ring for keyboard nav. */
|
||||
display: block; width: 100%; text-align: left;
|
||||
background: none; border: none; padding: 0;
|
||||
font: inherit; font-weight: 700;
|
||||
margin: 4px 0;
|
||||
color: rgb(var(--v-theme-accent));
|
||||
cursor: pointer;
|
||||
}
|
||||
.fc-prov__post:hover { text-decoration: underline; }
|
||||
.fc-prov__post:focus-visible {
|
||||
outline: 2px solid rgb(var(--v-theme-accent));
|
||||
outline-offset: 2px; border-radius: 3px;
|
||||
}
|
||||
.fc-prov__meta {
|
||||
font-size: 13px; color: rgb(var(--v-theme-on-surface-variant));
|
||||
|
||||
@@ -1,19 +1,33 @@
|
||||
<template>
|
||||
<!-- Chip-card row: visible border + hover/focus state unifies the
|
||||
name, score, and action buttons as one "object" (operator-asked
|
||||
2026-06-01). The row itself is informational; the explicit
|
||||
Accept button + 3-dot menu are the action affordances. -->
|
||||
<div class="fc-suggestion">
|
||||
<span class="fc-suggestion__name">
|
||||
{{ suggestion.display_name }}
|
||||
<span v-if="suggestion.creates_new_tag" class="fc-suggestion__new"
|
||||
title="No matching tag yet — accepting creates it">+new</span>
|
||||
title="No matching tag yet — accepting creates it">+ new</span>
|
||||
</span>
|
||||
<span class="fc-suggestion__score">{{ scorePct }}</span>
|
||||
<v-btn
|
||||
icon="mdi-plus" size="x-small" variant="text" color="accent"
|
||||
class="fc-suggestion__accept"
|
||||
size="small" variant="tonal" color="accent"
|
||||
density="compact" rounded="pill"
|
||||
:aria-label="`Accept ${suggestion.display_name}`"
|
||||
@click="$emit('accept', suggestion)"
|
||||
/>
|
||||
>
|
||||
Accept
|
||||
</v-btn>
|
||||
<v-menu>
|
||||
<template #activator="{ props }">
|
||||
<v-btn icon="mdi-dots-vertical" size="x-small" variant="text" v-bind="props" />
|
||||
<v-btn
|
||||
class="fc-suggestion__menu"
|
||||
icon="mdi-dots-vertical" size="small"
|
||||
variant="outlined" density="compact"
|
||||
:aria-label="`More actions for ${suggestion.display_name}`"
|
||||
v-bind="props"
|
||||
/>
|
||||
</template>
|
||||
<v-list density="compact">
|
||||
<v-list-item @click="$emit('alias', suggestion)">
|
||||
@@ -38,17 +52,45 @@ const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
||||
|
||||
<style scoped>
|
||||
.fc-suggestion {
|
||||
display: flex; align-items: center; gap: 6px;
|
||||
padding: 2px 0;
|
||||
display: flex; align-items: center; gap: 8px;
|
||||
padding: 6px 10px; margin-bottom: 4px;
|
||||
background: rgb(var(--v-theme-surface));
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
border-radius: 6px;
|
||||
transition: background 120ms ease, border-color 120ms ease;
|
||||
}
|
||||
.fc-suggestion:hover {
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
border-color: rgb(var(--v-theme-accent), 0.4);
|
||||
}
|
||||
.fc-suggestion__name {
|
||||
flex: 1; min-width: 0;
|
||||
font-size: 14px;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-suggestion__name { flex: 1; min-width: 0; }
|
||||
.fc-suggestion__new {
|
||||
font-size: 10px; color: rgb(var(--v-theme-accent));
|
||||
margin-left: 4px;
|
||||
display: inline-block;
|
||||
font-size: 10px; font-weight: 600;
|
||||
color: rgb(var(--v-theme-accent));
|
||||
background: rgba(var(--v-theme-accent), 0.12);
|
||||
border: 1px solid rgb(var(--v-theme-accent), 0.4);
|
||||
padding: 1px 6px; border-radius: 999px;
|
||||
margin-left: 6px;
|
||||
text-transform: uppercase; letter-spacing: 0.04em;
|
||||
}
|
||||
.fc-suggestion__score {
|
||||
flex: 0 0 auto; min-width: 38px; text-align: right;
|
||||
font-size: 11px;
|
||||
color: rgb(var(--v-theme-on-surface-variant, var(--v-theme-on-surface)));
|
||||
font-family: 'JetBrains Mono', monospace;
|
||||
}
|
||||
/* Vuetify's compact density doesn't shrink the tonal button enough
|
||||
for a tight row; clamp the min-width so Accept stays compact. */
|
||||
.fc-suggestion__accept :deep(.v-btn__content) {
|
||||
font-size: 12px; letter-spacing: 0.02em;
|
||||
}
|
||||
.fc-suggestion__menu {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
<SuggestionsCategoryGroup
|
||||
v-if="store.byCategory.general && store.byCategory.general.length"
|
||||
label="General" :items="store.byCategory.general"
|
||||
collapsible :default-open="false"
|
||||
collapsible :default-open="true"
|
||||
@accept="onAccept" @alias="onAlias" @dismiss="store.dismiss"
|
||||
/>
|
||||
</template>
|
||||
@@ -41,13 +41,18 @@
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import { useSuggestionsStore, CATEGORY_LABELS } from '../../stores/suggestions.js'
|
||||
import { useModalStore } from '../../stores/modal.js'
|
||||
import SuggestionsCategoryGroup from './SuggestionsCategoryGroup.vue'
|
||||
import AliasPickerDialog from './AliasPickerDialog.vue'
|
||||
|
||||
const props = defineProps({ imageId: { type: Number, required: true } })
|
||||
const store = useSuggestionsStore()
|
||||
const modal = useModalStore()
|
||||
|
||||
const peopleCats = ['artist', 'character', 'copyright']
|
||||
// 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired as
|
||||
// suggestion categories. Only 'character' remains as a people-style
|
||||
// category alongside the general bucket.
|
||||
const peopleCats = ['character']
|
||||
function labelFor(c) { return CATEGORY_LABELS[c] || c }
|
||||
|
||||
const isEmpty = computed(() =>
|
||||
@@ -56,9 +61,20 @@ const isEmpty = computed(() =>
|
||||
|
||||
watch(() => props.imageId, (id) => { if (id != null) store.load(id) }, { immediate: true })
|
||||
|
||||
// After a successful accept/alias-accept, refresh the modal's current
|
||||
// tag list so TagPanel's chip rail reflects the newly-attached tag.
|
||||
// Operator-flagged 2026-06-01: the suggestion store dropped the
|
||||
// suggestion (correct) but didn't propagate the new tag back into the
|
||||
// modal store, so the chip rail looked unchanged even though the
|
||||
// backend had recorded the application. Mirrors the addExistingTag /
|
||||
// createAndAdd flows which already call reloadTags() after applying.
|
||||
async function onAccept(s) {
|
||||
try { await store.accept(s) }
|
||||
catch (e) { toast({ text: `Accept failed: ${e.message}`, type: 'error' }) }
|
||||
try {
|
||||
await store.accept(s)
|
||||
await modal.reloadTags()
|
||||
} catch (e) {
|
||||
toast({ text: `Accept failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
|
||||
const aliasDialog = ref(false)
|
||||
@@ -68,6 +84,7 @@ async function onAliasConfirm(canonicalTagId) {
|
||||
try {
|
||||
await store.aliasAccept(aliasTarget.value, canonicalTagId)
|
||||
aliasDialog.value = false
|
||||
await modal.reloadTags()
|
||||
} catch (e) {
|
||||
toast({ text: `Alias failed: ${e.message}`, type: 'error' })
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<template>
|
||||
<div class="fc-tag-autocomplete">
|
||||
<v-text-field
|
||||
ref="inputRef"
|
||||
v-model="query"
|
||||
placeholder="Add tag (or kind:name — character/fandom/series)"
|
||||
density="compact" hide-details
|
||||
@@ -52,13 +53,21 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, ref, watch } from 'vue'
|
||||
import { computed, nextTick, onMounted, ref, watch } from 'vue'
|
||||
import { useTagStore } from '../../stores/tags.js'
|
||||
import FandomPicker from './FandomPicker.vue'
|
||||
|
||||
const emit = defineEmits(['pick-existing', 'pick-new', 'cancel'])
|
||||
const store = useTagStore()
|
||||
|
||||
// Autofocus on modal open so the operator can type the moment the view
|
||||
// modal renders, no extra click required (operator-asked 2026-06-01).
|
||||
// Vuetify's v-text-field exposes .focus() on the component instance;
|
||||
// nextTick waits for the modal's mount to finish so the inner <input>
|
||||
// element exists.
|
||||
const inputRef = ref(null)
|
||||
onMounted(() => { nextTick(() => inputRef.value?.focus?.()) })
|
||||
|
||||
// Single text input; no kind dropdown. Client-side mirror of the
|
||||
// backend's parse_kind_prefix lives below — kept in sync with
|
||||
// KNOWN_KINDS in backend/app/utils/tag_prefix.py. The backend is the
|
||||
|
||||
@@ -7,7 +7,12 @@
|
||||
@keydown.enter="onCardClick"
|
||||
>
|
||||
<div class="fc-post-card__head">
|
||||
<v-chip size="x-small" variant="tonal">{{ post.source.platform }}</v-chip>
|
||||
<!-- Posts with no live subscription have source=null (alembic
|
||||
0030); show a "filesystem import" affordance instead of a
|
||||
platform chip. -->
|
||||
<v-chip size="x-small" variant="tonal">
|
||||
{{ post.source?.platform ?? 'filesystem import' }}
|
||||
</v-chip>
|
||||
<RouterLink
|
||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||
class="fc-post-card__artist"
|
||||
@@ -25,7 +30,7 @@
|
||||
v-if="post.post_url"
|
||||
:href="post.post_url" target="_blank" rel="noopener"
|
||||
icon="mdi-open-in-new" size="x-small" variant="text"
|
||||
:aria-label="`open original post on ${post.source.platform}`"
|
||||
:aria-label="`open original post on ${post.source?.platform ?? 'web'}`"
|
||||
@click.stop
|
||||
/>
|
||||
<v-btn
|
||||
|
||||
@@ -22,10 +22,10 @@ import { reactive, watch } from 'vue'
|
||||
import { useMLStore } from '../../stores/ml.js'
|
||||
|
||||
const store = useMLStore()
|
||||
// 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired as
|
||||
// suggestion categories; their threshold rows are gone.
|
||||
const fields = [
|
||||
{ key: 'suggestion_threshold_artist', label: 'Artist' },
|
||||
{ key: 'suggestion_threshold_character', label: 'Character' },
|
||||
{ key: 'suggestion_threshold_copyright', label: 'Copyright' },
|
||||
{ key: 'suggestion_threshold_general', label: 'General' },
|
||||
{ key: 'centroid_similarity_threshold', label: 'Centroid similarity' }
|
||||
]
|
||||
|
||||
@@ -10,7 +10,14 @@
|
||||
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||
<v-icon start>mdi-image-refresh</v-icon> Run backfill now
|
||||
</v-btn>
|
||||
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
||||
<span v-if="result" class="ml-3 text-caption">
|
||||
Scanned <strong>{{ result.scanned }}</strong> · enqueued
|
||||
<strong>{{ result.enqueued }}</strong>
|
||||
<span v-if="result.regenerated > 0">
|
||||
({{ result.regenerated }} regenerated)
|
||||
</span>
|
||||
· {{ result.ok }} ok
|
||||
</span>
|
||||
<QueueStatusBar queue="thumbnail" queue-label="Thumbnail" />
|
||||
</v-card-text>
|
||||
</v-card>
|
||||
@@ -23,10 +30,11 @@ import { useThumbnailsStore } from '../../stores/thumbnails.js'
|
||||
import QueueStatusBar from './QueueStatusBar.vue'
|
||||
const store = useThumbnailsStore()
|
||||
const busy = ref(false)
|
||||
const done = ref(false)
|
||||
const result = ref(null)
|
||||
async function run () {
|
||||
busy.value = true
|
||||
try { await store.triggerBackfill(); done.value = true }
|
||||
result.value = null
|
||||
try { result.value = await store.triggerBackfill() }
|
||||
catch (e) { toast({ text: e.message, type: 'error' }) }
|
||||
finally { busy.value = false }
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
:retrying-all="retryingAll"
|
||||
@retry="onRetrySource"
|
||||
@retry-all="onRetryAll"
|
||||
@view-logs="onViewFailingLogs"
|
||||
/>
|
||||
|
||||
<div v-if="store.loading && store.events.length === 0" class="fc-dl__loading">
|
||||
@@ -364,6 +365,23 @@ watch(filterModel, async (m) => {
|
||||
async function openDetail(id) {
|
||||
await store.loadOne(id)
|
||||
}
|
||||
|
||||
async function onViewFailingLogs(source) {
|
||||
// Find and open the most recent DownloadEvent for this source.
|
||||
// Reuses the existing DownloadDetailModal — same stdout/stderr/error
|
||||
// surface the row-click in the events feed shows.
|
||||
try {
|
||||
const ev = await store.loadLastForSource(source.id)
|
||||
if (!ev) {
|
||||
toast({
|
||||
text: `No download events recorded for ${source.artist_name || source.platform} yet.`,
|
||||
type: 'warning',
|
||||
})
|
||||
}
|
||||
} catch (e) {
|
||||
toast({ text: `Failed to load logs: ${e.message}`, type: 'error' })
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -29,6 +29,14 @@
|
||||
{{ s.last_error || 'no error message recorded' }}
|
||||
</span>
|
||||
<v-spacer />
|
||||
<v-btn
|
||||
size="x-small" variant="text" prepend-icon="mdi-text-box-search-outline"
|
||||
:loading="logLoadingIds.has(s.id)"
|
||||
@click="onViewLogs(s)"
|
||||
title="Show the most recent download event's stdout/stderr/error"
|
||||
>
|
||||
Logs
|
||||
</v-btn>
|
||||
<v-btn
|
||||
size="x-small" variant="text" prepend-icon="mdi-refresh"
|
||||
:loading="retryingIds.has(s.id)"
|
||||
@@ -52,9 +60,23 @@ defineProps({
|
||||
retryingIds: { type: Set, default: () => new Set() },
|
||||
retryingAll: { type: Boolean, default: false },
|
||||
})
|
||||
defineEmits(['retry', 'retry-all'])
|
||||
const emit = defineEmits(['retry', 'retry-all', 'view-logs'])
|
||||
|
||||
const open = ref(true)
|
||||
// Per-row loading flag so the spinner lives on the row whose Logs
|
||||
// button was clicked, not on every row.
|
||||
const logLoadingIds = ref(new Set())
|
||||
async function onViewLogs(s) {
|
||||
if (logLoadingIds.value.has(s.id)) return
|
||||
logLoadingIds.value = new Set(logLoadingIds.value).add(s.id)
|
||||
try {
|
||||
await emit('view-logs', s)
|
||||
} finally {
|
||||
const next = new Set(logLoadingIds.value)
|
||||
next.delete(s.id)
|
||||
logLoadingIds.value = next
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
@@ -68,13 +90,28 @@ const open = ref(true)
|
||||
.fc-fail__title { font-weight: 600; }
|
||||
.fc-fail__body {
|
||||
padding: 0 14px 10px;
|
||||
display: flex; flex-direction: column; gap: 2px;
|
||||
display: flex; flex-direction: column;
|
||||
/* Borders, not gap, so the row separators are visible inside the
|
||||
* tonal error card (where surface tints get washed out and gap is
|
||||
* just empty space). */
|
||||
}
|
||||
.fc-fail__row {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
padding: 6px 8px;
|
||||
padding: 8px 10px;
|
||||
border-radius: 4px;
|
||||
background: rgb(var(--v-theme-surface) / 0.4);
|
||||
/* Hover-darken gives the eye a horizontal track from the artist
|
||||
* name on the left to the Logs/Retry buttons on the right.
|
||||
* Bottom border replaces the previous-too-subtle zebra striping —
|
||||
* inside the tonal error card, surface-tint contrast was negligible.
|
||||
* Operator-flagged 2026-06-01 (twice). */
|
||||
border-bottom: 1px solid rgb(255 255 255 / 0.08);
|
||||
transition: background 80ms ease;
|
||||
}
|
||||
.fc-fail__row:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
.fc-fail__row:hover {
|
||||
background: rgb(0 0 0 / 0.25);
|
||||
}
|
||||
.fc-fail__artist { font-weight: 600; white-space: nowrap; }
|
||||
.fc-fail__count { flex: 0 0 auto; }
|
||||
|
||||
@@ -31,6 +31,12 @@
|
||||
v-if="(source.consecutive_failures || 0) > 0"
|
||||
size="x-small" color="error" variant="tonal" label
|
||||
>{{ source.consecutive_failures }}</v-chip>
|
||||
<v-chip
|
||||
v-else-if="(source.backfill_runs_remaining || 0) > 0"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
>
|
||||
backfill ({{ source.backfill_runs_remaining }}×)
|
||||
</v-chip>
|
||||
<span v-else class="fc-source-row__zero">0</span>
|
||||
</td>
|
||||
<td class="fc-source-row__actions">
|
||||
@@ -42,6 +48,16 @@
|
||||
<v-icon>mdi-play</v-icon>
|
||||
<v-tooltip activator="parent" location="top">Check now</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon="mdi-magnify-scan" size="x-small" variant="text"
|
||||
:disabled="(source.backfill_runs_remaining || 0) > 0"
|
||||
@click.stop="$emit('backfill', source)"
|
||||
>
|
||||
<v-icon>mdi-magnify-scan</v-icon>
|
||||
<v-tooltip activator="parent" location="top">
|
||||
Deep scan — walk full history for next few runs
|
||||
</v-tooltip>
|
||||
</v-btn>
|
||||
<v-btn
|
||||
icon="mdi-pencil" size="x-small" variant="text"
|
||||
@click.stop="$emit('edit', source)"
|
||||
@@ -69,7 +85,7 @@ const props = defineProps({
|
||||
checking: { type: Boolean, default: false },
|
||||
warningThreshold: { type: Number, default: 5 },
|
||||
})
|
||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check'])
|
||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill'])
|
||||
|
||||
function onToggleEnabled(value) {
|
||||
emit('toggle', { source: props.source, enabled: value })
|
||||
|
||||
@@ -174,6 +174,7 @@
|
||||
@remove="removeSource"
|
||||
@toggle="toggleSourceEnabled"
|
||||
@check="onCheck"
|
||||
@backfill="onBackfill"
|
||||
/>
|
||||
<tr v-if="item.sources.length === 0">
|
||||
<td colspan="8" class="fc-subs__sources-empty">
|
||||
@@ -431,6 +432,36 @@ async function onCheck(source) {
|
||||
}
|
||||
}
|
||||
|
||||
// Plan #544: arm a source for backfill mode (gallery-dl walks the full
|
||||
// post history) for the next N download runs. Default 3 — enough budget
|
||||
// to finish a deep creator without re-prompting the operator across
|
||||
// timeout boundaries. The chip on the row reflects the remaining count.
|
||||
async function onBackfill(source) {
|
||||
const raw = globalThis.window?.prompt(
|
||||
`Deep scan "${source.artist_name} (${source.platform})" — walk full history for the next how many download runs? (1–10, default 3)`,
|
||||
'3',
|
||||
)
|
||||
if (raw == null) return
|
||||
const runs = parseInt(raw, 10)
|
||||
if (!Number.isFinite(runs) || runs < 1 || runs > 10) {
|
||||
toast({ text: 'Deep scan: runs must be 1–10', type: 'error' })
|
||||
return
|
||||
}
|
||||
try {
|
||||
await store.setBackfill(source.id, runs, source.artist_id)
|
||||
toast({
|
||||
text: `Deep scan armed for ${runs} run${runs === 1 ? '' : 's'}`,
|
||||
type: 'success',
|
||||
})
|
||||
await store.loadAll()
|
||||
} catch (e) {
|
||||
toast({
|
||||
text: `Deep scan failed: ${e?.detail || e?.message || e}`,
|
||||
type: 'error',
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async function checkAll(group) {
|
||||
let ok = 0
|
||||
let conflict = 0
|
||||
|
||||
@@ -23,7 +23,11 @@ export const useCredentialsStore = defineStore('credentials', () => {
|
||||
const rec = await api.post('/api/credentials', {
|
||||
body: { platform, credential_type, data },
|
||||
})
|
||||
byPlatform.value.delete(platform)
|
||||
// Reflect the returned record immediately — the previous .delete()
|
||||
// call left the card rendering "no credential" for the gap between
|
||||
// upload completion and the caller's follow-up loadAll(). Audit
|
||||
// 2026-06-02.
|
||||
byPlatform.value.set(platform, rec)
|
||||
return rec
|
||||
}
|
||||
|
||||
|
||||
@@ -55,6 +55,21 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
||||
return selected.value
|
||||
}
|
||||
|
||||
// Open the detail modal for the most recent DownloadEvent of a given
|
||||
// source. Used by the failing-sources rollup's "Logs" button so the
|
||||
// operator can troubleshoot without leaving the Downloads tab to find
|
||||
// the row (operator-flagged 2026-06-01).
|
||||
async function loadLastForSource(sourceId) {
|
||||
const events = await api.get('/api/downloads', {
|
||||
params: { source_id: sourceId, limit: 1 },
|
||||
})
|
||||
if (!events.length) {
|
||||
selected.value = null
|
||||
return null
|
||||
}
|
||||
return await loadOne(events[0].id)
|
||||
}
|
||||
|
||||
async function applyFilter(patch) {
|
||||
filter.value = { ...filter.value, ...patch }
|
||||
await loadFirst()
|
||||
@@ -98,7 +113,8 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
||||
return {
|
||||
events, cursor, hasMore, filter, selected, loading, error, stats,
|
||||
activity, failing, activeEvents,
|
||||
loadFirst, loadMore, loadOne, applyFilter, closeDetail, loadStats,
|
||||
loadFirst, loadMore, loadOne, loadLastForSource, applyFilter,
|
||||
closeDetail, loadStats,
|
||||
loadActivity, loadFailing, loadActive, recoverStalled,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -59,14 +59,37 @@ export const usePostsStore = defineStore('posts', () => {
|
||||
return await api.get(`/api/posts/${id}`)
|
||||
}
|
||||
|
||||
// Filter overlay for the around/older/newer (in-context anchored)
|
||||
// path. Keep this distinct from `filters.value` (the down-only feed)
|
||||
// so a normal-feed filter change doesn't leak into an active anchored
|
||||
// view (or vice versa). Caller of loadAround passes the snapshot; the
|
||||
// subsequent loadOlder/loadNewer use it verbatim.
|
||||
function _aroundParams(extra) {
|
||||
const p = { ...extra }
|
||||
if (filters.value.artist_id != null) p.artist_id = filters.value.artist_id
|
||||
if (filters.value.platform) p.platform = filters.value.platform
|
||||
return p
|
||||
}
|
||||
|
||||
// Load a window centered on `postId`: newer posts above, the post, older
|
||||
// posts below. Sets both directional cursors for subsequent scrolling.
|
||||
async function loadAround(postId) {
|
||||
// Accepts the same filter shape as loadInitial so the anchored view
|
||||
// stays artist/platform-scoped (operator-flagged 2026-06-01: clicking a
|
||||
// post title from the modal's Provenance card opens the post in the
|
||||
// posts feed; without this the older/newer scroll loaded unfiltered
|
||||
// global posts instead of staying in the artist's stream).
|
||||
async function loadAround(postId, newFilters) {
|
||||
filters.value = {
|
||||
artist_id: newFilters?.artist_id ?? null,
|
||||
platform: newFilters?.platform ?? null,
|
||||
}
|
||||
loading.value = true
|
||||
error.value = null
|
||||
anchorId.value = null
|
||||
try {
|
||||
const body = await api.get('/api/posts', { params: { around: postId } })
|
||||
const body = await api.get('/api/posts', {
|
||||
params: _aroundParams({ around: postId }),
|
||||
})
|
||||
items.value = body.items
|
||||
cursorOlder.value = body.cursor_older
|
||||
cursorNewer.value = body.cursor_newer
|
||||
@@ -85,7 +108,9 @@ export const usePostsStore = defineStore('posts', () => {
|
||||
loading.value = true
|
||||
try {
|
||||
const body = await api.get('/api/posts', {
|
||||
params: { cursor: cursorOlder.value, direction: 'older' },
|
||||
params: _aroundParams({
|
||||
cursor: cursorOlder.value, direction: 'older',
|
||||
}),
|
||||
})
|
||||
items.value.push(...body.items)
|
||||
cursorOlder.value = body.next_cursor
|
||||
@@ -102,7 +127,9 @@ export const usePostsStore = defineStore('posts', () => {
|
||||
loading.value = true
|
||||
try {
|
||||
const body = await api.get('/api/posts', {
|
||||
params: { cursor: cursorNewer.value, direction: 'newer' },
|
||||
params: _aroundParams({
|
||||
cursor: cursorNewer.value, direction: 'newer',
|
||||
}),
|
||||
})
|
||||
items.value.unshift(...body.items)
|
||||
cursorNewer.value = body.next_cursor
|
||||
|
||||
@@ -19,6 +19,16 @@ import { useApi } from '../composables/useApi.js'
|
||||
const PAGE = 3
|
||||
const INITIAL_BATCHES = 20
|
||||
const APPEND_DELAY_MS = 80 // ≈ the MasonryGrid stagger animation (70 ms)
|
||||
// Operator-flagged 2026-06-01: scrolling the showcase eventually hit a
|
||||
// premature "End." because /api/showcase returns a *random sample* and
|
||||
// after enough scrolling the `seen` Set accumulated enough to fully
|
||||
// collide with a 3-item batch. The showcase is supposed to be endless;
|
||||
// only a genuinely empty API response (library has zero images) should
|
||||
// mark it exhausted. Retry up to FETCH_RETRY_CAP times on all-dupe
|
||||
// batches; only flip `exhausted` when the API returns 0 items OR every
|
||||
// retry came back dupe-only (graceful fallback for tiny libraries
|
||||
// where retries will keep returning the same handful of items).
|
||||
const FETCH_RETRY_CAP = 8
|
||||
|
||||
|
||||
function _sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
|
||||
@@ -48,17 +58,32 @@ export const useShowcaseStore = defineStore('showcase', () => {
|
||||
}
|
||||
}
|
||||
|
||||
// Single batch — used by infinite-scroll appends. Trickles its 5 items
|
||||
// in for the same one-at-a-time cadence as the initial load.
|
||||
// Single batch — used by infinite-scroll appends. Trickles its items
|
||||
// in for the same one-at-a-time cadence as the initial load. Retries
|
||||
// up to FETCH_RETRY_CAP times when the API's random sample comes back
|
||||
// all-duplicates (the showcase is endless by design; only a genuinely
|
||||
// empty API response should mark it exhausted, not an unlucky sample).
|
||||
async function fetchPage() {
|
||||
if (loading.value) return
|
||||
loading.value = true
|
||||
error.value = null
|
||||
try {
|
||||
const body = await api.get('/api/showcase', { params: { limit: PAGE } })
|
||||
const fresh = (body.images || []).filter(i => !seen.has(i.id))
|
||||
if (fresh.length === 0) { exhausted.value = true; return }
|
||||
await _trickleAppend(fresh, _seq)
|
||||
for (let attempt = 0; attempt < FETCH_RETRY_CAP; attempt++) {
|
||||
const body = await api.get('/api/showcase', { params: { limit: PAGE } })
|
||||
const items = body.images || []
|
||||
// API genuinely empty → library is empty / endpoint exhausted.
|
||||
if (items.length === 0) { exhausted.value = true; return }
|
||||
const fresh = items.filter(i => !seen.has(i.id))
|
||||
if (fresh.length > 0) {
|
||||
await _trickleAppend(fresh, _seq)
|
||||
return
|
||||
}
|
||||
// All-dupes batch — keep trying. Showcase is endless by intent.
|
||||
}
|
||||
// Retry cap hit with zero fresh items: library is probably much
|
||||
// smaller than the running `seen` set, fall back to exhausted so
|
||||
// the UI stops trying. Operator can shuffle to reset `seen`.
|
||||
exhausted.value = true
|
||||
} catch (e) {
|
||||
error.value = e.message || String(e)
|
||||
} finally {
|
||||
|
||||
@@ -85,6 +85,15 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
}
|
||||
}
|
||||
|
||||
// Plan #544: arm a source for backfill mode. The next `runs` download
|
||||
// runs (default 3) walk gallery-dl's full post history instead of
|
||||
// exiting early at the first contiguous archived block.
|
||||
async function setBackfill(id, runs = 3, artistIdHint = null) {
|
||||
const body = await api.post(`/api/sources/${id}/backfill`, { body: { runs } })
|
||||
_invalidate(artistIdHint ?? body.artist_id)
|
||||
return body
|
||||
}
|
||||
|
||||
function sourcesByArtistGrouped() {
|
||||
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
||||
const arr = byArtist.value.get(null) ?? []
|
||||
@@ -111,6 +120,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
loadAll, loadForArtist,
|
||||
create, update, remove,
|
||||
checkNow,
|
||||
setBackfill,
|
||||
findOrCreateArtist, autocompleteArtist,
|
||||
loadScheduleStatus,
|
||||
sourcesByArtistGrouped,
|
||||
|
||||
@@ -4,12 +4,12 @@ import { ref } from 'vue'
|
||||
import { useApi } from '../composables/useApi.js'
|
||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||
|
||||
// Category display order: people/sources first, general last.
|
||||
export const CATEGORY_ORDER = ['artist', 'character', 'copyright', 'general']
|
||||
// Category display order: people first, general last.
|
||||
// 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired — only
|
||||
// character and general surface as suggestion categories now.
|
||||
export const CATEGORY_ORDER = ['character', 'general']
|
||||
export const CATEGORY_LABELS = {
|
||||
artist: 'Artist',
|
||||
character: 'Character',
|
||||
copyright: 'Copyright',
|
||||
general: 'General'
|
||||
}
|
||||
|
||||
|
||||
@@ -4,8 +4,11 @@ import { useApi } from '../composables/useApi.js'
|
||||
export const useThumbnailsStore = defineStore('thumbnails', () => {
|
||||
const api = useApi()
|
||||
|
||||
// Returns { scanned, enqueued, ok, regenerated } — the API now runs
|
||||
// the scan synchronously so the operator gets immediate feedback
|
||||
// instead of just a Celery task id with no visible outcome.
|
||||
async function triggerBackfill () {
|
||||
await api.post('/api/thumbnails/backfill')
|
||||
return await api.post('/api/thumbnails/backfill')
|
||||
}
|
||||
|
||||
return { triggerBackfill }
|
||||
|
||||
@@ -161,7 +161,15 @@ function setupAroundObservers() {
|
||||
}
|
||||
async function loadAroundAndAnchor() {
|
||||
teardownFeed()
|
||||
await store.loadAround(postIdFilter.value)
|
||||
// Pass artist_id + platform through so the anchored view stays
|
||||
// scoped — the older/newer infinite scrolls then read these filters
|
||||
// back via the store's _aroundParams (operator-flagged 2026-06-01:
|
||||
// post-title click from the modal landed scoped but the scroll then
|
||||
// pulled unfiltered global posts).
|
||||
await store.loadAround(postIdFilter.value, {
|
||||
artist_id: artistFilter.value,
|
||||
platform: platformFilter.value,
|
||||
})
|
||||
await nextTick()
|
||||
const el = document.getElementById(`fc-post-${store.anchorId}`)
|
||||
if (el) el.scrollIntoView({ block: 'center' })
|
||||
|
||||
@@ -70,7 +70,12 @@ function onKeydown(e) {
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
if (store.images.length === 0) store.loadInitial()
|
||||
// Operator-stated 2026-06-02: every load of the showcase view should
|
||||
// show new images. Pinia persists the store across navigations, so
|
||||
// returning to /showcase used to render the same set as before. Now
|
||||
// we always reshuffle on mount — loadInitial() resets `seen`,
|
||||
// `images`, and `exhausted` and pipelines a fresh batch.
|
||||
store.loadInitial()
|
||||
window.addEventListener('keydown', onKeydown)
|
||||
})
|
||||
onUnmounted(() => window.removeEventListener('keydown', onKeydown))
|
||||
|
||||
@@ -27,12 +27,17 @@ describe('credentials store', () => {
|
||||
expect(s.byPlatform.get('patreon').credential_type).toBe('cookies')
|
||||
})
|
||||
|
||||
it('upload invalidates the cache', async () => {
|
||||
it('upload reflects the returned record into the cache', async () => {
|
||||
// The previous behavior was to delete the cache entry on upload, which
|
||||
// briefly showed "no credential" until a follow-up loadAll() resolved.
|
||||
// Audit 2026-06-02 corrected this: upload now stores the returned
|
||||
// record directly so the card updates immediately.
|
||||
const s = useCredentialsStore()
|
||||
s.byPlatform.set('patreon', { platform: 'patreon' })
|
||||
stubFetch(() => ({ status: 201, body: { platform: 'patreon', credential_type: 'cookies' } }))
|
||||
s.byPlatform.set('patreon', { platform: 'patreon', credential_type: 'token' })
|
||||
const fresh = { platform: 'patreon', credential_type: 'cookies' }
|
||||
stubFetch(() => ({ status: 201, body: fresh }))
|
||||
await s.upload('patreon', 'cookies', 'NETSCAPE_CONTENT')
|
||||
expect(s.byPlatform.has('patreon')).toBe(false)
|
||||
expect(s.byPlatform.get('patreon')).toEqual(fresh)
|
||||
})
|
||||
|
||||
it('remove deletes and invalidates the cache', async () => {
|
||||
|
||||
@@ -37,8 +37,8 @@ async def test_artist_overview_post_count(client, db):
|
||||
)
|
||||
db.add(s)
|
||||
await db.flush()
|
||||
db.add(Post(source_id=s.id, external_post_id="p1"))
|
||||
db.add(Post(source_id=s.id, external_post_id="p2"))
|
||||
db.add(Post(source_id=s.id, artist_id=a.id, external_post_id="p1"))
|
||||
db.add(Post(source_id=s.id, artist_id=a.id, external_post_id="p2"))
|
||||
await db.flush()
|
||||
await db.commit()
|
||||
resp = await client.get("/api/artist/lyra")
|
||||
|
||||
@@ -135,6 +135,114 @@ async def test_quick_add_source_wrong_key_401(client, ext_key):
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
# --- /api/extension/probe ---------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_returns_new_when_nothing_exists(client, ext_key):
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
query_string={"url": "https://www.patreon.com/freshcreator"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["state"] == "new"
|
||||
assert body["platform"] == "patreon"
|
||||
assert body["slug"] == "freshcreator"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_returns_source_match_for_already_added(client, ext_key, db):
|
||||
artist = Artist(name="Alice", slug="alice", is_subscription=True)
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://www.patreon.com/alice", enabled=True, config_overrides={},
|
||||
)
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
query_string={"url": "https://www.patreon.com/alice"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["state"] == "source_match"
|
||||
assert body["artist"]["slug"] == "alice"
|
||||
assert body["source"]["url"] == "https://www.patreon.com/alice"
|
||||
assert body["source"]["platform"] == "patreon"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_returns_artist_match_when_only_synthetic_anchor_exists(
|
||||
client, ext_key, db,
|
||||
):
|
||||
"""Filesystem-imported artist with only a sidecar synthetic Source
|
||||
for the (artist, platform) — the URL the operator's browsing isn't
|
||||
yet a real Source. The probe should collapse this into artist_match
|
||||
so the chip says '+ Add Patreon source to Dymkens' rather than
|
||||
'+ Add to FabledCurator' (which would re-create the artist)."""
|
||||
artist = Artist(name="Dymkens", slug="dymkens", is_subscription=False)
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
synthetic = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="sidecar:patreon:dymkens", enabled=False, config_overrides={},
|
||||
)
|
||||
db.add(synthetic)
|
||||
await db.commit()
|
||||
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
query_string={"url": "https://www.patreon.com/dymkens"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["state"] == "artist_match"
|
||||
assert body["artist"]["slug"] == "dymkens"
|
||||
assert "source" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_returns_unknown_platform_for_non_artist_url(client, ext_key):
|
||||
"""A patreon URL that isn't an artist page (e.g. /home, /posts/N)
|
||||
shouldn't trigger the button. Sentinel 'unknown_platform' state
|
||||
tells the content script to skip injection."""
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
query_string={"url": "https://www.patreon.com/posts/12345"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["state"] == "unknown_platform"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_missing_key_401(client):
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
query_string={"url": "https://www.patreon.com/maewix"},
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_missing_url_400(client, ext_key):
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
body = await resp.get_json()
|
||||
assert body["error"] == "invalid_body"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_source_missing_body_400(client, ext_key):
|
||||
resp = await client.post(
|
||||
|
||||
@@ -19,7 +19,12 @@ async def test_get_and_patch_settings(client):
|
||||
resp = await client.get("/api/ml/settings")
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["suggestion_threshold_general"] == pytest.approx(0.95)
|
||||
# Default lowered 0.95 → 0.50 on 2026-06-01 (alembic 0029) — 0.95
|
||||
# hid most general suggestions in the view modal.
|
||||
assert body["suggestion_threshold_general"] == pytest.approx(0.50)
|
||||
# Retired threshold columns must not appear in the payload.
|
||||
assert "suggestion_threshold_artist" not in body
|
||||
assert "suggestion_threshold_copyright" not in body
|
||||
|
||||
resp = await client.patch(
|
||||
"/api/ml/settings", json={"suggestion_threshold_general": 0.90}
|
||||
|
||||
@@ -25,7 +25,7 @@ async def seeded_post(db):
|
||||
db.add(source)
|
||||
await db.flush()
|
||||
post = Post(
|
||||
source_id=source.id, external_post_id="API1",
|
||||
source_id=source.id, artist_id=artist.id, external_post_id="API1",
|
||||
post_title="Hello", post_url="https://p/alice-api/1",
|
||||
post_date=datetime.now(UTC),
|
||||
description="<p>hi</p>",
|
||||
@@ -123,7 +123,8 @@ async def post_timeline(db):
|
||||
posts = []
|
||||
for i in range(5):
|
||||
p = Post(
|
||||
source_id=source.id, external_post_id=f"TL{i}",
|
||||
source_id=source.id, artist_id=artist.id,
|
||||
external_post_id=f"TL{i}",
|
||||
post_title=f"post {i}", post_date=base + timedelta(days=i),
|
||||
)
|
||||
db.add(p)
|
||||
@@ -189,7 +190,7 @@ async def test_detail_returns_uncapped_thumbnails(client, db):
|
||||
db.add(s)
|
||||
await db.flush()
|
||||
p = Post(
|
||||
source_id=s.id, external_post_id="DETAIL10",
|
||||
source_id=s.id, artist_id=a.id, external_post_id="DETAIL10",
|
||||
post_title="big post", description="<p>body</p>",
|
||||
)
|
||||
db.add(p)
|
||||
|
||||
@@ -26,7 +26,8 @@ async def _seed_full(db):
|
||||
url="https://patreon.test/alice")
|
||||
db.add(source)
|
||||
await db.flush()
|
||||
post = Post(source_id=source.id, external_post_id="555",
|
||||
post = Post(source_id=source.id, artist_id=artist.id,
|
||||
external_post_id="555",
|
||||
post_url="https://patreon.test/p/555", post_title="Set 1",
|
||||
post_date=datetime(2023, 8, 1, tzinfo=UTC),
|
||||
description="<p>hi</p>", attachment_count=2)
|
||||
|
||||
@@ -193,3 +193,61 @@ async def test_list_derives_next_check_at_when_last_checked_set(
|
||||
)
|
||||
assert target["next_check_at"] is not None
|
||||
assert "T" in target["next_check_at"] # ISO 8601
|
||||
|
||||
|
||||
# --- Plan #544: POST /api/sources/{id}/backfill ----------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_arms_source(client, artist, db):
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-backfill", enabled=True,
|
||||
)
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
sid = src.id
|
||||
|
||||
resp = await client.post(f"/api/sources/{sid}/backfill", json={"runs": 5})
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["backfill_runs_remaining"] == 5
|
||||
|
||||
# GET reflects the new state.
|
||||
one = await client.get(f"/api/sources/{sid}")
|
||||
assert (await one.get_json())["backfill_runs_remaining"] == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_defaults_to_three(client, artist, db):
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-backfill-default", enabled=True,
|
||||
)
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
resp = await client.post(f"/api/sources/{src.id}/backfill", json={})
|
||||
body = await resp.get_json()
|
||||
assert body["backfill_runs_remaining"] == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_rejects_out_of_range(client, artist, db):
|
||||
src = Source(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-backfill-bad", enabled=True,
|
||||
)
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
bad = await client.post(f"/api/sources/{src.id}/backfill", json={"runs": 0})
|
||||
assert bad.status_code == 400
|
||||
too_big = await client.post(
|
||||
f"/api/sources/{src.id}/backfill", json={"runs": 99}
|
||||
)
|
||||
assert too_big.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_endpoint_404_when_source_missing(client):
|
||||
resp = await client.post("/api/sources/999999/backfill", json={"runs": 3})
|
||||
assert resp.status_code == 404
|
||||
|
||||
@@ -13,8 +13,16 @@ def eager():
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_trigger_thumbnail_backfill(client):
|
||||
async def test_trigger_thumbnail_backfill_returns_counts(client):
|
||||
"""The API runs the scan synchronously now and returns
|
||||
{scanned, enqueued, ok, regenerated} — operator-flagged 2026-06-01:
|
||||
the previous fire-and-forget shape returned only a celery_task_id,
|
||||
so the admin UI couldn't show whether backfill found 0 or 5000
|
||||
candidates. \"Found nothing\" was indistinguishable from \"the
|
||||
worker isn't picking up the task.\""""
|
||||
r = await client.post("/api/thumbnails/backfill")
|
||||
assert r.status_code == 202
|
||||
assert r.status_code == 200
|
||||
body = await r.get_json()
|
||||
assert "celery_task_id" in body
|
||||
assert set(body) == {"scanned", "enqueued", "ok", "regenerated"}
|
||||
for key in ("scanned", "enqueued", "ok", "regenerated"):
|
||||
assert isinstance(body[key], int)
|
||||
|
||||
@@ -25,7 +25,7 @@ async def _fixture(db):
|
||||
db.add(src)
|
||||
await db.flush()
|
||||
post = Post(
|
||||
source_id=src.id, external_post_id="p1",
|
||||
source_id=src.id, artist_id=artist.id, external_post_id="p1",
|
||||
post_date=datetime(2026, 3, 1, tzinfo=UTC),
|
||||
)
|
||||
db.add(post)
|
||||
|
||||
@@ -12,13 +12,16 @@ pytestmark = pytest.mark.integration
|
||||
|
||||
def test_thumb_is_valid_jpeg(tmp_path):
|
||||
p = tmp_path / "good.jpg"
|
||||
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100)
|
||||
# Real thumbnails are at least ~2KB; size check (MIN_THUMB_BYTES=256)
|
||||
# requires the file body be plausible. 300 bytes here clears the
|
||||
# floor with margin.
|
||||
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 300)
|
||||
assert _thumb_is_valid(p) is True
|
||||
|
||||
|
||||
def test_thumb_is_valid_png(tmp_path):
|
||||
p = tmp_path / "good.png"
|
||||
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100)
|
||||
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 300)
|
||||
assert _thumb_is_valid(p) is True
|
||||
|
||||
|
||||
@@ -38,6 +41,16 @@ def test_thumb_is_valid_missing_file(tmp_path):
|
||||
assert _thumb_is_valid(tmp_path / "nope") is False
|
||||
|
||||
|
||||
def test_thumb_is_valid_header_only_below_min_size(tmp_path):
|
||||
"""Operator-flagged 2026-06-01: header-only corrupt files were
|
||||
silently passing the magic-byte check and backfill counted them as
|
||||
`ok`, so the UI's broken-image tiles never got regenerated. Files
|
||||
smaller than MIN_THUMB_BYTES are now invalid even with valid magic."""
|
||||
p = tmp_path / "header_only.jpg"
|
||||
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 50) # 54 bytes total
|
||||
assert _thumb_is_valid(p) is False
|
||||
|
||||
|
||||
# --- backfill_thumbnails planner tests ------------------------------------
|
||||
|
||||
|
||||
@@ -80,13 +93,14 @@ def _rec(db_sync, path, *, sha, thumb_path=None, mime="image/jpeg"):
|
||||
|
||||
def _write_jpeg(p: Path) -> Path:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100)
|
||||
# ≥ MIN_THUMB_BYTES (256) so the size floor doesn't reject it.
|
||||
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 300)
|
||||
return p
|
||||
|
||||
|
||||
def _write_png(p: Path) -> Path:
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100)
|
||||
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 300)
|
||||
return p
|
||||
|
||||
|
||||
@@ -111,7 +125,7 @@ def test_backfill_null_path_enqueued(db_sync, tmp_path, monkeypatch):
|
||||
)
|
||||
|
||||
result = m.backfill_thumbnails()
|
||||
assert result == {"enqueued": 1, "ok": 0, "regenerated": 0}
|
||||
assert result == {"scanned": 1, "enqueued": 1, "ok": 0, "regenerated": 0}
|
||||
assert delayed == [rec.id]
|
||||
|
||||
|
||||
@@ -134,7 +148,7 @@ def test_backfill_missing_file_clears_and_enqueues(db_sync, tmp_path, monkeypatc
|
||||
|
||||
result = m.backfill_thumbnails()
|
||||
db_sync.expire_all()
|
||||
assert result == {"enqueued": 1, "ok": 0, "regenerated": 1}
|
||||
assert result == {"scanned": 1, "enqueued": 1, "ok": 0, "regenerated": 1}
|
||||
assert delayed == [rec.id]
|
||||
assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None
|
||||
|
||||
@@ -156,7 +170,7 @@ def test_backfill_valid_jpeg_skipped(db_sync, tmp_path, monkeypatch):
|
||||
|
||||
result = m.backfill_thumbnails()
|
||||
db_sync.expire_all()
|
||||
assert result == {"enqueued": 0, "ok": 1, "regenerated": 0}
|
||||
assert result == {"scanned": 1, "enqueued": 0, "ok": 1, "regenerated": 0}
|
||||
assert delayed == []
|
||||
assert db_sync.get(ImageRecord, rec.id).thumbnail_path == str(thumb)
|
||||
|
||||
@@ -177,7 +191,7 @@ def test_backfill_valid_png_skipped(db_sync, tmp_path, monkeypatch):
|
||||
)
|
||||
|
||||
result = m.backfill_thumbnails()
|
||||
assert result == {"enqueued": 0, "ok": 1, "regenerated": 0}
|
||||
assert result == {"scanned": 1, "enqueued": 0, "ok": 1, "regenerated": 0}
|
||||
assert delayed == []
|
||||
|
||||
|
||||
@@ -198,7 +212,7 @@ def test_backfill_corrupt_magic_clears_and_enqueues(db_sync, tmp_path, monkeypat
|
||||
|
||||
result = m.backfill_thumbnails()
|
||||
db_sync.expire_all()
|
||||
assert result == {"enqueued": 1, "ok": 0, "regenerated": 1}
|
||||
assert result == {"scanned": 1, "enqueued": 1, "ok": 0, "regenerated": 1}
|
||||
assert delayed == [rec.id]
|
||||
assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None
|
||||
|
||||
@@ -238,5 +252,5 @@ def test_backfill_mixed_aggregate(db_sync, tmp_path, monkeypatch):
|
||||
)
|
||||
|
||||
result = m.backfill_thumbnails()
|
||||
assert result == {"enqueued": 3, "ok": 2, "regenerated": 2}
|
||||
assert result == {"scanned": 5, "enqueued": 3, "ok": 2, "regenerated": 2}
|
||||
assert sorted(delayed) == sorted([r_null.id, r_missing.id, r_bad.id])
|
||||
|
||||
@@ -370,3 +370,246 @@ async def test_finalize_skipped_preserves_failures_clears_error(db):
|
||||
assert row.consecutive_failures == 4
|
||||
assert row.last_error is None
|
||||
assert row.last_checked_at is not None
|
||||
|
||||
|
||||
# --- Plan #544: backfill lifecycle + PARTIAL → status=ok -------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_decrements_after_run(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""When backfill_runs_remaining > 0 going in, a non-clean / non-empty
|
||||
run decrements by 1 — operator gets N runs to complete the deep scan
|
||||
before tick mode resumes."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.backfill_runs_remaining = 3
|
||||
await db.commit()
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||
_make_jpg(f1, split="h")
|
||||
|
||||
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||
success=True, written_paths=[str(f1)], files_downloaded=1,
|
||||
stdout=f"{f1}\n",
|
||||
))
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining = (await db.execute(
|
||||
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert remaining == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backfill_auto_resets_on_clean_zero_files(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""A clean run (rc=0) that downloaded zero files means the backfill
|
||||
queue drained — reset to 0 immediately instead of wasting the rest of
|
||||
the N-run budget on no-op walks."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
source.backfill_runs_remaining = 3
|
||||
await db.commit()
|
||||
|
||||
fake_result = _make_fake_dl_result(
|
||||
success=True, written_paths=[], files_downloaded=0,
|
||||
)
|
||||
fake_gdl = _fake_gdl_with_result(fake_result)
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=tmp_path / "images",
|
||||
import_root=tmp_path / "images",
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining = (await db.execute(
|
||||
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert remaining == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tick_mode_does_not_touch_backfill_counter(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""When backfill_runs_remaining is already 0, downloads don't go
|
||||
negative or otherwise mutate the counter."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
assert source.backfill_runs_remaining == 0
|
||||
|
||||
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||
success=True, written_paths=[], files_downloaded=0,
|
||||
))
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=tmp_path / "images",
|
||||
import_root=tmp_path / "images",
|
||||
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||
settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
remaining = (await db.execute(
|
||||
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert remaining == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_error_type_maps_to_ok_status(
|
||||
db, db_sync, tmp_path, seed_artist_and_source,
|
||||
):
|
||||
"""A non-zero gallery-dl exit accompanied by PARTIAL error_type
|
||||
(real files were downloaded before the run was cut short) is treated
|
||||
as status=ok by the orchestrator — next tick continues."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.gallery_dl import ErrorType
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||
_make_jpg(f1, split="h")
|
||||
|
||||
fake_result = _make_fake_dl_result(
|
||||
success=False, written_paths=[str(f1)], files_downloaded=1,
|
||||
error_type=ErrorType.PARTIAL,
|
||||
error_message="Downloaded 1 file; run did not complete in budget",
|
||||
stdout=f"{f1}\n",
|
||||
)
|
||||
fake_gdl = _fake_gdl_with_result(fake_result)
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
event_id = await svc.download_source(source.id)
|
||||
|
||||
ev = (await db.execute(
|
||||
select(DownloadEvent).where(DownloadEvent.id == event_id)
|
||||
)).scalar_one()
|
||||
assert ev.status == "ok"
|
||||
assert ev.error is None
|
||||
# And the source's failure counter isn't bumped — PARTIAL isn't a failure.
|
||||
src_after = (await db.execute(
|
||||
select(Source.consecutive_failures).where(Source.id == source.id)
|
||||
)).scalar_one()
|
||||
assert src_after == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
|
||||
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
|
||||
):
|
||||
"""Operator-flagged 2026-06-01: downloaded images stayed at
|
||||
thumbnail_path=NULL until periodic backfill swept them up, surfacing
|
||||
as broken-thumbnail tiles in the gallery. Importer.attach_in_place
|
||||
deliberately skips inline generation; the calling code MUST enqueue
|
||||
the thumbnail + ML tasks per attached image (matching the pattern in
|
||||
tasks/import_file.py)."""
|
||||
from backend.app.services.download_service import DownloadService
|
||||
from backend.app.services.importer import Importer
|
||||
|
||||
_artist, source = seed_artist_and_source
|
||||
|
||||
images_root = tmp_path / "images"
|
||||
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||
f2 = images_root / "alice" / "patreon" / "post" / "b.jpg"
|
||||
_make_jpg(f1, split="h")
|
||||
_make_jpg(f2, split="v")
|
||||
|
||||
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||
success=True, written_paths=[str(f1), str(f2)],
|
||||
files_downloaded=2, stdout=f"{f1}\n{f2}\n",
|
||||
))
|
||||
|
||||
sync_settings = db_sync.execute(
|
||||
select(ImportSettings).where(ImportSettings.id == 1)
|
||||
).scalar_one()
|
||||
sync_settings.phash_threshold = 0
|
||||
importer = Importer(
|
||||
session=db_sync, images_root=images_root, import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||
)
|
||||
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64"))
|
||||
|
||||
# Capture the IDs that the orchestrator hands off to each Celery task.
|
||||
# The .delay() shim runs inside DownloadService._phase3_persist (lazy
|
||||
# imports under ..tasks.thumbnail / ..tasks.ml), so monkeypatch the
|
||||
# symbols on those modules.
|
||||
thumb_calls: list[int] = []
|
||||
ml_calls: list[int] = []
|
||||
from backend.app.tasks import ml as ml_mod
|
||||
from backend.app.tasks import thumbnail as thumb_mod
|
||||
monkeypatch.setattr(
|
||||
thumb_mod.generate_thumbnail, "delay",
|
||||
lambda image_id: thumb_calls.append(image_id),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
ml_mod.tag_and_embed, "delay",
|
||||
lambda image_id: ml_calls.append(image_id),
|
||||
)
|
||||
|
||||
svc = DownloadService(
|
||||
async_session=db, sync_session=db_sync,
|
||||
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||
)
|
||||
await svc.download_source(source.id)
|
||||
|
||||
# Two files attached → two thumbnail enqueues + two ML enqueues, IDs
|
||||
# match the actually-imported records (not ad-hoc — drawn from the
|
||||
# importer's returned image_id so future supersede paths stay covered).
|
||||
assert len(thumb_calls) == 2
|
||||
assert len(ml_calls) == 2
|
||||
assert sorted(thumb_calls) == sorted(ml_calls)
|
||||
|
||||
@@ -249,3 +249,80 @@ def test_truncate_log_caps_large_text(gdl):
|
||||
def test_truncate_log_passes_short_text(gdl):
|
||||
text = "small\n"
|
||||
assert gdl._truncate_log(text) == text
|
||||
|
||||
|
||||
# --- Plan #544: tick/backfill skip emission + PARTIAL classifier ------------
|
||||
|
||||
|
||||
def test_build_config_emits_tick_skip_value(gdl):
|
||||
"""Tick mode emits gallery-dl's exit:20 to short-circuit catch-up
|
||||
scans once 20 contiguous archived items are seen."""
|
||||
from backend.app.services.gallery_dl import TICK_SKIP_VALUE
|
||||
cfg = gdl._build_config_for_source(
|
||||
platform="patreon",
|
||||
source_config=SourceConfig(),
|
||||
artist_slug="alice",
|
||||
skip_value=TICK_SKIP_VALUE,
|
||||
)
|
||||
assert cfg["extractor"]["skip"] == "exit:20"
|
||||
|
||||
|
||||
def test_build_config_emits_backfill_skip_value(gdl):
|
||||
"""Backfill mode keeps gallery-dl's default skip=True so it walks the
|
||||
full post history."""
|
||||
from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE
|
||||
cfg = gdl._build_config_for_source(
|
||||
platform="patreon",
|
||||
source_config=SourceConfig(),
|
||||
artist_slug="alice",
|
||||
skip_value=BACKFILL_SKIP_VALUE,
|
||||
)
|
||||
assert cfg["extractor"]["skip"] is True
|
||||
|
||||
|
||||
def test_categorize_partial_when_files_downloaded_then_nonzero_exit(gdl):
|
||||
"""Non-zero exit + ≥1 file downloaded + no source-level error →
|
||||
PARTIAL. Downstream maps to status=ok because the next tick continues."""
|
||||
# stdout has a single downloaded file (line starting with `/`); no
|
||||
# error-class indicators in stderr; non-zero return code.
|
||||
etype, msg = gdl._categorize_error(
|
||||
return_code=1,
|
||||
stdout="/images/alice/patreon/post1/img.jpg\n",
|
||||
stderr="",
|
||||
)
|
||||
assert etype == ErrorType.PARTIAL
|
||||
assert "Downloaded 1 file" in msg
|
||||
|
||||
|
||||
def test_categorize_unknown_when_no_files_and_no_pattern(gdl):
|
||||
"""PARTIAL only kicks in when files were actually written. A non-zero
|
||||
exit with no downloads and no recognized pattern still falls through
|
||||
to UNKNOWN_ERROR — distinct from the partial-success case."""
|
||||
etype, _ = gdl._categorize_error(
|
||||
return_code=2,
|
||||
stdout="",
|
||||
stderr="some unrecognized noise\n",
|
||||
)
|
||||
assert etype == ErrorType.UNKNOWN_ERROR
|
||||
|
||||
|
||||
def test_categorize_tier_limited_wins_over_partial(gdl):
|
||||
"""A run that hit tier-limited warnings AND downloaded some files
|
||||
classifies as TIER_LIMITED (the more specific category), not PARTIAL."""
|
||||
stdout = "/images/alice/patreon/post1/img.jpg\n"
|
||||
stderr = "[patreon][warning] Not allowed to view post 123\n"
|
||||
etype, _ = gdl._categorize_error(
|
||||
return_code=1, stdout=stdout, stderr=stderr,
|
||||
)
|
||||
assert etype == ErrorType.TIER_LIMITED
|
||||
|
||||
|
||||
def test_default_config_forwards_patreon_referer_to_ytdl(gdl):
|
||||
"""Operator-flagged 2026-06-01: Mux video playback restrictions reject
|
||||
yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The
|
||||
fix is a static `downloader.ytdl.raw-options.http_headers` block, so
|
||||
it's enough to assert the keys land in the default config."""
|
||||
cfg = gdl._get_default_config()
|
||||
headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"]
|
||||
assert headers["Referer"] == "https://www.patreon.com/"
|
||||
assert headers["Origin"] == "https://www.patreon.com"
|
||||
|
||||
@@ -35,7 +35,7 @@ async def _post(db, artist_name, slug, ext):
|
||||
url=f"https://patreon.test/{slug}")
|
||||
db.add(s)
|
||||
await db.flush()
|
||||
p = Post(source_id=s.id, external_post_id=ext)
|
||||
p = Post(source_id=s.id, artist_id=a.id, external_post_id=ext)
|
||||
db.add(p)
|
||||
await db.flush()
|
||||
return a, s, p
|
||||
|
||||
@@ -152,7 +152,8 @@ async def _seed_image_with_post(
|
||||
db.add(source)
|
||||
await db.flush()
|
||||
post = Post(
|
||||
source_id=source.id, external_post_id=external_post_id,
|
||||
source_id=source.id, artist_id=artist.id,
|
||||
external_post_id=external_post_id,
|
||||
post_title="A Post", post_date=post_date,
|
||||
)
|
||||
db.add(post)
|
||||
|
||||
@@ -159,3 +159,30 @@ def test_attach_in_place_invalid_image_returns_skipped(importer):
|
||||
result = importer.attach_in_place(bad)
|
||||
assert result.status == "skipped"
|
||||
assert result.skip_reason.value == "invalid_image"
|
||||
|
||||
|
||||
def test_attach_in_place_non_media_routes_to_attachment(importer, db_sync):
|
||||
"""FC-2d-iii dispatch parity for the download path. A non-media file
|
||||
(.pdf, .txt etc.) downloaded by gallery-dl must become a PostAttachment,
|
||||
not bounce back as `skipped+invalid_image` — the latter flipped
|
||||
otherwise-successful runs to status="error" downstream. Operator-flagged
|
||||
2026-06-02 (Lustria patreon OST zip)."""
|
||||
from backend.app.models import PostAttachment
|
||||
|
||||
images_root = importer.images_root
|
||||
artist = Artist(name="Gus", slug="gus")
|
||||
db_sync.add(artist)
|
||||
db_sync.flush()
|
||||
|
||||
txt = images_root / "gus" / "patreon" / "post" / "notes.txt"
|
||||
txt.parent.mkdir(parents=True, exist_ok=True)
|
||||
txt.write_bytes(b"some non-media payload that the importer should preserve")
|
||||
|
||||
result = importer.attach_in_place(txt, artist=artist)
|
||||
assert result.status == "attached"
|
||||
|
||||
row = db_sync.execute(
|
||||
select(PostAttachment).where(PostAttachment.original_filename == "notes.txt")
|
||||
).scalar_one()
|
||||
assert row.ext == ".txt"
|
||||
assert row.artist_id == artist.id
|
||||
|
||||
@@ -29,7 +29,6 @@ from backend.app.models import (
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
Source,
|
||||
)
|
||||
from backend.app.services.importer import Importer
|
||||
from backend.app.services.thumbnailer import Thumbnailer
|
||||
@@ -139,9 +138,11 @@ def test_apply_sidecar_recovers_from_integrity_error(
|
||||
r = importer.import_one(m)
|
||||
assert r.status == "imported"
|
||||
rec = importer.session.get(ImageRecord, r.image_id)
|
||||
src = importer.session.execute(select(Source)).scalar_one()
|
||||
# alembic 0030 stopped creating synthetic Source rows for filesystem
|
||||
# sidecars; the Post sits null-source and the provenance row points at
|
||||
# it directly. The race-recovery path tested below operates on
|
||||
# ImageProvenance regardless of Source presence.
|
||||
assert rec is not None
|
||||
assert src is not None
|
||||
|
||||
# Monkeypatch session.execute so the FIRST select inside _apply_sidecar's
|
||||
# existence-check returns a "no row" wrapper. Subsequent selects (e.g.
|
||||
|
||||
@@ -88,11 +88,36 @@ def test_find_or_create_post_idempotent(importer, artist_row, db_sync):
|
||||
)
|
||||
p1 = importer._find_or_create_post(
|
||||
source_id=src.id, external_post_id="ext-001",
|
||||
artist_id=artist_row.id,
|
||||
)
|
||||
p2 = importer._find_or_create_post(
|
||||
source_id=src.id, external_post_id="ext-001",
|
||||
artist_id=artist_row.id,
|
||||
)
|
||||
assert p1.id == p2.id
|
||||
assert p1.artist_id == artist_row.id
|
||||
|
||||
|
||||
def test_find_or_create_post_idempotent_with_null_source(
|
||||
importer, artist_row, db_sync,
|
||||
):
|
||||
"""Post.source_id is nullable since alembic 0030 — filesystem-imported
|
||||
posts with no live subscription have NULL source_id. The partial
|
||||
unique index `uq_post_artist_external_id_null_source` on
|
||||
(artist_id, external_post_id) WHERE source_id IS NULL guards the
|
||||
dedup; the helper matches on the same (artist_id, external_post_id)
|
||||
keys when source_id is None."""
|
||||
p1 = importer._find_or_create_post(
|
||||
source_id=None, external_post_id="fs-001",
|
||||
artist_id=artist_row.id,
|
||||
)
|
||||
p2 = importer._find_or_create_post(
|
||||
source_id=None, external_post_id="fs-001",
|
||||
artist_id=artist_row.id,
|
||||
)
|
||||
assert p1.id == p2.id
|
||||
assert p1.source_id is None
|
||||
assert p1.artist_id == artist_row.id
|
||||
|
||||
|
||||
def test_find_or_create_source_recovers_from_integrity_error(
|
||||
@@ -147,13 +172,13 @@ def test_find_or_create_source_recovers_from_integrity_error(
|
||||
assert recovered.id == pre_existing.id
|
||||
|
||||
|
||||
def test_source_for_sidecar_reuses_existing_subscription(
|
||||
def test_lookup_source_for_sidecar_returns_existing_subscription(
|
||||
importer, artist_row, db_sync,
|
||||
):
|
||||
"""The filesystem-import sidecar resolver should attach to whatever
|
||||
Source already exists for (artist, platform) — the canonical subscription
|
||||
Source — regardless of its URL. Without this, every imported post
|
||||
spawned its own Source row.
|
||||
"""The sidecar-import Source lookup returns the existing Source for
|
||||
(artist, platform) when one exists — letting filesystem-imported
|
||||
content attach to the artist's real subscription instead of being
|
||||
orphaned.
|
||||
"""
|
||||
canonical = Source(
|
||||
artist_id=artist_row.id, platform="patreon",
|
||||
@@ -162,84 +187,30 @@ def test_source_for_sidecar_reuses_existing_subscription(
|
||||
db_sync.add(canonical)
|
||||
db_sync.flush()
|
||||
|
||||
resolved = importer._source_for_sidecar(
|
||||
resolved = importer._lookup_source_for_sidecar(
|
||||
artist_id=artist_row.id, platform="patreon",
|
||||
artist_slug=artist_row.slug,
|
||||
)
|
||||
assert resolved is not None
|
||||
assert resolved.id == canonical.id
|
||||
|
||||
|
||||
def test_source_for_sidecar_creates_synthetic_anchor_when_none_exists(
|
||||
def test_lookup_source_for_sidecar_returns_none_when_no_subscription(
|
||||
importer, artist_row, db_sync,
|
||||
):
|
||||
"""No subscription Source for this (artist, platform) yet. The helper
|
||||
creates one synthetic anchor (enabled=False, url='sidecar:<plat>:<slug>')
|
||||
so subsequent imports reuse it instead of spawning per-post Sources.
|
||||
"""Alembic 0030 made Post.source_id nullable; the importer no
|
||||
longer creates synthetic `sidecar:<platform>:<slug>` anchor rows
|
||||
when no real subscription exists. The lookup returns None and the
|
||||
caller carries that through as a NULL source_id on the new Post.
|
||||
"""
|
||||
resolved = importer._source_for_sidecar(
|
||||
resolved = importer._lookup_source_for_sidecar(
|
||||
artist_id=artist_row.id, platform="pixiv",
|
||||
artist_slug=artist_row.slug,
|
||||
)
|
||||
assert resolved.url == f"sidecar:pixiv:{artist_row.slug}"
|
||||
assert resolved.enabled is False
|
||||
assert resolved.artist_id == artist_row.id
|
||||
assert resolved.platform == "pixiv"
|
||||
assert resolved is None
|
||||
|
||||
# Second call returns the same row (no new Source spawned).
|
||||
again = importer._source_for_sidecar(
|
||||
artist_id=artist_row.id, platform="pixiv",
|
||||
artist_slug=artist_row.slug,
|
||||
)
|
||||
assert again.id == resolved.id
|
||||
|
||||
|
||||
def test_source_for_sidecar_distinct_platforms_distinct_anchors(
|
||||
importer, artist_row, db_sync,
|
||||
):
|
||||
"""One synthetic anchor per (artist, platform). Different platforms get
|
||||
different anchors even when no campaign Source exists for either.
|
||||
"""
|
||||
p = importer._source_for_sidecar(
|
||||
artist_id=artist_row.id, platform="patreon",
|
||||
artist_slug=artist_row.slug,
|
||||
)
|
||||
x = importer._source_for_sidecar(
|
||||
artist_id=artist_row.id, platform="pixiv",
|
||||
artist_slug=artist_row.slug,
|
||||
)
|
||||
assert p.id != x.id
|
||||
assert p.platform == "patreon"
|
||||
assert x.platform == "pixiv"
|
||||
|
||||
|
||||
def test_source_for_sidecar_prefers_real_over_synthetic_when_both_exist(
|
||||
importer, artist_row, db_sync,
|
||||
):
|
||||
"""When BOTH a synthetic anchor AND a real Source exist for the same
|
||||
(artist, platform), the resolver must return the REAL one. This is the
|
||||
fix for the 2026-05-31 phantom-subscription bug: alembic 0022 had
|
||||
rewritten an older per-post Source row into a sidecar synthetic, and
|
||||
the operator later added the real subscription. The old `ORDER BY
|
||||
id ASC LIMIT 1` lookup picked the older synthetic (lower id),
|
||||
silently attaching every gallery-dl download to the wrong Source.
|
||||
"""
|
||||
synthetic = Source(
|
||||
artist_id=artist_row.id, platform="patreon",
|
||||
url=f"sidecar:patreon:{artist_row.slug}", enabled=False,
|
||||
)
|
||||
db_sync.add(synthetic)
|
||||
db_sync.flush()
|
||||
real = Source(
|
||||
artist_id=artist_row.id, platform="patreon",
|
||||
url="https://www.patreon.com/testartist", enabled=True,
|
||||
)
|
||||
db_sync.add(real)
|
||||
db_sync.flush()
|
||||
assert synthetic.id < real.id # ordering precondition
|
||||
|
||||
resolved = importer._source_for_sidecar(
|
||||
artist_id=artist_row.id, platform="patreon",
|
||||
artist_slug=artist_row.slug,
|
||||
)
|
||||
assert resolved.id == real.id
|
||||
assert resolved.url == "https://www.patreon.com/testartist"
|
||||
# Verify no Source row was created as a side effect.
|
||||
count = db_sync.execute(
|
||||
select(Source).where(
|
||||
Source.artist_id == artist_row.id, Source.platform == "pixiv",
|
||||
)
|
||||
).all()
|
||||
assert count == []
|
||||
|
||||
@@ -17,7 +17,7 @@ async def _post(db, **post_kwargs):
|
||||
db.add(src)
|
||||
await db.flush()
|
||||
post = Post(
|
||||
source_id=src.id, external_post_id="p1",
|
||||
source_id=src.id, artist_id=artist.id, external_post_id="p1",
|
||||
post_date=datetime(2026, 3, 1, tzinfo=UTC),
|
||||
**post_kwargs,
|
||||
)
|
||||
|
||||
@@ -62,7 +62,7 @@ async def _run_backfill(db):
|
||||
async def test_backfill_primary_post(db):
|
||||
rec = await _img(db, 1)
|
||||
a, s = await _artist_source(db, "Alice", "alice")
|
||||
post = Post(source_id=s.id, external_post_id="1")
|
||||
post = Post(source_id=s.id, artist_id=a.id, external_post_id="1")
|
||||
db.add(post)
|
||||
await db.flush()
|
||||
rec.primary_post_id = post.id
|
||||
@@ -78,7 +78,7 @@ async def test_backfill_primary_post(db):
|
||||
async def test_backfill_provenance_fallback(db):
|
||||
rec = await _img(db, 1)
|
||||
a, s = await _artist_source(db, "Bob", "bob")
|
||||
post = Post(source_id=s.id, external_post_id="2")
|
||||
post = Post(source_id=s.id, artist_id=a.id, external_post_id="2")
|
||||
db.add(post)
|
||||
await db.flush()
|
||||
db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id,
|
||||
|
||||
@@ -19,9 +19,9 @@ def test_threshold_for_artist_is_unsurfaced():
|
||||
|
||||
class _S:
|
||||
suggestion_threshold_character = 0.5
|
||||
suggestion_threshold_copyright = 0.5
|
||||
suggestion_threshold_general = 0.5
|
||||
|
||||
svc = SuggestionService.__new__(SuggestionService)
|
||||
# 'artist' must fall through to the 1.01 "never surfaces" default
|
||||
# 'artist' and 'copyright' both retired — fall through to 1.01
|
||||
assert svc._threshold_for(_S(), "artist") == 1.01
|
||||
assert svc._threshold_for(_S(), "copyright") == 1.01
|
||||
|
||||
@@ -25,10 +25,13 @@ def _img(sha: str, predictions: dict) -> ImageRecord:
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_threshold_filters_low_confidence_general(db):
|
||||
# Default general threshold is 0.50 (alembic 0029 lowered it from
|
||||
# 0.95). Use 0.30/0.60 to keep the test asserting threshold behavior
|
||||
# rather than the exact cutoff number.
|
||||
img = _img(
|
||||
"a" * 64,
|
||||
{
|
||||
"smile": {"category": "general", "confidence": 0.80},
|
||||
"lowconf": {"category": "general", "confidence": 0.30},
|
||||
"sword": {"category": "general", "confidence": 0.97},
|
||||
},
|
||||
)
|
||||
@@ -37,7 +40,7 @@ async def test_threshold_filters_low_confidence_general(db):
|
||||
sl = await SuggestionService(db).for_image(img.id)
|
||||
names = [s.display_name for s in sl.by_category.get("general", [])]
|
||||
assert "sword" in names
|
||||
assert "smile" not in names
|
||||
assert "lowconf" not in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -17,10 +17,13 @@ from backend.app.services.ml.tagger import (
|
||||
|
||||
|
||||
def test_surfaced_categories():
|
||||
# FC-2d-vii-c: 'artist' retired — artist identity is acquisition-derived
|
||||
# (image_record.artist_id), never ML-inferred.
|
||||
assert SURFACED_CATEGORIES == {"character", "copyright", "general"}
|
||||
# FC-2d-vii-c: 'artist' retired — artist identity is acquisition-
|
||||
# derived (image_record.artist_id), never ML-inferred.
|
||||
# 2026-06-01: 'copyright' retired — fandom serves as the franchise/
|
||||
# copyright concept; operator doesn't use a separate copyright kind.
|
||||
assert SURFACED_CATEGORIES == {"character", "general"}
|
||||
assert "artist" not in SURFACED_CATEGORIES
|
||||
assert "copyright" not in SURFACED_CATEGORIES
|
||||
|
||||
|
||||
def test_store_floor_is_low():
|
||||
|
||||
@@ -15,7 +15,6 @@ from backend.app.models import (
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
Post,
|
||||
Source,
|
||||
Tag,
|
||||
TagKind,
|
||||
)
|
||||
@@ -197,10 +196,11 @@ def test_supersede_applies_new_file_sidecar(importer, import_layout):
|
||||
assert post.post_title == "Set 1"
|
||||
assert "big version" in (post.description or "")
|
||||
|
||||
source = importer.session.execute(
|
||||
select(Source).where(Source.id == post.source_id)
|
||||
).scalar_one()
|
||||
assert source.platform == "patreon"
|
||||
# Filesystem sidecars no longer create a synthetic Source (alembic 0030).
|
||||
# The Post sits null-source; the platform context is captured in the
|
||||
# sidecar JSON / raw_metadata, not in a phantom Source row.
|
||||
assert post.source_id is None
|
||||
assert post.artist_id is not None
|
||||
|
||||
prov_count = importer.session.execute(
|
||||
select(func.count(ImageProvenance.id))
|
||||
|
||||
@@ -7,6 +7,7 @@ boundary, and the thumbnails/attachments composition.
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from backend.app.models import (
|
||||
Artist,
|
||||
@@ -59,8 +60,13 @@ async def _seed_post(
|
||||
db, source_id: int, *, external_id: str,
|
||||
post_date=None, downloaded_at=None, title=None, description=None,
|
||||
):
|
||||
# Post.artist_id is NOT NULL since alembic 0030; look it up from
|
||||
# the source so callsites don't have to thread the artist through.
|
||||
artist_id = (await db.execute(
|
||||
select(Source.artist_id).where(Source.id == source_id)
|
||||
)).scalar_one()
|
||||
p = Post(
|
||||
source_id=source_id, external_post_id=external_id,
|
||||
source_id=source_id, artist_id=artist_id, external_post_id=external_id,
|
||||
post_title=title, post_date=post_date,
|
||||
description=description,
|
||||
)
|
||||
|
||||
@@ -37,7 +37,7 @@ async def _seed_post(db, *, artist_name, slug, platform, ext_id,
|
||||
db.add(source)
|
||||
await db.flush()
|
||||
post = Post(
|
||||
source_id=source.id, external_post_id=ext_id,
|
||||
source_id=source.id, artist_id=artist.id, external_post_id=ext_id,
|
||||
post_url=f"https://{platform}.test/p/{ext_id}",
|
||||
post_title=title, post_date=datetime(2023, 8, 1, tzinfo=UTC),
|
||||
description=desc, attachment_count=count,
|
||||
|
||||
@@ -75,9 +75,15 @@ def test_sidecar_creates_provenance(importer, import_layout):
|
||||
r = importer.import_one(m)
|
||||
assert r.status == "imported"
|
||||
rec = importer.session.get(ImageRecord, r.image_id)
|
||||
src = importer.session.execute(select(Source)).scalar_one()
|
||||
post = importer.session.execute(select(Post)).scalar_one()
|
||||
assert src.platform == "patreon"
|
||||
# Filesystem-imported sidecar posts no longer create a synthetic Source
|
||||
# (alembic 0030 / nullable post.source_id refactor). The Post is linked
|
||||
# to the artist via Post.artist_id; Post.source_id stays NULL until a
|
||||
# real subscription for the (artist, platform) gets added.
|
||||
assert post.source_id is None
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(Source)
|
||||
).scalar_one() == 0
|
||||
assert post.external_post_id == "555"
|
||||
assert post.post_url == "https://patreon.com/posts/555"
|
||||
assert post.post_title == "Set 1"
|
||||
@@ -106,9 +112,11 @@ def test_reimport_same_post_idempotent(importer, import_layout):
|
||||
_sidecar(m2, payload)
|
||||
r2 = importer.import_one(m2)
|
||||
assert r2.status == "imported"
|
||||
# No synthetic Source after alembic 0030; both imports still resolve to
|
||||
# a single null-source Post (deduped by uq_post_artist_external_id_null_source).
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(Source)
|
||||
).scalar_one() == 1
|
||||
).scalar_one() == 0
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(Post)
|
||||
).scalar_one() == 1
|
||||
@@ -164,5 +172,11 @@ def test_sidecar_artist_used_when_no_folder_artist(importer, import_layout):
|
||||
a = importer.session.execute(
|
||||
select(Artist).where(Artist.slug == "yuki")
|
||||
).scalar_one()
|
||||
src = importer.session.execute(select(Source)).scalar_one()
|
||||
assert src.artist_id == a.id
|
||||
# No synthetic Source after alembic 0030; the artist linkage lives on
|
||||
# Post.artist_id (NOT NULL FK).
|
||||
post = importer.session.execute(select(Post)).scalar_one()
|
||||
assert post.artist_id == a.id
|
||||
assert post.source_id is None
|
||||
assert importer.session.execute(
|
||||
select(func.count()).select_from(Source)
|
||||
).scalar_one() == 0
|
||||
|
||||
@@ -138,7 +138,7 @@ async def test_update_changes_fields(db):
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_hides_sidecar_synthetic_anchors(db):
|
||||
"""Filesystem-import synthetic Sources (url='sidecar:<platform>:<slug>',
|
||||
enabled=False — see importer._source_for_sidecar) used to leak into the
|
||||
enabled=False — historical pre-alembic-0030 artifact) used to leak into the
|
||||
Subscriptions UI as phantom subscriptions because list() didn't filter
|
||||
them. They aren't pollable feeds; hide by default."""
|
||||
artist = await _artist(db, "Alice")
|
||||
@@ -169,3 +169,75 @@ async def test_list_hides_sidecar_synthetic_anchors(db):
|
||||
assert {s.url for s in everything} >= {
|
||||
"https://patreon.com/alice", "sidecar:patreon:alice",
|
||||
}
|
||||
|
||||
|
||||
# --- Plan #544: backfill counter -------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_backfill_runs_arms_source(db):
|
||||
"""The service method overrides backfill_runs_remaining (regardless of
|
||||
the auto-arm-on-create starting value) and returns the updated record
|
||||
so the API can echo it back."""
|
||||
artist = await _artist(db, "Alice")
|
||||
svc = SourceService(db)
|
||||
rec = await svc.create(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice",
|
||||
)
|
||||
updated = await svc.set_backfill_runs(rec.id, 5)
|
||||
assert updated.backfill_runs_remaining == 5
|
||||
|
||||
db_value = (await db.execute(
|
||||
select(Source.backfill_runs_remaining).where(Source.id == rec.id)
|
||||
)).scalar_one()
|
||||
assert db_value == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_backfill_runs_rejects_out_of_range(db):
|
||||
artist = await _artist(db, "Alice")
|
||||
svc = SourceService(db)
|
||||
rec = await svc.create(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice",
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
await svc.set_backfill_runs(rec.id, 0)
|
||||
with pytest.raises(ValueError):
|
||||
await svc.set_backfill_runs(rec.id, 11)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_backfill_runs_raises_when_source_missing(db):
|
||||
svc = SourceService(db)
|
||||
with pytest.raises(LookupError):
|
||||
await svc.set_backfill_runs(99999, 3)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_enabled_source_starts_in_backfill_mode(db):
|
||||
"""Plan #544 follow-up: freshly added enabled sources have no archive
|
||||
yet, so the first few polls would blow the wall-clock cap in tick
|
||||
mode. Pre-arm backfill so the initial walks use the longer timeout."""
|
||||
artist = await _artist(db, "Alice")
|
||||
svc = SourceService(db)
|
||||
rec = await svc.create(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-new",
|
||||
)
|
||||
assert rec.backfill_runs_remaining == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_new_disabled_source_skips_backfill(db):
|
||||
"""Disabled sources (incl. sidecar synthetics that arrive disabled) are
|
||||
never polled, so don't burn a backfill budget on them."""
|
||||
artist = await _artist(db, "Alice")
|
||||
svc = SourceService(db)
|
||||
rec = await svc.create(
|
||||
artist_id=artist.id, platform="patreon",
|
||||
url="https://patreon.com/alice-disabled",
|
||||
enabled=False,
|
||||
)
|
||||
assert rec.backfill_runs_remaining == 0
|
||||
|
||||
Reference in New Issue
Block a user