Compare commits
60 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0533807669 | |||
| d3245f0c22 | |||
| 279dff3fb6 | |||
| e450145304 | |||
| a6e8d4b52e | |||
| 37e66cddc4 | |||
| f1860866de | |||
| 9cf6b2d363 | |||
| b181d779fe | |||
| 0fbb19dc24 | |||
| 8326e5447a | |||
| 1fd594baaf | |||
| ecac6c4bda | |||
| 6ef0fed41f | |||
| 9f7261b9c0 | |||
| f05aaa707b | |||
| 4df98171ab | |||
| 8d75ade1d5 | |||
| 75c63e1511 | |||
| 98673d4dca | |||
| 89b48f8f35 | |||
| 4bff1d8558 | |||
| d60e0b9494 | |||
| e30f50e6fe | |||
| 9c27a2d3c7 | |||
| e66987f092 | |||
| 93e37681b7 | |||
| 80ef9bce48 | |||
| 3898ce7be4 | |||
| 64ca858574 | |||
| 91be9df671 | |||
| 412edec028 | |||
| 937421485d | |||
| 9d0c0b7da8 | |||
| 43b778aa04 | |||
| 9cbdb70e13 | |||
| 8e4d252ae4 | |||
| bd06794647 | |||
| fdd3e01f56 | |||
| f575cfb93b | |||
| c82fb308b6 | |||
| 717b601c81 | |||
| cfa4fb4084 | |||
| 2aa2002f22 | |||
| 66ff671f09 | |||
| 19aece1fc4 | |||
| c9089b1d03 | |||
| 644d538bab | |||
| ff35da4743 | |||
| 2f66de2928 | |||
| 8cf8d2ca4d | |||
| 94e7d20792 | |||
| fb605af959 | |||
| 4c56cf121f | |||
| b1d58bc3b8 | |||
| 9564d073b9 | |||
| 65386f02a0 | |||
| f87a06a6bd | |||
| 5d284aae9f | |||
| af7b5c95e9 |
@@ -258,7 +258,11 @@ jobs:
|
|||||||
# anything else → safety net; shouldn't fire given the `on:`
|
# anything else → safety net; shouldn't fire given the `on:`
|
||||||
# config above. Tag :dev to surface the
|
# config above. Tag :dev to surface the
|
||||||
# unexpected run in the registry.
|
# unexpected run in the registry.
|
||||||
SHORT_SHA="${GITHUB_SHA:0:7}"
|
# POSIX-safe substring (the runner shell is dash/BusyBox sh, not
|
||||||
|
# bash — `${var:0:7}` errors with "Bad substitution"; cut works
|
||||||
|
# everywhere). Operator-flagged 2026-06-01 after first :c-<sha>
|
||||||
|
# main-push build failed at this step.
|
||||||
|
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||||
@@ -297,7 +301,11 @@ jobs:
|
|||||||
# safety-net dev) including the per-commit :c-<short_sha> tag
|
# safety-net dev) including the per-commit :c-<short_sha> tag
|
||||||
# on main-push per the family release-posture rule. The -ml
|
# on main-push per the family release-posture rule. The -ml
|
||||||
# image follows the same release cadence as the web image.
|
# image follows the same release cadence as the web image.
|
||||||
SHORT_SHA="${GITHUB_SHA:0:7}"
|
# POSIX-safe substring (the runner shell is dash/BusyBox sh, not
|
||||||
|
# bash — `${var:0:7}` errors with "Bad substitution"; cut works
|
||||||
|
# everywhere). Operator-flagged 2026-06-01 after first :c-<sha>
|
||||||
|
# main-push build failed at this step.
|
||||||
|
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||||
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
if [ "${GITHUB_REF#refs/tags/}" != "${GITHUB_REF}" ]; then
|
||||||
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
TAG_NAME="${GITHUB_REF#refs/tags/}"
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:${TAG_NAME}" >> "$GITHUB_OUTPUT"
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
"""drop artist + copyright ml thresholds; lower general default to 0.50
|
||||||
|
|
||||||
|
Revision ID: 0029
|
||||||
|
Revises: 0028
|
||||||
|
Create Date: 2026-06-01
|
||||||
|
|
||||||
|
Operator-flagged 2026-06-01: the view modal's Suggestions panel hides
|
||||||
|
most general-category predictions because the default threshold is
|
||||||
|
0.95. Lowering the default to 0.50 (matches character) so general
|
||||||
|
suggestions surface more aggressively; the value remains tunable in
|
||||||
|
Settings → ML.
|
||||||
|
|
||||||
|
Same change retires two ML suggestion categories whose Tag.kind
|
||||||
|
surfaces are unused:
|
||||||
|
|
||||||
|
- `artist`: retired in FC-2d-vii-c — artist identity is acquisition-
|
||||||
|
derived (image_record.artist_id), never ML-inferred. The threshold
|
||||||
|
column was a leftover from before that retirement.
|
||||||
|
- `copyright`: retired 2026-06-01 — the app uses `fandom` for the
|
||||||
|
franchise/copyright concept (per TagsView.vue's doc comment); no
|
||||||
|
Tag rows of kind=copyright exist, and the threshold column never
|
||||||
|
fed anything user-visible.
|
||||||
|
|
||||||
|
Both columns are dropped from ml_settings; the existing row's
|
||||||
|
suggestion_threshold_general value is bumped from 0.95 to 0.50 iff
|
||||||
|
it's still at the old default, so deployed installs pick up the new
|
||||||
|
UX without overriding any operator tuning.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
revision: str = "0029"
|
||||||
|
down_revision: Union[str, None] = "0028"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# Bump the general threshold for installs still at the old default.
|
||||||
|
op.execute(text(
|
||||||
|
"UPDATE ml_settings "
|
||||||
|
"SET suggestion_threshold_general = 0.50 "
|
||||||
|
"WHERE id = 1 AND suggestion_threshold_general = 0.95"
|
||||||
|
))
|
||||||
|
op.drop_column("ml_settings", "suggestion_threshold_artist")
|
||||||
|
op.drop_column("ml_settings", "suggestion_threshold_copyright")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Restore the columns with their prior defaults. The bump from
|
||||||
|
# 0.95 → 0.50 isn't reversible without remembering whether the
|
||||||
|
# operator had explicitly set 0.95 (unlikely — that was just the
|
||||||
|
# default) so we leave the current general value as-is.
|
||||||
|
from sqlalchemy import Column, Float
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
Column(
|
||||||
|
"suggestion_threshold_artist",
|
||||||
|
Float, nullable=False, server_default="0.30",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
Column(
|
||||||
|
"suggestion_threshold_copyright",
|
||||||
|
Float, nullable=False, server_default="0.50",
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -0,0 +1,145 @@
|
|||||||
|
"""nullable post.source_id + denormalized post.artist_id; retire sidecar synthetics
|
||||||
|
|
||||||
|
Revision ID: 0030
|
||||||
|
Revises: 0029
|
||||||
|
Create Date: 2026-06-01
|
||||||
|
|
||||||
|
Operator-asked 2026-06-01 after the Dymkens orphan investigation: the
|
||||||
|
sidecar synthetic Source pattern (`sidecar:<platform>:<slug>` rows
|
||||||
|
with enabled=false) was technically correct but misled the operator
|
||||||
|
into thinking they had phantom subscriptions. The synthetics existed
|
||||||
|
solely to satisfy `Post.source_id NOT NULL` for filesystem-imported
|
||||||
|
content with no real subscription.
|
||||||
|
|
||||||
|
This migration makes the data model honest:
|
||||||
|
|
||||||
|
1. **Post gets a denormalized `artist_id` column** so artist filters
|
||||||
|
work without traversing `Post → Source.artist_id`. Backfilled from
|
||||||
|
the existing Source linkage, then NOT NULL'd.
|
||||||
|
2. **`Post.source_id` becomes nullable**, FK ondelete `CASCADE` → `SET
|
||||||
|
NULL`. Deleting a Source detaches its Posts instead of destroying
|
||||||
|
imported content (semantically: subscription ends, archive stays).
|
||||||
|
3. **`ImageProvenance.source_id` becomes nullable** with the same FK
|
||||||
|
semantic change.
|
||||||
|
4. **Sidecar synthetic Sources are deleted** — first NULL out the
|
||||||
|
FKs from Post + ImageProvenance pointing at them (so the implicit
|
||||||
|
CASCADE doesn't fire), then delete. DownloadEvent FK is unchanged
|
||||||
|
(still CASCADE'd, NOT NULL'd) — synthetics have `enabled=false`
|
||||||
|
so no events exist for them.
|
||||||
|
|
||||||
|
Uniqueness handling: the existing `uq_post_source_external_id`
|
||||||
|
(source_id, external_post_id) keeps working for source-bound Posts
|
||||||
|
(Postgres treats NULL != NULL so NULL-source rows aren't deduped by
|
||||||
|
it). A second partial unique index covers the NULL-source case on
|
||||||
|
(artist_id, external_post_id) so filesystem-imported posts still
|
||||||
|
dedupe within an artist.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
from sqlalchemy import text
|
||||||
|
|
||||||
|
revision: str = "0030"
|
||||||
|
down_revision: Union[str, None] = "0029"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
conn = op.get_bind()
|
||||||
|
|
||||||
|
# Step 1: add Post.artist_id, initially nullable for backfill.
|
||||||
|
# FK naming follows the Base.metadata naming_convention
|
||||||
|
# (fk_<table>_<column>_<referred_table>) — alembic 0001 set this up.
|
||||||
|
op.add_column(
|
||||||
|
"post",
|
||||||
|
sa.Column("artist_id", sa.Integer, nullable=True),
|
||||||
|
)
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_post_artist_id_artist", "post", "artist",
|
||||||
|
["artist_id"], ["id"], ondelete="CASCADE",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 2: backfill from Source.artist_id (every existing Post has a
|
||||||
|
# Source today, so every row gets populated).
|
||||||
|
conn.execute(text("""
|
||||||
|
UPDATE post p
|
||||||
|
SET artist_id = s.artist_id
|
||||||
|
FROM source s
|
||||||
|
WHERE p.source_id = s.id AND p.artist_id IS NULL
|
||||||
|
"""))
|
||||||
|
|
||||||
|
# Sanity: count any remaining NULLs. Should be zero pre-this-migration.
|
||||||
|
remaining = conn.execute(text(
|
||||||
|
"SELECT COUNT(*) FROM post WHERE artist_id IS NULL"
|
||||||
|
)).scalar_one()
|
||||||
|
if remaining:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"alembic 0030: {remaining} post rows have no resolvable "
|
||||||
|
f"artist_id after backfill. Investigate before continuing."
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 3: enforce NOT NULL + add index for artist-filter queries.
|
||||||
|
op.alter_column("post", "artist_id", nullable=False)
|
||||||
|
op.create_index("ix_post_artist_id", "post", ["artist_id"])
|
||||||
|
|
||||||
|
# Step 4: relax post.source_id + flip FK to SET NULL. The original FK
|
||||||
|
# name from alembic 0001 is `fk_post_source_id_source` per the
|
||||||
|
# NAMING_CONVENTION in models/base.py.
|
||||||
|
op.alter_column("post", "source_id", nullable=True)
|
||||||
|
op.drop_constraint("fk_post_source_id_source", "post", type_="foreignkey")
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_post_source_id_source", "post", "source",
|
||||||
|
["source_id"], ["id"], ondelete="SET NULL",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 5: relax image_provenance.source_id + flip FK to SET NULL.
|
||||||
|
op.alter_column("image_provenance", "source_id", nullable=True)
|
||||||
|
op.drop_constraint(
|
||||||
|
"fk_image_provenance_source_id_source", "image_provenance",
|
||||||
|
type_="foreignkey",
|
||||||
|
)
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_image_provenance_source_id_source", "image_provenance", "source",
|
||||||
|
["source_id"], ["id"], ondelete="SET NULL",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 6: partial unique index on (artist_id, external_post_id) for
|
||||||
|
# NULL-source Posts. The existing uq_post_source_external_id keeps
|
||||||
|
# guarding source-bound rows; NULL-source rows now dedupe within
|
||||||
|
# an artist.
|
||||||
|
op.execute(
|
||||||
|
"CREATE UNIQUE INDEX uq_post_artist_external_id_null_source "
|
||||||
|
"ON post (artist_id, external_post_id) "
|
||||||
|
"WHERE source_id IS NULL"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Step 7: retire sidecar synthetic Sources. NULL out the references
|
||||||
|
# FIRST (the new FK is SET NULL so CASCADE wouldn't fire anyway, but
|
||||||
|
# being explicit makes the intent clear). Then delete the synthetic
|
||||||
|
# source rows. Any DownloadEvent rows under synthetics CASCADE-die
|
||||||
|
# with the source — synthetics have enabled=false so there shouldn't
|
||||||
|
# be any in practice.
|
||||||
|
conn.execute(text("""
|
||||||
|
UPDATE post
|
||||||
|
SET source_id = NULL
|
||||||
|
WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%')
|
||||||
|
"""))
|
||||||
|
conn.execute(text("""
|
||||||
|
UPDATE image_provenance
|
||||||
|
SET source_id = NULL
|
||||||
|
WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%')
|
||||||
|
"""))
|
||||||
|
deleted = conn.execute(text(
|
||||||
|
"DELETE FROM source WHERE url LIKE 'sidecar:%' RETURNING id"
|
||||||
|
)).rowcount
|
||||||
|
print(f"alembic 0030: deleted {deleted} sidecar synthetic source rows")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
# Lossy migration — the deleted sidecar synthetics can't be
|
||||||
|
# restored from the orphan post.source_id / image_provenance.source_id
|
||||||
|
# values, and the partial unique index encodes a constraint that
|
||||||
|
# NULL-source Posts may now exist. No safe downgrade.
|
||||||
|
pass
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
"""source.backfill_runs_remaining: sticky deep-scan mode
|
||||||
|
|
||||||
|
Revision ID: 0031
|
||||||
|
Revises: 0030
|
||||||
|
Create Date: 2026-06-01
|
||||||
|
|
||||||
|
Tick vs backfill mode for subscription downloads. When
|
||||||
|
`backfill_runs_remaining > 0`, the next N download runs use
|
||||||
|
`skip: True` + 30-min timeout (walk full history). When 0, runs use
|
||||||
|
`skip: "exit:20"` + 14.5-min timeout (catch-up mode, exits early once
|
||||||
|
20 contiguous archived items are seen).
|
||||||
|
|
||||||
|
Operator-flagged 2026-06-01 (Knuxy run #38887): a creator with ~550
|
||||||
|
archived posts saturates the 870s catch-up timeout even when there is
|
||||||
|
no new content, because gallery-dl's default `skip: True` keeps walking.
|
||||||
|
Tick mode short-circuits that; backfill mode is the explicit opt-in for
|
||||||
|
deep history scans.
|
||||||
|
|
||||||
|
Default 0 (all existing subscriptions start in tick mode).
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0031"
|
||||||
|
down_revision: Union[str, None] = "0030"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"source",
|
||||||
|
sa.Column(
|
||||||
|
"backfill_runs_remaining",
|
||||||
|
sa.Integer,
|
||||||
|
nullable=False,
|
||||||
|
server_default="0",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("source", "backfill_runs_remaining")
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""source.error_type: surface ErrorType taxonomy in FailingSourcesCard
|
||||||
|
|
||||||
|
Revision ID: 0032
|
||||||
|
Revises: 0031
|
||||||
|
Create Date: 2026-06-02
|
||||||
|
|
||||||
|
Audit 2026-06-02: the backend computes 13 ErrorType categories (auth_error,
|
||||||
|
rate_limited, not_found, access_denied, validation_failed, etc.) and
|
||||||
|
stamps each one on DownloadEvent.metadata, but the Source row only carried
|
||||||
|
the free-text last_error. Operators couldn't bulk-triage failing sources
|
||||||
|
("all auth_error → rotate cookies, all rate_limited → just wait") without
|
||||||
|
opening Logs per row.
|
||||||
|
|
||||||
|
This column receives the last error_type from _update_source_health
|
||||||
|
and gets cleared on a successful run. Nullable + indexed so the failing-
|
||||||
|
sources rollup can filter/group cheaply.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0032"
|
||||||
|
down_revision: Union[str, None] = "0031"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"source",
|
||||||
|
sa.Column("error_type", sa.String(length=32), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_source_error_type", "source", ["error_type"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_index("ix_source_error_type", table_name="source")
|
||||||
|
op.drop_column("source", "error_type")
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
"""suggestion_threshold default 0.50 → 0.70
|
||||||
|
|
||||||
|
Revision ID: 0033
|
||||||
|
Revises: 0032
|
||||||
|
Create Date: 2026-06-02
|
||||||
|
|
||||||
|
Operator-flagged 2026-06-02 — the 0.50 default (set on 2026-06-01) is
|
||||||
|
too noisy in practice; raise to 0.70 for both suggestion categories.
|
||||||
|
|
||||||
|
Only conditionally updates singletons whose current value is still the
|
||||||
|
2026-06-01 default (0.50). Operators who deliberately tuned their row
|
||||||
|
to some other value (0.55, 0.65, 0.80, etc. via the Settings UI) keep
|
||||||
|
their pick — the migration only catches the unchanged-default case.
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0033"
|
||||||
|
down_revision: Union[str, None] = "0032"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.execute(
|
||||||
|
"UPDATE ml_settings "
|
||||||
|
"SET suggestion_threshold_character = 0.70 "
|
||||||
|
"WHERE id = 1 AND suggestion_threshold_character = 0.50"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"UPDATE ml_settings "
|
||||||
|
"SET suggestion_threshold_general = 0.70 "
|
||||||
|
"WHERE id = 1 AND suggestion_threshold_general = 0.50"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.execute(
|
||||||
|
"UPDATE ml_settings "
|
||||||
|
"SET suggestion_threshold_character = 0.50 "
|
||||||
|
"WHERE id = 1 AND suggestion_threshold_character = 0.70"
|
||||||
|
)
|
||||||
|
op.execute(
|
||||||
|
"UPDATE ml_settings "
|
||||||
|
"SET suggestion_threshold_general = 0.50 "
|
||||||
|
"WHERE id = 1 AND suggestion_threshold_general = 0.70"
|
||||||
|
)
|
||||||
@@ -43,7 +43,6 @@ async def scroll():
|
|||||||
"height": i.height,
|
"height": i.height,
|
||||||
"created_at": i.created_at.isoformat(),
|
"created_at": i.created_at.isoformat(),
|
||||||
"posted_at": i.posted_at.isoformat() if i.posted_at else None,
|
"posted_at": i.posted_at.isoformat() if i.posted_at else None,
|
||||||
"effective_date": i.effective_date.isoformat(),
|
|
||||||
"thumbnail_url": i.thumbnail_url,
|
"thumbnail_url": i.thumbnail_url,
|
||||||
"artist": i.artist,
|
"artist": i.artist,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,10 +35,26 @@ async def trigger_scan():
|
|||||||
@import_admin_bp.route("/status", methods=["GET"])
|
@import_admin_bp.route("/status", methods=["GET"])
|
||||||
async def status():
|
async def status():
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
|
# Active batch = running batch that still has outstanding work.
|
||||||
|
# Plain "most recent running" picks freshly-created scans that
|
||||||
|
# enqueued zero new files and hides the older batch that's
|
||||||
|
# actually being processed. Mirrors the EXISTS predicate
|
||||||
|
# /api/system/stats already uses (api/settings.py:145-160).
|
||||||
|
# Audit 2026-06-02 — /api/import/status and /api/system/stats
|
||||||
|
# used to disagree on the active-batch predicate; the UI banner
|
||||||
|
# said "Scanning…" indefinitely while the stats card said idle.
|
||||||
active = (
|
active = (
|
||||||
await session.execute(
|
await session.execute(
|
||||||
select(ImportBatch)
|
select(ImportBatch)
|
||||||
.where(ImportBatch.status == "running")
|
.where(
|
||||||
|
ImportBatch.status == "running",
|
||||||
|
select(ImportTask.id)
|
||||||
|
.where(
|
||||||
|
ImportTask.batch_id == ImportBatch.id,
|
||||||
|
ImportTask.status.in_(["pending", "queued", "processing"]),
|
||||||
|
)
|
||||||
|
.exists(),
|
||||||
|
)
|
||||||
.order_by(ImportBatch.started_at.desc())
|
.order_by(ImportBatch.started_at.desc())
|
||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,9 +9,7 @@ ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
|||||||
|
|
||||||
|
|
||||||
_EDITABLE = (
|
_EDITABLE = (
|
||||||
"suggestion_threshold_artist",
|
|
||||||
"suggestion_threshold_character",
|
"suggestion_threshold_character",
|
||||||
"suggestion_threshold_copyright",
|
|
||||||
"suggestion_threshold_general",
|
"suggestion_threshold_general",
|
||||||
"centroid_similarity_threshold",
|
"centroid_similarity_threshold",
|
||||||
"min_reference_images",
|
"min_reference_images",
|
||||||
@@ -28,9 +26,7 @@ async def get_settings():
|
|||||||
).scalar_one()
|
).scalar_one()
|
||||||
return jsonify(
|
return jsonify(
|
||||||
{
|
{
|
||||||
"suggestion_threshold_artist": s.suggestion_threshold_artist,
|
|
||||||
"suggestion_threshold_character": s.suggestion_threshold_character,
|
"suggestion_threshold_character": s.suggestion_threshold_character,
|
||||||
"suggestion_threshold_copyright": s.suggestion_threshold_copyright,
|
|
||||||
"suggestion_threshold_general": s.suggestion_threshold_general,
|
"suggestion_threshold_general": s.suggestion_threshold_general,
|
||||||
"centroid_similarity_threshold": s.centroid_similarity_threshold,
|
"centroid_similarity_threshold": s.centroid_similarity_threshold,
|
||||||
"min_reference_images": s.min_reference_images,
|
"min_reference_images": s.min_reference_images,
|
||||||
|
|||||||
@@ -120,6 +120,31 @@ async def delete_source(source_id: int):
|
|||||||
return "", 204
|
return "", 204
|
||||||
|
|
||||||
|
|
||||||
|
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
||||||
|
async def set_backfill(source_id: int):
|
||||||
|
"""Plan #544: arm a source for backfill mode for the next N download
|
||||||
|
runs. Body: `{"runs": int}` (1..10, default 3). Returns the updated
|
||||||
|
source dict. While backfill_runs_remaining > 0, downloads use
|
||||||
|
gallery-dl's full-walk config (skip: True + 30-min timeout) instead
|
||||||
|
of the catch-up default (skip: "exit:20" + 14.5-min timeout)."""
|
||||||
|
payload = await request.get_json(silent=True) or {}
|
||||||
|
runs = payload.get("runs", 3)
|
||||||
|
try:
|
||||||
|
runs = int(runs)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return _bad("invalid_runs", detail="runs must be an integer")
|
||||||
|
async with get_session() as session:
|
||||||
|
try:
|
||||||
|
record = await SourceService(session).set_backfill_runs(
|
||||||
|
source_id, runs,
|
||||||
|
)
|
||||||
|
except LookupError:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
except ValueError as exc:
|
||||||
|
return _bad("invalid_runs", detail=str(exc))
|
||||||
|
return jsonify(record.to_dict())
|
||||||
|
|
||||||
|
|
||||||
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
||||||
async def check_source(source_id: int):
|
async def check_source(source_id: int):
|
||||||
"""FC-3c: enqueue a download for this source.
|
"""FC-3c: enqueue a download for this source.
|
||||||
|
|||||||
@@ -245,6 +245,12 @@ async def merge_tag(source_id: int):
|
|||||||
from ..tasks.ml import apply_allowlist_tags
|
from ..tasks.ml import apply_allowlist_tags
|
||||||
|
|
||||||
apply_allowlist_tags.delay(tag_id=result.target_id)
|
apply_allowlist_tags.delay(tag_id=result.target_id)
|
||||||
|
# Tag merge invalidates the target's centroid (the merged-in source
|
||||||
|
# tag's images now contribute to it). Daily list_drifted catches it
|
||||||
|
# within 24h, but eager recompute closes the suggestion-quality dip
|
||||||
|
# in the meantime. Audit 2026-06-02.
|
||||||
|
from ..tasks.ml import recompute_centroid
|
||||||
|
recompute_centroid.delay(result.target_id)
|
||||||
return jsonify(
|
return jsonify(
|
||||||
{
|
{
|
||||||
"target": {
|
"target": {
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
"""Thumbnail admin API: backfill trigger."""
|
"""Thumbnail admin API: backfill trigger."""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
|
||||||
from quart import Blueprint, jsonify
|
from quart import Blueprint, jsonify
|
||||||
|
|
||||||
thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
||||||
@@ -7,7 +9,20 @@ thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
|||||||
|
|
||||||
@thumbnails_bp.route("/backfill", methods=["POST"])
|
@thumbnails_bp.route("/backfill", methods=["POST"])
|
||||||
async def trigger_backfill():
|
async def trigger_backfill():
|
||||||
from ..tasks.thumbnail import backfill_thumbnails
|
"""Run the backfill scan synchronously, return the counts. The actual
|
||||||
|
thumbnail generation work is still off-loaded to the thumbnail Celery
|
||||||
|
queue via `generate_thumbnail.delay()` per missing row — so this
|
||||||
|
handler is fast even on a 100k-image library (a scan is just SELECT
|
||||||
|
id, thumbnail_path + a file.stat() per row, no heavy work).
|
||||||
|
|
||||||
r = backfill_thumbnails.delay()
|
Operator-flagged 2026-06-01: the previous fire-and-forget shape
|
||||||
return jsonify({"celery_task_id": r.id}), 202
|
returned `{celery_task_id}` only, so the admin UI had no idea whether
|
||||||
|
backfill found 0 or 5000 candidates — \"found nothing\" was
|
||||||
|
indistinguishable from \"the worker isn't picking up the task.\""""
|
||||||
|
from ..tasks.thumbnail import _run_backfill_scan
|
||||||
|
|
||||||
|
# Sync scan inside an executor so we don't block the event loop.
|
||||||
|
counts = await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None, _run_backfill_scan,
|
||||||
|
)
|
||||||
|
return jsonify(counts), 200
|
||||||
|
|||||||
@@ -105,6 +105,41 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.backup.prune_backups",
|
"task": "backend.app.tasks.backup.prune_backups",
|
||||||
"schedule": 86400.0, # daily
|
"schedule": 86400.0, # daily
|
||||||
},
|
},
|
||||||
|
# Audit 2026-06-02 — three new per-entity recovery sweeps.
|
||||||
|
# Each runs every 5 min like the other recover_stalled_*
|
||||||
|
# sweeps; each is a no-op when nothing is stuck.
|
||||||
|
"recover-stalled-backup-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_stalled_backup_runs",
|
||||||
|
"schedule": 300.0,
|
||||||
|
},
|
||||||
|
"recover-stalled-library-audit-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_stalled_library_audit_runs",
|
||||||
|
"schedule": 300.0,
|
||||||
|
},
|
||||||
|
"recover-stalled-import-batches": {
|
||||||
|
"task": "backend.app.tasks.maintenance.recover_stalled_import_batches",
|
||||||
|
"schedule": 300.0,
|
||||||
|
},
|
||||||
|
# Audit 2026-06-02 — daily retention for two entities
|
||||||
|
# whose terminal rows otherwise accumulate forever.
|
||||||
|
"prune-library-audit-runs": {
|
||||||
|
"task": "backend.app.tasks.maintenance.prune_library_audit_runs",
|
||||||
|
"schedule": 86400.0,
|
||||||
|
},
|
||||||
|
"prune-import-batches": {
|
||||||
|
"task": "backend.app.tasks.maintenance.prune_import_batches",
|
||||||
|
"schedule": 86400.0,
|
||||||
|
},
|
||||||
|
# Audit 2026-06-02 — backfill_thumbnails's docstring claimed
|
||||||
|
# "periodic Beat" but the entry was never registered, so the
|
||||||
|
# library got no self-healing thumbnail repair; only the
|
||||||
|
# manual admin-UI button fired it. Daily cadence is gentle
|
||||||
|
# (the task is idempotent and only enqueues regen for rows
|
||||||
|
# whose stored thumbnails are missing or corrupt).
|
||||||
|
"backfill-thumbnails-daily": {
|
||||||
|
"task": "backend.app.tasks.thumbnail.backfill_thumbnails",
|
||||||
|
"schedule": 86400.0,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
timezone="UTC",
|
timezone="UTC",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -54,7 +54,14 @@ _INT32_MIN = -2_147_483_648
|
|||||||
|
|
||||||
def _queue_for(task) -> str:
|
def _queue_for(task) -> str:
|
||||||
"""Reverse the task→queue routing from celery_app.task_routes.
|
"""Reverse the task→queue routing from celery_app.task_routes.
|
||||||
Keep in sync if task_routes is reordered."""
|
Keep in sync if task_routes is reordered.
|
||||||
|
|
||||||
|
Audit 2026-06-02: backup/admin/library_audit prefixes were
|
||||||
|
missing here even though task_routes sent all three to
|
||||||
|
'maintenance'. The TaskRun.queue column then lied for those
|
||||||
|
rows (claimed 'default') so per-queue dashboard filters and
|
||||||
|
per-queue threshold overrides silently missed them.
|
||||||
|
"""
|
||||||
name = getattr(task, "name", "") or ""
|
name = getattr(task, "name", "") or ""
|
||||||
if name.startswith("backend.app.tasks.import_file."):
|
if name.startswith("backend.app.tasks.import_file."):
|
||||||
return "import"
|
return "import"
|
||||||
@@ -66,7 +73,12 @@ def _queue_for(task) -> str:
|
|||||||
return "download"
|
return "download"
|
||||||
if name.startswith("backend.app.tasks.scan."):
|
if name.startswith("backend.app.tasks.scan."):
|
||||||
return "scan"
|
return "scan"
|
||||||
if name.startswith("backend.app.tasks.maintenance."):
|
if name.startswith((
|
||||||
|
"backend.app.tasks.maintenance.",
|
||||||
|
"backend.app.tasks.backup.",
|
||||||
|
"backend.app.tasks.admin.",
|
||||||
|
"backend.app.tasks.library_audit.",
|
||||||
|
)):
|
||||||
return "maintenance"
|
return "maintenance"
|
||||||
return "default"
|
return "default"
|
||||||
|
|
||||||
|
|||||||
@@ -34,8 +34,12 @@ class ImageProvenance(Base):
|
|||||||
post_id: Mapped[int] = mapped_column(
|
post_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
)
|
)
|
||||||
source_id: Mapped[int] = mapped_column(
|
# Nullable since alembic 0030 — provenance rows for filesystem-imported
|
||||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
# content with no subscription have NULL source_id. FK ondelete SET
|
||||||
|
# NULL so deleting a Source detaches its provenance rows instead of
|
||||||
|
# destroying the linkage between image and post.
|
||||||
|
source_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
)
|
)
|
||||||
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
captured_at: Mapped[datetime] = mapped_column(
|
captured_at: Mapped[datetime] = mapped_column(
|
||||||
|
|||||||
@@ -15,17 +15,15 @@ class MLSettings(Base):
|
|||||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
suggestion_threshold_artist: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.30
|
|
||||||
)
|
|
||||||
suggestion_threshold_character: Mapped[float] = mapped_column(
|
suggestion_threshold_character: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.50
|
Float, nullable=False, default=0.70
|
||||||
)
|
|
||||||
suggestion_threshold_copyright: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.50
|
|
||||||
)
|
)
|
||||||
|
# Default raised 0.50 → 0.70 on 2026-06-02 — operator-flagged 0.50
|
||||||
|
# surfaced too many low-confidence picks; 0.70 keeps the rail
|
||||||
|
# signal-rich while still surfacing more than the original 0.95
|
||||||
|
# which hid almost everything. Operator-tunable via Settings → ML.
|
||||||
suggestion_threshold_general: Mapped[float] = mapped_column(
|
suggestion_threshold_general: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.95
|
Float, nullable=False, default=0.70
|
||||||
)
|
)
|
||||||
centroid_similarity_threshold: Mapped[float] = mapped_column(
|
centroid_similarity_threshold: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.55
|
Float, nullable=False, default=0.55
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
"""Post — provenance anchor for content downloaded from a Source.
|
"""Post — provenance anchor for one creator post (may contain many images).
|
||||||
|
|
||||||
A Post is one creator post; it may contain many images/videos.
|
`source_id` is nullable since alembic 0030 — filesystem-imported posts
|
||||||
|
with no live subscription have NULL source_id. `artist_id` is the
|
||||||
|
denormalized always-present link to the creator (added in 0030 so
|
||||||
|
artist-filter queries don't depend on the Source detour).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
@@ -14,12 +17,25 @@ from .base import Base
|
|||||||
class Post(Base):
|
class Post(Base):
|
||||||
__tablename__ = "post"
|
__tablename__ = "post"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
# Source-bound dedup. Postgres treats NULL != NULL so rows
|
||||||
|
# with source_id IS NULL aren't deduped by this constraint;
|
||||||
|
# the partial unique index `uq_post_artist_external_id_null_source`
|
||||||
|
# (created in alembic 0030) covers that case via
|
||||||
|
# (artist_id, external_post_id).
|
||||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
source_id: Mapped[int] = mapped_column(
|
source_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("source.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
# Denormalized; always equals source.artist_id when source_id is set
|
||||||
|
# (the importer is responsible for keeping them consistent on insert).
|
||||||
|
# Filter queries (artist detail, artist-scoped posts feed) use this
|
||||||
|
# directly instead of joining through Source.
|
||||||
|
artist_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("artist.id", ondelete="CASCADE"),
|
||||||
|
nullable=False, index=True,
|
||||||
)
|
)
|
||||||
external_post_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
external_post_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
post_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
post_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|||||||
@@ -26,7 +26,21 @@ class Source(Base):
|
|||||||
|
|
||||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
# alembic 0032: last ErrorType category (auth_error, rate_limited,
|
||||||
|
# not_found, ...). Lets FailingSourcesCard surface the taxonomy as
|
||||||
|
# a colored chip so operators can bulk-triage by error class. Set
|
||||||
|
# by _update_source_health alongside last_error; cleared on 'ok'.
|
||||||
|
error_type: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||||
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
|
||||||
|
# alembic 0031: sticky deep-scan budget. When > 0, the next N download
|
||||||
|
# runs use gallery-dl's full-walk config (skip: True + 1800s timeout);
|
||||||
|
# when 0, runs use tick mode (skip: "exit:20" + 870s, exits early once
|
||||||
|
# 20 contiguous archived items are seen). Auto-decrements per run, with
|
||||||
|
# an auto-reset to 0 on clean exit + zero downloads (queue drained).
|
||||||
|
backfill_runs_remaining: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=0, server_default="0",
|
||||||
|
)
|
||||||
|
|
||||||
artist = relationship("Artist", back_populates="sources")
|
artist = relationship("Artist", back_populates="sources")
|
||||||
|
|||||||
@@ -58,13 +58,17 @@ class ArtistService:
|
|||||||
)
|
)
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
|
# Posts under this artist that have at least one image attached.
|
||||||
|
# Use Post.artist_id (alembic 0030) for the artist filter; keep
|
||||||
|
# the ImageProvenance JOIN so date bounds reflect only image-
|
||||||
|
# bearing posts (matches the original semantic). NULL-source
|
||||||
|
# posts now surface too.
|
||||||
date_row = (
|
date_row = (
|
||||||
await self.session.execute(
|
await self.session.execute(
|
||||||
select(func.min(Post.post_date), func.max(Post.post_date))
|
select(func.min(Post.post_date), func.max(Post.post_date))
|
||||||
.select_from(Post)
|
.select_from(Post)
|
||||||
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
||||||
.join(Source, Source.id == ImageProvenance.source_id)
|
.where(Post.artist_id == aid)
|
||||||
.where(Source.artist_id == aid)
|
|
||||||
)
|
)
|
||||||
).first()
|
).first()
|
||||||
dmin, dmax = date_row if date_row else (None, None)
|
dmin, dmax = date_row if date_row else (None, None)
|
||||||
@@ -98,14 +102,14 @@ class ArtistService:
|
|||||||
)
|
)
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
|
# Same Post.artist_id direct filter — counts NULL-source posts too.
|
||||||
month = func.date_trunc("month", Post.post_date).label("m")
|
month = func.date_trunc("month", Post.post_date).label("m")
|
||||||
activity = (
|
activity = (
|
||||||
await self.session.execute(
|
await self.session.execute(
|
||||||
select(month, func.count(func.distinct(ImageProvenance.image_record_id)))
|
select(month, func.count(func.distinct(ImageProvenance.image_record_id)))
|
||||||
.select_from(Post)
|
.select_from(Post)
|
||||||
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
.join(ImageProvenance, ImageProvenance.post_id == Post.id)
|
||||||
.join(Source, Source.id == ImageProvenance.source_id)
|
.where(and_(Post.artist_id == aid, Post.post_date.isnot(None)))
|
||||||
.where(and_(Source.artist_id == aid, Post.post_date.isnot(None)))
|
|
||||||
.group_by(month)
|
.group_by(month)
|
||||||
.order_by(month)
|
.order_by(month)
|
||||||
)
|
)
|
||||||
@@ -114,9 +118,7 @@ class ArtistService:
|
|||||||
post_count = (
|
post_count = (
|
||||||
await self.session.execute(
|
await self.session.execute(
|
||||||
select(func.count(func.distinct(Post.id)))
|
select(func.count(func.distinct(Post.id)))
|
||||||
.select_from(Post)
|
.where(Post.artist_id == aid)
|
||||||
.join(Source, Source.id == Post.source_id)
|
|
||||||
.where(Source.artist_id == aid)
|
|
||||||
)
|
)
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
@@ -206,27 +208,32 @@ class ArtistService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def find_or_create(self, name: str) -> tuple[Artist, bool]:
|
async def find_or_create(self, name: str) -> tuple[Artist, bool]:
|
||||||
"""Return (artist, created). Slug-keyed; idempotent under races."""
|
"""Return (artist, created). Slug-keyed; idempotent under races.
|
||||||
|
|
||||||
|
Audit 2026-06-02: switched from session.rollback() to a
|
||||||
|
begin_nested savepoint + IntegrityError recovery so a lost
|
||||||
|
race doesn't unwind the calling request's surrounding work.
|
||||||
|
Mirrors importer._get_or_create.
|
||||||
|
"""
|
||||||
cleaned = (name or "").strip()
|
cleaned = (name or "").strip()
|
||||||
if not cleaned:
|
if not cleaned:
|
||||||
raise ValueError("artist name must not be empty")
|
raise ValueError("artist name must not be empty")
|
||||||
slug = slugify(cleaned)
|
slug = slugify(cleaned)
|
||||||
|
|
||||||
existing = (await self.session.execute(
|
select_existing = select(Artist).where(Artist.slug == slug)
|
||||||
select(Artist).where(Artist.slug == slug)
|
existing = (await self.session.execute(select_existing)).scalar_one_or_none()
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return existing, False
|
return existing, False
|
||||||
|
|
||||||
artist = Artist(name=cleaned, slug=slug)
|
sp = await self.session.begin_nested()
|
||||||
self.session.add(artist)
|
|
||||||
try:
|
try:
|
||||||
|
artist = Artist(name=cleaned, slug=slug)
|
||||||
|
self.session.add(artist)
|
||||||
await self.session.flush()
|
await self.session.flush()
|
||||||
|
await sp.commit()
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
await self.session.rollback()
|
await sp.rollback()
|
||||||
existing = (await self.session.execute(
|
existing = (await self.session.execute(select_existing)).scalar_one()
|
||||||
select(Artist).where(Artist.slug == slug)
|
|
||||||
)).scalar_one()
|
|
||||||
return existing, False
|
return existing, False
|
||||||
await self.session.commit()
|
await self.session.commit()
|
||||||
return artist, True
|
return artist, True
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ the one-and-done GS/IR migration tooling.)
|
|||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -187,10 +187,14 @@ def unlink_image_files(
|
|||||||
out["thumbnail"] = True
|
out["thumbnail"] = True
|
||||||
except OSError:
|
except OSError:
|
||||||
out["thumbnail"] = False
|
out["thumbnail"] = False
|
||||||
# Convention thumbs dir — try all extensions; missing OK.
|
# Convention thumbs dir — try both extensions thumbnailer writes
|
||||||
|
# (.jpg for opaque, .png for alpha). `.webp` used to be in this
|
||||||
|
# tuple but the thumbnailer never writes it (operator-flagged in
|
||||||
|
# the 2026-06-02 audit) — keep the tuple aligned with what
|
||||||
|
# actually lands on disk.
|
||||||
if image.sha256:
|
if image.sha256:
|
||||||
bucket = image.sha256[:3]
|
bucket = image.sha256[:3]
|
||||||
for ext in ("jpg", "png", "webp"):
|
for ext in ("jpg", "png"):
|
||||||
try:
|
try:
|
||||||
(images_root / "thumbs" / bucket / f"{image.sha256}.{ext}").unlink(
|
(images_root / "thumbs" / bucket / f"{image.sha256}.{ext}").unlink(
|
||||||
missing_ok=True,
|
missing_ok=True,
|
||||||
@@ -354,18 +358,35 @@ def prune_unused_tags(session: Session, *, dry_run: bool = False) -> dict:
|
|||||||
Returns:
|
Returns:
|
||||||
dry_run=True: {"count": N, "sample_names": [first 50]}
|
dry_run=True: {"count": N, "sample_names": [first 50]}
|
||||||
dry_run=False: {"deleted": N, "sample_names": [first 50]}
|
dry_run=False: {"deleted": N, "sample_names": [first 50]}
|
||||||
|
|
||||||
|
Implementation note: the previous SELECT-ids → DELETE-WHERE-IN
|
||||||
|
pattern was vulnerable to the psycopg 65535-parameter ceiling on
|
||||||
|
libraries with tag explosions. The live delete now runs a single
|
||||||
|
DELETE with the same NOT-IN predicate find_unused_tags uses, so
|
||||||
|
the row count scales without binding every id as a parameter.
|
||||||
|
Audit 2026-06-02.
|
||||||
"""
|
"""
|
||||||
unused = find_unused_tags(session)
|
sample_rows = find_unused_tags(session, limit=50)
|
||||||
sample = [t.name for t in unused[:50]]
|
sample = [t.name for t in sample_rows]
|
||||||
|
used_via_image_tag = select(image_tag.c.tag_id).distinct()
|
||||||
|
used_via_series = select(SeriesPage.series_tag_id).where(
|
||||||
|
SeriesPage.series_tag_id.is_not(None)
|
||||||
|
).distinct()
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return {"count": len(unused), "sample_names": sample}
|
count = session.execute(
|
||||||
ids = [t.id for t in unused]
|
select(func.count())
|
||||||
if ids:
|
.select_from(Tag)
|
||||||
session.execute(
|
.where(Tag.id.not_in(used_via_image_tag))
|
||||||
Tag.__table__.delete().where(Tag.id.in_(ids))
|
.where(Tag.id.not_in(used_via_series))
|
||||||
)
|
).scalar_one()
|
||||||
session.commit()
|
return {"count": count, "sample_names": sample}
|
||||||
return {"deleted": len(ids), "sample_names": sample}
|
result = session.execute(
|
||||||
|
Tag.__table__.delete()
|
||||||
|
.where(Tag.id.not_in(used_via_image_tag))
|
||||||
|
.where(Tag.id.not_in(used_via_series))
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return {"deleted": result.rowcount or 0, "sample_names": sample}
|
||||||
|
|
||||||
|
|
||||||
# Legacy tags FC no longer uses, in two shapes:
|
# Legacy tags FC no longer uses, in two shapes:
|
||||||
@@ -500,6 +521,9 @@ class ConfirmTokenMismatch(Exception):
|
|||||||
_VALID_RULES = ("transparency", "single_color")
|
_VALID_RULES = ("transparency", "single_color")
|
||||||
|
|
||||||
|
|
||||||
|
_AUDIT_GUARD_THRESHOLD_MINUTES = 135 # matches LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES
|
||||||
|
|
||||||
|
|
||||||
def start_audit_run(
|
def start_audit_run(
|
||||||
session: Session, *, rule: str, params: dict[str, Any],
|
session: Session, *, rule: str, params: dict[str, Any],
|
||||||
) -> int:
|
) -> int:
|
||||||
@@ -507,11 +531,21 @@ def start_audit_run(
|
|||||||
scan_library_for_rule Celery task. Returns the new audit_id.
|
scan_library_for_rule Celery task. Returns the new audit_id.
|
||||||
|
|
||||||
Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run
|
Concurrent-runs guard: raises AuditAlreadyRunning if any audit_run
|
||||||
has status='running'. Operator must cancel or wait."""
|
has status='running' AND started recently. Audit 2026-06-02 made
|
||||||
|
the guard age-aware: a SIGKILL'd run leaves a row in 'running'
|
||||||
|
that the recovery sweep flips on its next pass (~5 min), but a
|
||||||
|
fresh start_audit_run between the SIGKILL and the sweep would
|
||||||
|
previously block forever. Past the threshold, treat the running
|
||||||
|
row as stale and let the sweep clean it up — the new run still
|
||||||
|
gets to start.
|
||||||
|
"""
|
||||||
if rule not in _VALID_RULES:
|
if rule not in _VALID_RULES:
|
||||||
raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}")
|
raise ValueError(f"unknown rule {rule!r}; expected one of {_VALID_RULES}")
|
||||||
|
cutoff = datetime.now(UTC) - timedelta(minutes=_AUDIT_GUARD_THRESHOLD_MINUTES)
|
||||||
existing = session.execute(
|
existing = session.execute(
|
||||||
select(LibraryAuditRun.id).where(LibraryAuditRun.status == "running")
|
select(LibraryAuditRun.id)
|
||||||
|
.where(LibraryAuditRun.status == "running")
|
||||||
|
.where(LibraryAuditRun.started_at >= cutoff)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
raise AuditAlreadyRunning(existing)
|
raise AuditAlreadyRunning(existing)
|
||||||
|
|||||||
@@ -1,40 +1,82 @@
|
|||||||
"""Fernet-based encryption for credential blobs.
|
"""Fernet-based encryption for credential blobs.
|
||||||
|
|
||||||
The key is a single 32-byte value (urlsafe-base64-encoded; what
|
The key is a single 32-byte value (urlsafe-base64-encoded; what
|
||||||
Fernet.generate_key produces) stored at a fixed path inside the
|
Fernet.generate_key produces) stored at /images/secrets/credential_key.b64
|
||||||
images/data root. Created on first boot if absent; mode 0600. No KDF
|
(mode 0600, parent dir 0700). The 2026-06-02 audit caught a silent
|
||||||
needed — the file contents are already maximum-entropy random bytes.
|
key-regeneration path: on a partial disaster restore where the DB was
|
||||||
|
restored but the secrets dir was lost, the old `_load_or_create_key`
|
||||||
|
would mint a fresh key with no log, producing a working-looking system
|
||||||
|
where every authenticated download failed AUTH_ERROR until the operator
|
||||||
|
re-uploaded every credential by hand. Now the constructor refuses to
|
||||||
|
auto-generate unless either:
|
||||||
|
|
||||||
Operator backup procedure must include this file alongside the rest
|
* the caller explicitly passes `bootstrap_ok=True` (tests, scripts), or
|
||||||
of /images/ — losing it makes existing encrypted_blob rows
|
* the env var `CURATOR_BOOTSTRAP_NEW_KEY=1` is set (operator opt-in
|
||||||
undecryptable (recovery = delete the rows and re-upload).
|
during first-time setup).
|
||||||
|
|
||||||
|
Otherwise it raises `MissingCredentialKey` so the app fails fast at
|
||||||
|
startup and the operator can restore the key file from backup.
|
||||||
|
|
||||||
|
Operator backup procedure must include /images/secrets/ alongside the
|
||||||
|
rest of /images/ — losing the key file makes existing encrypted_blob
|
||||||
|
rows undecryptable (recovery = delete the rows and re-upload).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from cryptography.fernet import Fernet, InvalidToken
|
from cryptography.fernet import Fernet, InvalidToken
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_BOOTSTRAP_ENV_VAR = "CURATOR_BOOTSTRAP_NEW_KEY"
|
||||||
|
|
||||||
|
|
||||||
class InvalidCredentialBlob(Exception):
|
class InvalidCredentialBlob(Exception):
|
||||||
"""Raised when decryption fails (wrong key, tampered blob, …)."""
|
"""Raised when decryption fails (wrong key, tampered blob, …)."""
|
||||||
|
|
||||||
|
|
||||||
|
class MissingCredentialKey(Exception):
|
||||||
|
"""The Fernet key file is missing AND the caller hasn't opted in to
|
||||||
|
generating a new one. Audit 2026-06-02: prevents silent key
|
||||||
|
regeneration on partial DB-restored / secrets-lost deployments.
|
||||||
|
Set CURATOR_BOOTSTRAP_NEW_KEY=1 for first-time setup, or restore the
|
||||||
|
key file from backup."""
|
||||||
|
|
||||||
|
|
||||||
class CredentialCrypto:
|
class CredentialCrypto:
|
||||||
"""Fernet encrypt/decrypt with an on-disk key file.
|
"""Fernet encrypt/decrypt with an on-disk key file.
|
||||||
|
|
||||||
Instantiate with a path; the file is created on first access and
|
Instantiate with a path; the file is loaded if present, or created
|
||||||
reused thereafter. Tests pass a tmp_path; production calls with
|
if absent AND the caller has opted in (bootstrap_ok=True or
|
||||||
|
CURATOR_BOOTSTRAP_NEW_KEY=1 env var). Production sites:
|
||||||
`IMAGES_ROOT / "secrets" / "credential_key.b64"`.
|
`IMAGES_ROOT / "secrets" / "credential_key.b64"`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, key_path: Path):
|
def __init__(self, key_path: Path, *, bootstrap_ok: bool | None = None):
|
||||||
self._key_path = Path(key_path)
|
self._key_path = Path(key_path)
|
||||||
self._fernet = Fernet(self._load_or_create_key())
|
if bootstrap_ok is None:
|
||||||
|
bootstrap_ok = os.environ.get(_BOOTSTRAP_ENV_VAR) == "1"
|
||||||
|
self._fernet = Fernet(self._load_or_create_key(bootstrap_ok))
|
||||||
|
|
||||||
def _load_or_create_key(self) -> bytes:
|
def _load_or_create_key(self, bootstrap_ok: bool) -> bytes:
|
||||||
if self._key_path.exists():
|
if self._key_path.exists():
|
||||||
return self._key_path.read_bytes()
|
return self._key_path.read_bytes()
|
||||||
|
if not bootstrap_ok:
|
||||||
|
raise MissingCredentialKey(
|
||||||
|
f"Fernet key file not found at {self._key_path}. "
|
||||||
|
f"For first-time setup, set {_BOOTSTRAP_ENV_VAR}=1. "
|
||||||
|
f"If this is a restored instance, restore the key file "
|
||||||
|
f"from backup — generating a new one would make every "
|
||||||
|
f"existing Credential row undecryptable."
|
||||||
|
)
|
||||||
|
log.warning(
|
||||||
|
"Generating NEW Fernet credential key at %s. Any existing "
|
||||||
|
"encrypted_blob rows in the DB will be undecryptable — "
|
||||||
|
"re-upload each credential after this completes.",
|
||||||
|
self._key_path,
|
||||||
|
)
|
||||||
parent = self._key_path.parent
|
parent = self._key_path.parent
|
||||||
parent.mkdir(parents=True, exist_ok=True)
|
parent.mkdir(parents=True, exist_ok=True)
|
||||||
os.chmod(parent, 0o700)
|
os.chmod(parent, 0o700)
|
||||||
|
|||||||
@@ -25,9 +25,17 @@ from sqlalchemy.orm import joinedload
|
|||||||
|
|
||||||
from ..models import Artist, DownloadEvent, Source
|
from ..models import Artist, DownloadEvent, Source
|
||||||
from .credential_service import CredentialService
|
from .credential_service import CredentialService
|
||||||
from .gallery_dl import GalleryDLService, SourceConfig
|
from .gallery_dl import (
|
||||||
|
BACKFILL_SKIP_VALUE,
|
||||||
|
BACKFILL_TIMEOUT_SECONDS,
|
||||||
|
TICK_SKIP_VALUE,
|
||||||
|
ErrorType,
|
||||||
|
GalleryDLService,
|
||||||
|
SourceConfig,
|
||||||
|
)
|
||||||
from .importer import Importer
|
from .importer import Importer
|
||||||
from .patreon_resolver import resolve_campaign_id
|
from .patreon_resolver import resolve_campaign_id
|
||||||
|
from .platforms import auth_type_for
|
||||||
from .scheduler_service import set_platform_cooldown
|
from .scheduler_service import set_platform_cooldown
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
@@ -84,6 +92,18 @@ class DownloadService:
|
|||||||
ctx = setup
|
ctx = setup
|
||||||
|
|
||||||
source_config = SourceConfig.from_dict(ctx["config_overrides"] or {})
|
source_config = SourceConfig.from_dict(ctx["config_overrides"] or {})
|
||||||
|
# alembic 0031 / plan #544: derive skip_value + timeout from the
|
||||||
|
# source's backfill_runs_remaining counter. When > 0, walk the full
|
||||||
|
# post history (skip: True + 1800s); when 0, exit gallery-dl after
|
||||||
|
# 20 contiguous archived items (skip: "exit:20" + the default
|
||||||
|
# 870s). Operator sets backfill via POST /api/sources/{id}/backfill.
|
||||||
|
backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
|
||||||
|
if backfill_remaining > 0:
|
||||||
|
skip_value: bool | str = BACKFILL_SKIP_VALUE
|
||||||
|
source_config.timeout = BACKFILL_TIMEOUT_SECONDS
|
||||||
|
else:
|
||||||
|
skip_value = TICK_SKIP_VALUE
|
||||||
|
|
||||||
effective_url = _effective_url(
|
effective_url = _effective_url(
|
||||||
ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
|
ctx["platform"], ctx["url"], ctx["config_overrides"] or {}
|
||||||
)
|
)
|
||||||
@@ -95,6 +115,7 @@ class DownloadService:
|
|||||||
source_config=source_config,
|
source_config=source_config,
|
||||||
cookies_path=ctx["cookies_path"],
|
cookies_path=ctx["cookies_path"],
|
||||||
auth_token=ctx["auth_token"],
|
auth_token=ctx["auth_token"],
|
||||||
|
skip_value=skip_value,
|
||||||
)
|
)
|
||||||
|
|
||||||
resolved_campaign_id: str | None = None
|
resolved_campaign_id: str | None = None
|
||||||
@@ -121,6 +142,7 @@ class DownloadService:
|
|||||||
source_config=source_config,
|
source_config=source_config,
|
||||||
cookies_path=ctx["cookies_path"],
|
cookies_path=ctx["cookies_path"],
|
||||||
auth_token=ctx["auth_token"],
|
auth_token=ctx["auth_token"],
|
||||||
|
skip_value=skip_value,
|
||||||
)
|
)
|
||||||
|
|
||||||
return await self._phase3_persist(
|
return await self._phase3_persist(
|
||||||
@@ -156,6 +178,13 @@ class DownloadService:
|
|||||||
return {"status": "in_flight", "event_id": existing.id}
|
return {"status": "in_flight", "event_id": existing.id}
|
||||||
if existing and existing.status == "pending":
|
if existing and existing.status == "pending":
|
||||||
existing.status = "running"
|
existing.status = "running"
|
||||||
|
# Reset started_at on the pending→running transition so the
|
||||||
|
# recovery sweep (DOWNLOAD_STALL_THRESHOLD_MINUTES, 30 min)
|
||||||
|
# measures from real start, not from enqueue. On heavy-queue
|
||||||
|
# days a freshly-promoted event whose original started_at
|
||||||
|
# predated the cutoff would otherwise get swept mid-flight,
|
||||||
|
# racing phase3's commit. Audit 2026-06-02.
|
||||||
|
existing.started_at = datetime.now(UTC)
|
||||||
await self.async_session.commit()
|
await self.async_session.commit()
|
||||||
event_id = existing.id
|
event_id = existing.id
|
||||||
else:
|
else:
|
||||||
@@ -166,7 +195,12 @@ class DownloadService:
|
|||||||
event_id = ev.id
|
event_id = ev.id
|
||||||
|
|
||||||
artist = source.artist
|
artist = source.artist
|
||||||
if source.platform in ("discord", "pixiv"):
|
# Drive cookies-vs-token selection from the platform registry's
|
||||||
|
# auth_type so a new 7th token-platform automatically picks the
|
||||||
|
# right credential path. The hardcoded tuple here used to drift
|
||||||
|
# out of sync with credential_service's auth_type_for(). Audit
|
||||||
|
# 2026-06-02.
|
||||||
|
if auth_type_for(source.platform) == "token":
|
||||||
cookies_path = None
|
cookies_path = None
|
||||||
auth_token = await self.cred_service.get_token(source.platform)
|
auth_token = await self.cred_service.get_token(source.platform)
|
||||||
else:
|
else:
|
||||||
@@ -185,6 +219,7 @@ class DownloadService:
|
|||||||
"config_overrides": dict(source.config_overrides or {}),
|
"config_overrides": dict(source.config_overrides or {}),
|
||||||
"cookies_path": cookies_path,
|
"cookies_path": cookies_path,
|
||||||
"auth_token": auth_token,
|
"auth_token": auth_token,
|
||||||
|
"backfill_runs_remaining": source.backfill_runs_remaining or 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
async def _phase3_persist(
|
async def _phase3_persist(
|
||||||
@@ -238,6 +273,40 @@ class DownloadService:
|
|||||||
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
|
# Enqueue thumbnail + ML for newly-attached images, matching
|
||||||
|
# the filesystem-import path (tasks/import_file.py:228-239).
|
||||||
|
# Importer.attach_in_place deliberately skips inline thumb
|
||||||
|
# generation to keep the import queue moving; the calling
|
||||||
|
# task is responsible for the enqueue. Operator-flagged
|
||||||
|
# 2026-06-01: without this, every downloaded image stayed
|
||||||
|
# at thumbnail_path=NULL until a periodic backfill swept
|
||||||
|
# it up, surfacing as broken-thumbnail tiles in the gallery
|
||||||
|
# for hours after a download landed. Lazy import to avoid
|
||||||
|
# circular-import risk between this service and the
|
||||||
|
# tasks/* modules that import it.
|
||||||
|
from ..tasks.ml import tag_and_embed
|
||||||
|
from ..tasks.thumbnail import generate_thumbnail
|
||||||
|
ids = list(result.member_image_ids)
|
||||||
|
if result.image_id is not None and result.image_id not in ids:
|
||||||
|
ids.append(result.image_id)
|
||||||
|
for img_id in ids:
|
||||||
|
generate_thumbnail.delay(img_id)
|
||||||
|
tag_and_embed.delay(img_id)
|
||||||
|
elif result.status == "attached":
|
||||||
|
# Non-media or extracted archive captured as PostAttachment
|
||||||
|
# (FC-2d-iii). The canonical copy lives in the attachments
|
||||||
|
# store; the original download path is now redundant —
|
||||||
|
# mirror duplicate_hash cleanup so we don't keep two copies.
|
||||||
|
# Operator-flagged 2026-06-02 (Lustria OST zip).
|
||||||
|
import_summary["attached"] += 1
|
||||||
|
try:
|
||||||
|
bytes_downloaded += path.stat().st_size # noqa: ASYNC240
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
|
elif result.status == "skipped" and result.skip_reason and result.skip_reason.value in (
|
||||||
"duplicate_hash", "duplicate_phash",
|
"duplicate_hash", "duplicate_phash",
|
||||||
):
|
):
|
||||||
@@ -246,6 +315,29 @@ class DownloadService:
|
|||||||
path.unlink(missing_ok=True) # noqa: ASYNC240
|
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
|
elif result.status == "skipped":
|
||||||
|
# Soft skip (too_small, too_transparent, invalid_image) —
|
||||||
|
# the file just didn't qualify, not a download/ingest
|
||||||
|
# failure. Don't flag the run as error; the file stays
|
||||||
|
# on disk for operator inspection.
|
||||||
|
import_summary["skipped"] += 1
|
||||||
|
elif result.status == "failed":
|
||||||
|
# Hard failure (today only: archive probe crash/timeout).
|
||||||
|
# The original archive sits in /images/ as an orphan; the
|
||||||
|
# filesystem scanner would re-import and re-crash on the
|
||||||
|
# same file, so delete the source file and surface the
|
||||||
|
# error in import_summary. Audit 2026-06-02.
|
||||||
|
import_summary["errors"] += 1
|
||||||
|
try:
|
||||||
|
path.unlink(missing_ok=True) # noqa: ASYNC240
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
elif result.status == "refreshed":
|
||||||
|
# Currently unreachable from attach_in_place (the download
|
||||||
|
# path never runs in deep=True mode), but the importer's
|
||||||
|
# ImportResult contract enumerates it. Treat the same as
|
||||||
|
# 'attached' — work happened, no error. Audit 2026-06-02.
|
||||||
|
import_summary["attached"] += 1
|
||||||
else:
|
else:
|
||||||
import_summary["errors"] += 1
|
import_summary["errors"] += 1
|
||||||
|
|
||||||
@@ -259,12 +351,21 @@ class DownloadService:
|
|||||||
run_stats["quarantined_count"] = dl_result.files_quarantined
|
run_stats["quarantined_count"] = dl_result.files_quarantined
|
||||||
stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr)
|
stderr_summary = self.gdl._extract_errors_warnings(dl_result.stderr)
|
||||||
|
|
||||||
status = "ok" if (dl_result.success and import_summary["errors"] == 0) else "error"
|
# Plan #544: PARTIAL means the run downloaded ≥1 file but the
|
||||||
|
# subprocess didn't finish in budget (typically wall-clock timeout
|
||||||
|
# mid-walk). Real work happened; the next tick continues via
|
||||||
|
# gallery-dl's archive. NOT a failure for status purposes.
|
||||||
|
if dl_result.success and import_summary["errors"] == 0:
|
||||||
|
status = "ok"
|
||||||
|
elif dl_result.error_type == ErrorType.PARTIAL and import_summary["errors"] == 0:
|
||||||
|
status = "ok"
|
||||||
|
else:
|
||||||
|
status = "error"
|
||||||
ev.status = status
|
ev.status = status
|
||||||
ev.finished_at = datetime.now(UTC)
|
ev.finished_at = datetime.now(UTC)
|
||||||
ev.files_count = import_summary["attached"]
|
ev.files_count = import_summary["attached"]
|
||||||
ev.bytes_downloaded = bytes_downloaded
|
ev.bytes_downloaded = bytes_downloaded
|
||||||
ev.error = dl_result.error_message if not dl_result.success else None
|
ev.error = dl_result.error_message if status == "error" else None
|
||||||
ev.metadata_ = {
|
ev.metadata_ = {
|
||||||
"run_stats": run_stats,
|
"run_stats": run_stats,
|
||||||
"error_type": dl_result.error_type.value if dl_result.error_type else None,
|
"error_type": dl_result.error_type.value if dl_result.error_type else None,
|
||||||
@@ -279,6 +380,33 @@ class DownloadService:
|
|||||||
source_id=ctx["source_id"], status=status, error_message=ev.error,
|
source_id=ctx["source_id"], status=status, error_message=ev.error,
|
||||||
error_type=dl_result.error_type.value if dl_result.error_type else None,
|
error_type=dl_result.error_type.value if dl_result.error_type else None,
|
||||||
)
|
)
|
||||||
|
# Plan #544: backfill lifecycle — auto-complete when a clean
|
||||||
|
# backfill run drained the queue (gallery-dl exited 0 + zero files
|
||||||
|
# downloaded means there was nothing to fetch); otherwise decrement
|
||||||
|
# the counter. Next tick falls back to tick mode once it hits 0.
|
||||||
|
#
|
||||||
|
# Audit 2026-06-02 gating: VALIDATION_FAILED also exits the
|
||||||
|
# subprocess with return_code=0 and files_downloaded=0 (every
|
||||||
|
# file was quarantined), which used to match the auto-complete
|
||||||
|
# predicate exactly — zeroing the operator's armed budget on
|
||||||
|
# the FIRST quarantine run instead of decrementing. Require
|
||||||
|
# dl_result.success + no error_type so only genuinely-empty
|
||||||
|
# successful runs drain the counter.
|
||||||
|
backfill_remaining = ctx.get("backfill_runs_remaining", 0) or 0
|
||||||
|
if backfill_remaining > 0:
|
||||||
|
src = (await self.async_session.execute(
|
||||||
|
select(Source).where(Source.id == ctx["source_id"])
|
||||||
|
)).scalar_one()
|
||||||
|
queue_drained = (
|
||||||
|
dl_result.success
|
||||||
|
and dl_result.error_type is None
|
||||||
|
and dl_result.return_code == 0
|
||||||
|
and dl_result.files_downloaded == 0
|
||||||
|
)
|
||||||
|
if queue_drained:
|
||||||
|
src.backfill_runs_remaining = 0
|
||||||
|
else:
|
||||||
|
src.backfill_runs_remaining = max(0, backfill_remaining - 1)
|
||||||
await self.async_session.commit()
|
await self.async_session.commit()
|
||||||
return event_id
|
return event_id
|
||||||
|
|
||||||
@@ -306,9 +434,15 @@ class DownloadService:
|
|||||||
if status == "ok":
|
if status == "ok":
|
||||||
source.consecutive_failures = 0
|
source.consecutive_failures = 0
|
||||||
source.last_error = None
|
source.last_error = None
|
||||||
|
# alembic 0032 — clear the failure-class chip on success.
|
||||||
|
source.error_type = None
|
||||||
elif status == "error":
|
elif status == "error":
|
||||||
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
||||||
source.last_error = error_message
|
source.last_error = error_message
|
||||||
|
# alembic 0032 — stamp the failure-class so FailingSourcesCard
|
||||||
|
# can render a colored chip and operators can bulk-triage
|
||||||
|
# by error class without opening Logs per row.
|
||||||
|
source.error_type = error_type
|
||||||
if error_type == "rate_limited":
|
if error_type == "rate_limited":
|
||||||
await set_platform_cooldown(self.async_session, source.platform)
|
await set_platform_cooldown(self.async_session, source.platform)
|
||||||
elif status == "skipped":
|
elif status == "skipped":
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
|
|
||||||
from ..models import Artist, Source
|
from ..models import Artist, Source
|
||||||
from ..utils.slug import slugify
|
from ..utils.slug import slugify
|
||||||
|
from .source_service import NEW_SOURCE_BACKFILL_RUNS
|
||||||
|
|
||||||
|
|
||||||
class UnknownPlatformError(Exception):
|
class UnknownPlatformError(Exception):
|
||||||
@@ -204,9 +205,16 @@ class ExtensionService:
|
|||||||
return existing, False
|
return existing, False
|
||||||
sp = await self.session.begin_nested()
|
sp = await self.session.begin_nested()
|
||||||
try:
|
try:
|
||||||
|
# New subscription sources arm a few backfill runs so the
|
||||||
|
# first ticks walk the full history (otherwise gallery-dl's
|
||||||
|
# exit:20 short-circuits before the archive is built).
|
||||||
|
# Mirrors SourceService.create — without it, Firefox quick-
|
||||||
|
# add on a creator with >20 unsynced posts would surface
|
||||||
|
# as "check failed" with no diagnosis. Audit 2026-06-02.
|
||||||
src = Source(
|
src = Source(
|
||||||
artist_id=artist_id, platform=platform,
|
artist_id=artist_id, platform=platform,
|
||||||
url=url, enabled=True,
|
url=url, enabled=True,
|
||||||
|
backfill_runs_remaining=NEW_SOURCE_BACKFILL_RUNS,
|
||||||
)
|
)
|
||||||
self.session.add(src)
|
self.session.add(src)
|
||||||
await self.session.flush()
|
await self.session.flush()
|
||||||
|
|||||||
@@ -39,9 +39,40 @@ class ErrorType(StrEnum):
|
|||||||
HTTP_ERROR = "http_error"
|
HTTP_ERROR = "http_error"
|
||||||
UNSUPPORTED_URL = "unsupported_url"
|
UNSUPPORTED_URL = "unsupported_url"
|
||||||
VALIDATION_FAILED = "validation_failed"
|
VALIDATION_FAILED = "validation_failed"
|
||||||
|
# Run made real progress (downloaded ≥1 file) but did not finish in the
|
||||||
|
# subprocess budget. Distinct from UNKNOWN_ERROR — the downstream status
|
||||||
|
# mapping classifies this as "ok" because the next tick continues.
|
||||||
|
PARTIAL = "partial"
|
||||||
UNKNOWN_ERROR = "unknown_error"
|
UNKNOWN_ERROR = "unknown_error"
|
||||||
|
|
||||||
|
|
||||||
|
# Tick mode (routine cron polls): skip ≤20 contiguous already-archived
|
||||||
|
# items, then exit gallery-dl. Established subscription with zero new
|
||||||
|
# content exits in ~30s of HEAD requests instead of walking to the bottom
|
||||||
|
# of the post history (which can be hours for prolific creators). 20 (not
|
||||||
|
# 5) is operator-set headroom against any edge case where paywalled or
|
||||||
|
# otherwise-non-downloadable items might interleave with archived ones —
|
||||||
|
# 20 contiguous HEADs is still negligible.
|
||||||
|
TICK_SKIP_VALUE = "exit:20"
|
||||||
|
|
||||||
|
# Backfill mode (operator-triggered deep scan): walk the full history.
|
||||||
|
# Source.backfill_runs_remaining > 0 selects this mode; the longer
|
||||||
|
# timeout below absorbs creators with thousands of posts.
|
||||||
|
#
|
||||||
|
# 30 seconds shy of Celery's hard `time_limit=1200` on download_source
|
||||||
|
# (tasks/download.py:33). subprocess.run MUST raise TimeoutExpired
|
||||||
|
# before Celery SIGKILLs the worker — same rationale as the tick
|
||||||
|
# default at line 74. The audit (2026-06-02) caught this at 1800,
|
||||||
|
# guaranteeing SIGKILL on any backfill that ran to its subprocess
|
||||||
|
# budget: stdout/stderr lost, backfill_runs_remaining never
|
||||||
|
# decrements, recovery sweep stamps generic "stranded" 30 min later.
|
||||||
|
# Recreates the exact Knuxy #38275 failure mode the tick 870s default
|
||||||
|
# was added to prevent. backfill_runs_remaining=3 still gives ~58
|
||||||
|
# minutes of cumulative walk across three runs for prolific creators.
|
||||||
|
BACKFILL_SKIP_VALUE = True
|
||||||
|
BACKFILL_TIMEOUT_SECONDS = 1170
|
||||||
|
|
||||||
|
|
||||||
# 30 seconds shy of download_source's Celery soft_time_limit (900s, see
|
# 30 seconds shy of download_source's Celery soft_time_limit (900s, see
|
||||||
# tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before
|
# tasks/download.py:32). subprocess.run MUST raise TimeoutExpired before
|
||||||
# Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race,
|
# Celery raises SoftTimeLimitExceeded — otherwise Celery wins the race,
|
||||||
@@ -56,12 +87,19 @@ _DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SourceConfig:
|
class SourceConfig:
|
||||||
|
"""Per-source overrides loaded from Source.config_overrides JSON.
|
||||||
|
|
||||||
|
Note: the gallery-dl `skip` value (tick vs backfill, see TICK_SKIP_VALUE /
|
||||||
|
BACKFILL_SKIP_VALUE) is NOT carried here — it derives from the
|
||||||
|
Source.backfill_runs_remaining column at the download_service layer
|
||||||
|
and is passed to _build_config_for_source as `skip_value`. Same for
|
||||||
|
the per-run subprocess timeout.
|
||||||
|
"""
|
||||||
content_types: list[str] = field(default_factory=lambda: ["all"])
|
content_types: list[str] = field(default_factory=lambda: ["all"])
|
||||||
sleep: float | None = None
|
sleep: float | None = None
|
||||||
sleep_request: float | None = None
|
sleep_request: float | None = None
|
||||||
directory_pattern: str | None = None
|
directory_pattern: str | None = None
|
||||||
filename_pattern: str | None = None
|
filename_pattern: str | None = None
|
||||||
skip_existing: bool = True
|
|
||||||
save_metadata: bool = True
|
save_metadata: bool = True
|
||||||
timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS
|
timeout: int = _DEFAULT_GDL_TIMEOUT_SECONDS
|
||||||
|
|
||||||
@@ -73,7 +111,6 @@ class SourceConfig:
|
|||||||
sleep_request=data.get("sleep_request"),
|
sleep_request=data.get("sleep_request"),
|
||||||
directory_pattern=data.get("directory_pattern"),
|
directory_pattern=data.get("directory_pattern"),
|
||||||
filename_pattern=data.get("filename_pattern"),
|
filename_pattern=data.get("filename_pattern"),
|
||||||
skip_existing=data.get("skip_existing", True),
|
|
||||||
save_metadata=data.get("save_metadata", True),
|
save_metadata=data.get("save_metadata", True),
|
||||||
timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS),
|
timeout=data.get("timeout", _DEFAULT_GDL_TIMEOUT_SECONDS),
|
||||||
)
|
)
|
||||||
@@ -231,6 +268,25 @@ class GalleryDLService:
|
|||||||
"part-directory": str(self._config_dir / "temp"),
|
"part-directory": str(self._config_dir / "temp"),
|
||||||
"retries": 3,
|
"retries": 3,
|
||||||
"timeout": 120.0,
|
"timeout": 120.0,
|
||||||
|
# Forward Patreon as Referer/Origin to yt-dlp when it
|
||||||
|
# fetches video manifests. Operator-flagged 2026-06-01
|
||||||
|
# (DaferQ patreon): video posts hosted on Mux carry a JWT
|
||||||
|
# playback restriction that checks Referer/Origin on every
|
||||||
|
# request — not just the token signature. gallery-dl's
|
||||||
|
# HEAD probe to stream.mux.com returns 200 (the token is
|
||||||
|
# valid), but yt-dlp's actual GET-with-Range to fetch the
|
||||||
|
# m3u8 manifest 403s because yt-dlp sends its own default
|
||||||
|
# Referer, which Mux's policy rejects. Forcing the right
|
||||||
|
# headers fixes the headers-only case; Mux IP-range
|
||||||
|
# restrictions are unfixable from here.
|
||||||
|
"ytdl": {
|
||||||
|
"raw-options": {
|
||||||
|
"http_headers": {
|
||||||
|
"Referer": "https://www.patreon.com/",
|
||||||
|
"Origin": "https://www.patreon.com",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
"output": {"progress": True},
|
"output": {"progress": True},
|
||||||
}
|
}
|
||||||
@@ -245,7 +301,18 @@ class GalleryDLService:
|
|||||||
platform: str,
|
platform: str,
|
||||||
source_config: SourceConfig,
|
source_config: SourceConfig,
|
||||||
artist_slug: str,
|
artist_slug: str,
|
||||||
|
skip_value: bool | str = BACKFILL_SKIP_VALUE,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
|
"""`skip_value` controls gallery-dl's archive-walk behavior:
|
||||||
|
- True (BACKFILL_SKIP_VALUE): walk full post history, skipping
|
||||||
|
archived items but continuing past them. Used in backfill mode.
|
||||||
|
- "exit:20" (TICK_SKIP_VALUE): exit gallery-dl after 20
|
||||||
|
contiguous archived items. Used in tick (routine catch-up)
|
||||||
|
mode for fast no-op syncs on creators with deep history.
|
||||||
|
- False: don't skip — redownload everything (not used in FC).
|
||||||
|
The caller (download_service) chooses based on
|
||||||
|
Source.backfill_runs_remaining.
|
||||||
|
"""
|
||||||
config = json.loads(json.dumps(self._get_default_config())) # deep copy
|
config = json.loads(json.dumps(self._get_default_config())) # deep copy
|
||||||
|
|
||||||
destination = str(self.images_root / artist_slug / platform)
|
destination = str(self.images_root / artist_slug / platform)
|
||||||
@@ -255,7 +322,7 @@ class GalleryDLService:
|
|||||||
config["extractor"]["sleep"] = source_config.sleep
|
config["extractor"]["sleep"] = source_config.sleep
|
||||||
if source_config.sleep_request is not None:
|
if source_config.sleep_request is not None:
|
||||||
config["extractor"]["sleep-request"] = source_config.sleep_request
|
config["extractor"]["sleep-request"] = source_config.sleep_request
|
||||||
config["extractor"]["skip"] = source_config.skip_existing
|
config["extractor"]["skip"] = skip_value
|
||||||
|
|
||||||
if source_config.save_metadata:
|
if source_config.save_metadata:
|
||||||
config["extractor"]["postprocessors"] = [
|
config["extractor"]["postprocessors"] = [
|
||||||
@@ -394,6 +461,22 @@ class GalleryDLService:
|
|||||||
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Partial-success: the subprocess exited non-zero (typically because
|
||||||
|
# the wall-clock timeout fired mid-walk), but it had downloaded ≥1
|
||||||
|
# file by then and no source-level error category fired. The work
|
||||||
|
# the run DID do is real; gallery-dl's archive will pick up where
|
||||||
|
# it left off on the next tick. Mapped to status="ok" downstream
|
||||||
|
# (download_service.py) so this doesn't flag the source as
|
||||||
|
# "needs attention." Operator-flagged 2026-06-01 after a Knuxy
|
||||||
|
# patreon run downloaded hundreds of files then ran red on timeout.
|
||||||
|
files_downloaded = self._count_downloaded_files(stdout)
|
||||||
|
if not has_actual_error and files_downloaded > 0:
|
||||||
|
return (
|
||||||
|
ErrorType.PARTIAL,
|
||||||
|
f"Downloaded {files_downloaded} file{'s' if files_downloaded != 1 else ''}; "
|
||||||
|
"run did not complete in budget — next tick will continue",
|
||||||
|
)
|
||||||
|
|
||||||
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
return ErrorType.UNKNOWN_ERROR, f"Unknown error (return code: {return_code})"
|
||||||
|
|
||||||
def _count_downloaded_files(self, stdout: str) -> int:
|
def _count_downloaded_files(self, stdout: str) -> int:
|
||||||
@@ -545,6 +628,7 @@ class GalleryDLService:
|
|||||||
source_config: SourceConfig | None = None,
|
source_config: SourceConfig | None = None,
|
||||||
cookies_path: str | None = None,
|
cookies_path: str | None = None,
|
||||||
auth_token: str | None = None,
|
auth_token: str | None = None,
|
||||||
|
skip_value: bool | str = BACKFILL_SKIP_VALUE,
|
||||||
) -> DownloadResult:
|
) -> DownloadResult:
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
started_at = datetime.now(UTC).isoformat()
|
started_at = datetime.now(UTC).isoformat()
|
||||||
@@ -552,7 +636,9 @@ class GalleryDLService:
|
|||||||
if source_config is None:
|
if source_config is None:
|
||||||
source_config = SourceConfig()
|
source_config = SourceConfig()
|
||||||
|
|
||||||
config = self._build_config_for_source(platform, source_config, artist_slug)
|
config = self._build_config_for_source(
|
||||||
|
platform, source_config, artist_slug, skip_value=skip_value,
|
||||||
|
)
|
||||||
|
|
||||||
if cookies_path:
|
if cookies_path:
|
||||||
config["extractor"]["cookies"] = cookies_path
|
config["extractor"]["cookies"] = cookies_path
|
||||||
@@ -769,7 +855,13 @@ class GalleryDLService:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
etype, msg = self._categorize_error(proc.returncode, proc.stdout, proc.stderr)
|
etype, msg = self._categorize_error(proc.returncode, proc.stdout, proc.stderr)
|
||||||
if proc.returncode == 0 or etype == ErrorType.NO_NEW_CONTENT:
|
# TIER_LIMITED proves auth worked — gallery-dl reached the
|
||||||
|
# post, was told it's tier-gated. The download path treats
|
||||||
|
# this as success (line 712); verify must too, or operators
|
||||||
|
# rotate working cookies for no reason. Audit 2026-06-02.
|
||||||
|
if proc.returncode == 0 or etype in (
|
||||||
|
ErrorType.NO_NEW_CONTENT, ErrorType.TIER_LIMITED,
|
||||||
|
):
|
||||||
return True, "Credentials valid — the feed authenticated."
|
return True, "Credentials valid — the feed authenticated."
|
||||||
if etype == ErrorType.AUTH_ERROR:
|
if etype == ErrorType.AUTH_ERROR:
|
||||||
return False, msg
|
return False, msg
|
||||||
|
|||||||
@@ -20,8 +20,9 @@ from datetime import datetime
|
|||||||
|
|
||||||
from sqlalchemy import Select, and_, exists, func, or_, select
|
from sqlalchemy import Select, and_, exists, func, or_, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
from sqlalchemy.orm import aliased
|
||||||
|
|
||||||
from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
|
from ..models import Artist, ImageProvenance, ImageRecord, Post, Tag
|
||||||
from ..models.tag import image_tag
|
from ..models.tag import image_tag
|
||||||
|
|
||||||
CURSOR_SEPARATOR = "|"
|
CURSOR_SEPARATOR = "|"
|
||||||
@@ -133,10 +134,23 @@ def _provenance_clause(post_id, artist_id):
|
|||||||
ImageProvenance.post_id == post_id,
|
ImageProvenance.post_id == post_id,
|
||||||
)
|
)
|
||||||
if artist_id is not None:
|
if artist_id is not None:
|
||||||
|
# Use Post.artist_id (alembic 0030 denormalized column) instead
|
||||||
|
# of joining through ImageProvenance.source_id → Source.artist_id.
|
||||||
|
# The denormalization is the always-present linkage; the source
|
||||||
|
# path now drops NULL-source provenance rows (filesystem-imported
|
||||||
|
# content) which would otherwise vanish from artist-filtered
|
||||||
|
# gallery views.
|
||||||
|
# ALIAS Post: the gallery query outer-joins Post on
|
||||||
|
# ImageRecord.primary_post_id (`_outer_join_primary_post`).
|
||||||
|
# SQLAlchemy would otherwise correlate a bare `Post` reference
|
||||||
|
# in this EXISTS subquery to that outer Post (which is NULL for
|
||||||
|
# images with no primary post), and the filter would silently
|
||||||
|
# match nothing.
|
||||||
|
post_inner = aliased(Post)
|
||||||
return exists().where(
|
return exists().where(
|
||||||
ImageProvenance.image_record_id == ImageRecord.id,
|
ImageProvenance.image_record_id == ImageRecord.id,
|
||||||
ImageProvenance.source_id == Source.id,
|
ImageProvenance.post_id == post_inner.id,
|
||||||
Source.artist_id == artist_id,
|
post_inner.artist_id == artist_id,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|||||||
@@ -212,10 +212,10 @@ class Importer:
|
|||||||
which would lose the surrounding scan's progress — and re-run `stmt`
|
which would lose the surrounding scan's progress — and re-run `stmt`
|
||||||
(scalar_one) to return the row the other worker created.
|
(scalar_one) to return the row the other worker created.
|
||||||
|
|
||||||
Centralizes the pattern shared by _find_or_create_source,
|
Centralizes the pattern shared by _find_or_create_source and
|
||||||
_source_for_sidecar, and _find_or_create_post. The plain
|
_find_or_create_post. The plain SELECT-then-INSERT version lost
|
||||||
SELECT-then-INSERT version lost races under the 5-min recovery sweep
|
races under the 5-min recovery sweep (operator-flagged
|
||||||
(operator-flagged 2026-05-26)."""
|
2026-05-26)."""
|
||||||
existing = self.session.execute(stmt).scalar_one_or_none()
|
existing = self.session.execute(stmt).scalar_one_or_none()
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
return existing
|
return existing
|
||||||
@@ -258,47 +258,25 @@ class Importer:
|
|||||||
lambda: Source(artist_id=artist_id, platform=platform, url=url),
|
lambda: Source(artist_id=artist_id, platform=platform, url=url),
|
||||||
)
|
)
|
||||||
|
|
||||||
def _source_for_sidecar(
|
def _lookup_source_for_sidecar(
|
||||||
self, *, artist_id: int, platform: str, artist_slug: str,
|
self, *, artist_id: int, platform: str,
|
||||||
) -> Source:
|
) -> Source | None:
|
||||||
"""Sidecar-import Source resolver. Used by both filesystem imports
|
"""Find the real subscription Source for (artist, platform), or
|
||||||
and gallery-dl downloads (both write sidecar JSON, both flow through
|
None if no subscription exists.
|
||||||
_apply_sidecar / _capture_attachment).
|
|
||||||
|
|
||||||
Source represents a subscription feed (one per artist+platform — the
|
Pre-alembic-0030 this method would CREATE a synthetic
|
||||||
URL polled by the FC-3 downloader). The filesystem importer used to
|
`sidecar:<platform>:<slug>` Source when no real one existed —
|
||||||
call _find_or_create_source(url=sd.post_url), creating one Source
|
because `Post.source_id` was NOT NULL and the importer needed
|
||||||
row per post URL — 100s of junk Sources per artist, all with
|
something to attach Posts to. Alembic 0030 relaxed both
|
||||||
enabled=True, polluting the artist detail page and tricking the
|
`Post.source_id` and `ImageProvenance.source_id` to nullable, so
|
||||||
subscription checker into trying to poll patreon post URLs as feeds.
|
synthetic anchors are obsolete; the importer now leaves
|
||||||
Operator-flagged 2026-05-26; consolidated via alembic 0022.
|
source_id as None when no subscription exists for the (artist,
|
||||||
|
platform). Operator-asked 2026-06-01: synthetic Sources had
|
||||||
Resolution order: prefer a real (non-sidecar) Source over a
|
leaked into the Subscriptions UI as phantom subscriptions and
|
||||||
synthetic anchor. When alembic 0022 ran, it may have rewritten
|
the operator wanted the data model to truthfully say "this
|
||||||
per-post Sources into `sidecar:<platform>:<slug>` synthetic
|
content has no live subscription."
|
||||||
anchors. If the operator later added the real subscription, both
|
|
||||||
rows now coexist. A naive `ORDER BY id ASC LIMIT 1` lookup would
|
|
||||||
pick the older synthetic and silently attach every gallery-dl
|
|
||||||
download to the wrong Source — operator-flagged 2026-05-31 after
|
|
||||||
the Subscriptions UI surfaced the phantom anchors. Pick the real
|
|
||||||
one when one exists; fall back to the synthetic; only create a
|
|
||||||
new synthetic when nothing exists for (artist, platform).
|
|
||||||
"""
|
"""
|
||||||
real_stmt = (
|
stmt = (
|
||||||
select(Source)
|
|
||||||
.where(
|
|
||||||
Source.artist_id == artist_id,
|
|
||||||
Source.platform == platform,
|
|
||||||
~Source.url.like("sidecar:%"),
|
|
||||||
)
|
|
||||||
.order_by(Source.id.asc())
|
|
||||||
.limit(1)
|
|
||||||
)
|
|
||||||
real = self.session.execute(real_stmt).scalar_one_or_none()
|
|
||||||
if real is not None:
|
|
||||||
return real
|
|
||||||
|
|
||||||
any_stmt = (
|
|
||||||
select(Source)
|
select(Source)
|
||||||
.where(
|
.where(
|
||||||
Source.artist_id == artist_id,
|
Source.artist_id == artist_id,
|
||||||
@@ -307,29 +285,37 @@ class Importer:
|
|||||||
.order_by(Source.id.asc())
|
.order_by(Source.id.asc())
|
||||||
.limit(1)
|
.limit(1)
|
||||||
)
|
)
|
||||||
return self._get_or_create(
|
return self.session.execute(stmt).scalar_one_or_none()
|
||||||
any_stmt,
|
|
||||||
lambda: Source(
|
|
||||||
artist_id=artist_id,
|
|
||||||
platform=platform,
|
|
||||||
url=f"sidecar:{platform}:{artist_slug}",
|
|
||||||
enabled=False,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _find_or_create_post(
|
def _find_or_create_post(
|
||||||
self, *, source_id: int, external_post_id: str,
|
self, *, source_id: int | None, external_post_id: str,
|
||||||
|
artist_id: int,
|
||||||
) -> Post:
|
) -> Post:
|
||||||
"""Race-safe find-or-create on `post` keyed by
|
"""Race-safe find-or-create on `post`. Keyed by
|
||||||
(source_id, external_post_id). Mirrors `_find_or_create_source`
|
(source_id, external_post_id) when source_id is set — the
|
||||||
— same savepoint + IntegrityError-recovery pattern."""
|
`uq_post_source_external_id` constraint guards. For NULL-source
|
||||||
stmt = select(Post).where(
|
posts the existence check matches on (artist_id, external_post_id),
|
||||||
Post.source_id == source_id,
|
which the partial unique index `uq_post_artist_external_id_null_source`
|
||||||
Post.external_post_id == external_post_id,
|
(alembic 0030) guards. Same savepoint + IntegrityError-recovery
|
||||||
)
|
pattern as the rest of the helpers."""
|
||||||
|
if source_id is not None:
|
||||||
|
stmt = select(Post).where(
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.external_post_id == external_post_id,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
stmt = select(Post).where(
|
||||||
|
Post.source_id.is_(None),
|
||||||
|
Post.artist_id == artist_id,
|
||||||
|
Post.external_post_id == external_post_id,
|
||||||
|
)
|
||||||
return self._get_or_create(
|
return self._get_or_create(
|
||||||
stmt,
|
stmt,
|
||||||
lambda: Post(source_id=source_id, external_post_id=external_post_id),
|
lambda: Post(
|
||||||
|
source_id=source_id,
|
||||||
|
artist_id=artist_id,
|
||||||
|
external_post_id=external_post_id,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
def import_one(self, source: Path) -> ImportResult:
|
def import_one(self, source: Path) -> ImportResult:
|
||||||
@@ -370,12 +356,14 @@ class Importer:
|
|||||||
return None
|
return None
|
||||||
sd = parse_sidecar(data)
|
sd = parse_sidecar(data)
|
||||||
platform = sd.platform or "unknown"
|
platform = sd.platform or "unknown"
|
||||||
src = self._source_for_sidecar(
|
src = self._lookup_source_for_sidecar(
|
||||||
artist_id=artist.id, platform=platform, artist_slug=artist.slug,
|
artist_id=artist.id, platform=platform,
|
||||||
)
|
)
|
||||||
epid = sd.external_post_id or sc.stem
|
epid = sd.external_post_id or sc.stem
|
||||||
return self._find_or_create_post(
|
return self._find_or_create_post(
|
||||||
source_id=src.id, external_post_id=epid,
|
source_id=src.id if src else None,
|
||||||
|
external_post_id=epid,
|
||||||
|
artist_id=artist.id,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _capture_attachment(
|
def _capture_attachment(
|
||||||
@@ -386,10 +374,19 @@ class Importer:
|
|||||||
artist = self._resolve_artist(source)
|
artist = self._resolve_artist(source)
|
||||||
post = self._post_for_sidecar(source, artist)
|
post = self._post_for_sidecar(source, artist)
|
||||||
sha = _sha256_of(source)
|
sha = _sha256_of(source)
|
||||||
existing = self.session.execute(
|
select_existing = select(PostAttachment).where(PostAttachment.sha256 == sha)
|
||||||
select(PostAttachment).where(PostAttachment.sha256 == sha)
|
existing = self.session.execute(select_existing).scalar_one_or_none()
|
||||||
).scalar_one_or_none()
|
if existing is not None:
|
||||||
if existing is None:
|
self.session.commit()
|
||||||
|
return ImportResult(status="attached")
|
||||||
|
# Savepoint + IntegrityError recovery — PostAttachment.sha256 is
|
||||||
|
# UNIQUE, so two workers can both pass the SELECT and only the
|
||||||
|
# second INSERT fails. Without savepoint, the outer transaction
|
||||||
|
# poisons and the calling task crashes. attachments.store is
|
||||||
|
# sha-addressed so both workers race to write the same target
|
||||||
|
# path; shutil.copy2 + rename is idempotent. Audit 2026-06-02.
|
||||||
|
sp = self.session.begin_nested()
|
||||||
|
try:
|
||||||
stored = self.attachments.store(source, sha)
|
stored = self.attachments.store(source, sha)
|
||||||
self.session.add(PostAttachment(
|
self.session.add(PostAttachment(
|
||||||
post_id=post.id if post else None,
|
post_id=post.id if post else None,
|
||||||
@@ -402,10 +399,19 @@ class Importer:
|
|||||||
size_bytes=source.stat().st_size,
|
size_bytes=source.stat().st_size,
|
||||||
))
|
))
|
||||||
self.session.flush()
|
self.session.flush()
|
||||||
|
sp.commit()
|
||||||
|
except IntegrityError:
|
||||||
|
sp.rollback()
|
||||||
|
# Lost the race — the other worker's row is canonical.
|
||||||
|
self.session.execute(select_existing).scalar_one()
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
return ImportResult(status="attached")
|
return ImportResult(status="attached")
|
||||||
|
|
||||||
def _import_archive(self, source: Path) -> ImportResult:
|
def _import_archive(
|
||||||
|
self, source: Path, *,
|
||||||
|
artist: Artist | None = None,
|
||||||
|
source_row: Source | None = None,
|
||||||
|
) -> ImportResult:
|
||||||
# Layer-3 isolation: bomb-size guard + integrity test in a
|
# Layer-3 isolation: bomb-size guard + integrity test in a
|
||||||
# spawned child BEFORE extracting in this process. A
|
# spawned child BEFORE extracting in this process. A
|
||||||
# decompression bomb or a native-lib crash on a malformed
|
# decompression bomb or a native-lib crash on a malformed
|
||||||
@@ -413,6 +419,14 @@ class Importer:
|
|||||||
# instead of OOMing/segfaulting the import worker. extract_archive
|
# instead of OOMing/segfaulting the import worker. extract_archive
|
||||||
# is already fail-soft for plain exceptions, so this only adds
|
# is already fail-soft for plain exceptions, so this only adds
|
||||||
# the hard-crash protection.
|
# the hard-crash protection.
|
||||||
|
#
|
||||||
|
# Audit 2026-06-02: optional artist/source_row kwargs let the
|
||||||
|
# download path thread its explicit subscription context
|
||||||
|
# through instead of having _resolve_artist re-derive from
|
||||||
|
# path-walk (which works by coincidence today because gallery-dl
|
||||||
|
# lays files out under /images/<artist_slug>/...). Filesystem
|
||||||
|
# import still calls bare _import_archive(source) and falls
|
||||||
|
# back to the path-walk derivation as before.
|
||||||
probe = safe_probe.probe_archive(source)
|
probe = safe_probe.probe_archive(source)
|
||||||
if not probe.ok:
|
if not probe.ok:
|
||||||
if probe.crashed:
|
if probe.crashed:
|
||||||
@@ -424,24 +438,28 @@ class Importer:
|
|||||||
# still preserve the archive file itself as an attachment so
|
# still preserve the archive file itself as an attachment so
|
||||||
# nothing silently vanishes, matching extract_archive's
|
# nothing silently vanishes, matching extract_archive's
|
||||||
# fail-soft contract.
|
# fail-soft contract.
|
||||||
artist = self._resolve_artist(source)
|
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||||
post = self._post_for_sidecar(source, artist)
|
post = self._post_for_sidecar(source, artist_use)
|
||||||
self._capture_attachment(source, post=post, artist=artist, resolved=True)
|
self._capture_attachment(
|
||||||
|
source, post=post, artist=artist_use, resolved=True,
|
||||||
|
)
|
||||||
return ImportResult(status="attached")
|
return ImportResult(status="attached")
|
||||||
|
|
||||||
artist = self._resolve_artist(source)
|
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||||
post = self._post_for_sidecar(source, artist)
|
post = self._post_for_sidecar(source, artist_use)
|
||||||
member_ids: list[int] = []
|
member_ids: list[int] = []
|
||||||
with extract_archive(source) as members:
|
with extract_archive(source) as members:
|
||||||
for _name, member_path in members:
|
for _name, member_path in members:
|
||||||
if not is_supported(member_path):
|
if not is_supported(member_path):
|
||||||
continue # non-media preserved via the stored archive
|
continue # non-media preserved via the stored archive
|
||||||
res = self._import_media(member_path, source)
|
res = self._import_media(
|
||||||
|
member_path, source, explicit_source=source_row,
|
||||||
|
)
|
||||||
if res.status in ("imported", "superseded") and res.image_id:
|
if res.status in ("imported", "superseded") and res.image_id:
|
||||||
member_ids.append(res.image_id)
|
member_ids.append(res.image_id)
|
||||||
# Preserve the archive itself (links to the same Post/Artist).
|
# Preserve the archive itself (links to the same Post/Artist).
|
||||||
self._capture_attachment(
|
self._capture_attachment(
|
||||||
source, post=post, artist=artist, resolved=True
|
source, post=post, artist=artist_use, resolved=True
|
||||||
)
|
)
|
||||||
if member_ids:
|
if member_ids:
|
||||||
return ImportResult(
|
return ImportResult(
|
||||||
@@ -451,7 +469,8 @@ class Importer:
|
|||||||
return ImportResult(status="attached")
|
return ImportResult(status="attached")
|
||||||
|
|
||||||
def _import_media(
|
def _import_media(
|
||||||
self, source: Path, attribution_path: Path
|
self, source: Path, attribution_path: Path,
|
||||||
|
*, explicit_source: Source | None = None,
|
||||||
) -> ImportResult:
|
) -> ImportResult:
|
||||||
"""The media import pipeline (filters, dedup, copy, provenance).
|
"""The media import pipeline (filters, dedup, copy, provenance).
|
||||||
|
|
||||||
@@ -598,7 +617,15 @@ class Importer:
|
|||||||
artist = self._attach_artist(record, artist_name)
|
artist = self._attach_artist(record, artist_name)
|
||||||
|
|
||||||
# Sidecar provenance (best-effort; never fails the import).
|
# Sidecar provenance (best-effort; never fails the import).
|
||||||
self._apply_sidecar(record, attribution_path, artist)
|
# explicit_source lets the FC-3c download path bind the new
|
||||||
|
# ImageProvenance row to its subscription Source instead of
|
||||||
|
# having _apply_sidecar re-derive via _lookup_source_for_sidecar.
|
||||||
|
# Audit 2026-06-02 — archive members extracted from a
|
||||||
|
# subscription-downloaded zip previously lost subscription
|
||||||
|
# linkage if the on-disk layout didn't match assumptions.
|
||||||
|
self._apply_sidecar(
|
||||||
|
record, attribution_path, artist, explicit_source=explicit_source,
|
||||||
|
)
|
||||||
|
|
||||||
# Thumbnail is queued separately by the calling task; the importer
|
# Thumbnail is queued separately by the calling task; the importer
|
||||||
# does not generate thumbnails inline so the import queue stays moving.
|
# does not generate thumbnails inline so the import queue stays moving.
|
||||||
@@ -662,16 +689,36 @@ class Importer:
|
|||||||
them through. The sidecar JSON gallery-dl emits next to each
|
them through. The sidecar JSON gallery-dl emits next to each
|
||||||
downloaded file is read by `_apply_sidecar` via `find_sidecar`.
|
downloaded file is read by `_apply_sidecar` via `find_sidecar`.
|
||||||
|
|
||||||
|
File-type dispatch parity with `import_one` (FC-2d-iii): zips,
|
||||||
|
PDFs, audio etc. become PostAttachments; archives are extracted.
|
||||||
|
Without this dispatch, gallery-dl-downloaded non-media bounced
|
||||||
|
back as `skipped+invalid_image`, which DownloadService counted
|
||||||
|
as an ingest error and flipped otherwise-successful runs to
|
||||||
|
status="error". Operator-flagged 2026-06-02 after a Lustria
|
||||||
|
patreon run with a 94MB OST zip went red despite 21 successful
|
||||||
|
image attaches.
|
||||||
|
|
||||||
Caller's responsibilities after this returns:
|
Caller's responsibilities after this returns:
|
||||||
- duplicate_hash / duplicate_phash skip → delete the on-disk file
|
- duplicate_hash / duplicate_phash skip → delete the on-disk file
|
||||||
- superseded → file stays where it is (now canonical)
|
- superseded → file stays where it is (now canonical)
|
||||||
- imported → file stays where it is
|
- imported → file stays where it is
|
||||||
|
- attached → the file's been copied into the attachments store;
|
||||||
|
caller may delete the on-disk original (mirrors duplicate_hash)
|
||||||
- failed → file untouched; caller decides
|
- failed → file untouched; caller decides
|
||||||
"""
|
"""
|
||||||
if not is_supported(path):
|
if path.suffix.lower() == ".json":
|
||||||
return ImportResult(
|
return ImportResult(
|
||||||
status="skipped", skip_reason=SkipReason.invalid_image,
|
status="skipped", skip_reason=SkipReason.invalid_image,
|
||||||
error=f"unsupported extension {path.suffix}",
|
error="sidecar json is metadata, not content",
|
||||||
|
)
|
||||||
|
if is_archive(path):
|
||||||
|
return self._import_archive(
|
||||||
|
path, artist=artist, source_row=source,
|
||||||
|
)
|
||||||
|
if not is_supported(path):
|
||||||
|
post = self._post_for_sidecar(path, artist) if artist else None
|
||||||
|
return self._capture_attachment(
|
||||||
|
path, post=post, artist=artist, resolved=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Format / dimension / transparency filters (mirror _import_media).
|
# Format / dimension / transparency filters (mirror _import_media).
|
||||||
@@ -743,7 +790,8 @@ class Importer:
|
|||||||
if rel == "smaller_exists":
|
if rel == "smaller_exists":
|
||||||
target = self.session.get(ImageRecord, match_id)
|
target = self.session.get(ImageRecord, match_id)
|
||||||
self._supersede(
|
self._supersede(
|
||||||
target, path, sha, phash, width, height, new_path=path
|
target, path, sha, phash, width, height,
|
||||||
|
new_path=path, artist=artist, source_row=source,
|
||||||
)
|
)
|
||||||
return ImportResult(status="superseded", image_id=match_id)
|
return ImportResult(status="superseded", image_id=match_id)
|
||||||
|
|
||||||
@@ -858,14 +906,15 @@ class Importer:
|
|||||||
src = explicit_source
|
src = explicit_source
|
||||||
else:
|
else:
|
||||||
platform = sd.platform or "unknown"
|
platform = sd.platform or "unknown"
|
||||||
src = self._source_for_sidecar(
|
src = self._lookup_source_for_sidecar(
|
||||||
artist_id=artist.id, platform=platform,
|
artist_id=artist.id, platform=platform,
|
||||||
artist_slug=artist.slug,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
epid = sd.external_post_id or sc.stem
|
epid = sd.external_post_id or sc.stem
|
||||||
post = self._find_or_create_post(
|
post = self._find_or_create_post(
|
||||||
source_id=src.id, external_post_id=epid,
|
source_id=src.id if src else None,
|
||||||
|
external_post_id=epid,
|
||||||
|
artist_id=artist.id,
|
||||||
)
|
)
|
||||||
if sd.post_url is not None:
|
if sd.post_url is not None:
|
||||||
post.post_url = sd.post_url
|
post.post_url = sd.post_url
|
||||||
@@ -901,7 +950,7 @@ class Importer:
|
|||||||
ImageProvenance(
|
ImageProvenance(
|
||||||
image_record_id=record.id,
|
image_record_id=record.id,
|
||||||
post_id=post.id,
|
post_id=post.id,
|
||||||
source_id=src.id,
|
source_id=src.id if src else None,
|
||||||
captured_metadata=sd.raw,
|
captured_metadata=sd.raw,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
@@ -937,6 +986,8 @@ class Importer:
|
|||||||
self, existing: ImageRecord, source: Path, sha: str,
|
self, existing: ImageRecord, source: Path, sha: str,
|
||||||
phash: str, width: int | None, height: int | None,
|
phash: str, width: int | None, height: int | None,
|
||||||
*, new_path: Path | None = None,
|
*, new_path: Path | None = None,
|
||||||
|
artist: Artist | None = None,
|
||||||
|
source_row: Source | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Replace `existing`'s file with the larger `source`, keeping the
|
"""Replace `existing`'s file with the larger `source`, keeping the
|
||||||
row id (so tags/series/curation stay attached). ML is cleared so
|
row id (so tags/series/curation stay attached). ML is cleared so
|
||||||
@@ -988,8 +1039,14 @@ class Importer:
|
|||||||
# _apply_sidecar resolves artist from the sidecar itself if the
|
# _apply_sidecar resolves artist from the sidecar itself if the
|
||||||
# existing row has none, and is internally guarded against
|
# existing row has none, and is internally guarded against
|
||||||
# missing-or-malformed sidecars (silent return).
|
# missing-or-malformed sidecars (silent return).
|
||||||
|
# Audit 2026-06-02: thread artist/source_row from the
|
||||||
|
# download-path caller (attach_in_place smaller_exists branch)
|
||||||
|
# so the supersede preserves explicit subscription linkage
|
||||||
|
# instead of re-deriving via path-walk.
|
||||||
try:
|
try:
|
||||||
self._apply_sidecar(existing, source, None)
|
self._apply_sidecar(
|
||||||
|
existing, source, artist, explicit_source=source_row,
|
||||||
|
)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
# Don't unwind the supersede DB swap if sidecar parsing
|
# Don't unwind the supersede DB swap if sidecar parsing
|
||||||
# blows up unexpectedly — the file replacement is the
|
# blows up unexpectedly — the file replacement is the
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ from ...models import (
|
|||||||
TagReferenceEmbedding,
|
TagReferenceEmbedding,
|
||||||
)
|
)
|
||||||
from ...models.tag import image_tag
|
from ...models.tag import image_tag
|
||||||
from .embedder import MODEL_VERSION as SIGLIP_VERSION
|
|
||||||
|
|
||||||
ELIGIBLE_KINDS = {
|
ELIGIBLE_KINDS = {
|
||||||
TagKind.character,
|
TagKind.character,
|
||||||
@@ -46,6 +45,21 @@ class CentroidService:
|
|||||||
)
|
)
|
||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
|
async def _model_version(self) -> str:
|
||||||
|
"""Audit 2026-06-02: SigLIP model-version stamp comes from the
|
||||||
|
DB row, not the env constant. tag_and_embed (tasks/ml.py:110)
|
||||||
|
already reads from MLSettings.embedder_model_version, so by
|
||||||
|
sourcing centroid stamps + drift checks from the same row, we
|
||||||
|
eliminate the silent-drift case the audit flagged. env
|
||||||
|
SIGLIP_MODEL_VERSION still drives which model embedder.py
|
||||||
|
loads at runtime; the version stamp is purely the operator-
|
||||||
|
controlled identifier."""
|
||||||
|
return (
|
||||||
|
await self.session.execute(
|
||||||
|
select(MLSettings.embedder_model_version).where(MLSettings.id == 1)
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
async def recompute_for_tag(self, tag_id: int) -> bool:
|
async def recompute_for_tag(self, tag_id: int) -> bool:
|
||||||
"""Recompute one tag's centroid. Returns True if a centroid was
|
"""Recompute one tag's centroid. Returns True if a centroid was
|
||||||
written, False if skipped (ineligible kind or too few members)."""
|
written, False if skipped (ineligible kind or too few members)."""
|
||||||
@@ -69,19 +83,20 @@ class CentroidService:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
centroid = np.mean(np.stack(embeddings), axis=0).astype(np.float32)
|
centroid = np.mean(np.stack(embeddings), axis=0).astype(np.float32)
|
||||||
|
model_version = await self._model_version()
|
||||||
|
|
||||||
stmt = insert(TagReferenceEmbedding).values(
|
stmt = insert(TagReferenceEmbedding).values(
|
||||||
tag_id=tag_id,
|
tag_id=tag_id,
|
||||||
embedding=centroid.tolist(),
|
embedding=centroid.tolist(),
|
||||||
reference_count=len(embeddings),
|
reference_count=len(embeddings),
|
||||||
model_version=SIGLIP_VERSION,
|
model_version=model_version,
|
||||||
)
|
)
|
||||||
stmt = stmt.on_conflict_do_update(
|
stmt = stmt.on_conflict_do_update(
|
||||||
index_elements=["tag_id"],
|
index_elements=["tag_id"],
|
||||||
set_={
|
set_={
|
||||||
"embedding": centroid.tolist(),
|
"embedding": centroid.tolist(),
|
||||||
"reference_count": len(embeddings),
|
"reference_count": len(embeddings),
|
||||||
"model_version": SIGLIP_VERSION,
|
"model_version": model_version,
|
||||||
"updated_at": func.now(),
|
"updated_at": func.now(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -92,6 +107,7 @@ class CentroidService:
|
|||||||
"""Tag ids whose centroid is stale: member count != reference_count,
|
"""Tag ids whose centroid is stale: member count != reference_count,
|
||||||
OR no centroid row, OR centroid built on a different SigLIP version.
|
OR no centroid row, OR centroid built on a different SigLIP version.
|
||||||
Only considers eligible-kind tags with embeddings present."""
|
Only considers eligible-kind tags with embeddings present."""
|
||||||
|
current_model_version = await self._model_version()
|
||||||
member_counts = (
|
member_counts = (
|
||||||
select(
|
select(
|
||||||
image_tag.c.tag_id.label("tag_id"),
|
image_tag.c.tag_id.label("tag_id"),
|
||||||
@@ -116,7 +132,7 @@ class CentroidService:
|
|||||||
TagReferenceEmbedding.reference_count
|
TagReferenceEmbedding.reference_count
|
||||||
!= member_counts.c.members
|
!= member_counts.c.members
|
||||||
)
|
)
|
||||||
| (TagReferenceEmbedding.model_version != SIGLIP_VERSION)
|
| (TagReferenceEmbedding.model_version != current_model_version)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return list((await self.session.execute(stmt)).scalars().all())
|
return list((await self.session.execute(stmt)).scalars().all())
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ threshold-filtered, category-grouped, ranked suggestions for one image.
|
|||||||
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ...models import (
|
from ...models import (
|
||||||
@@ -16,6 +16,7 @@ from ...models import (
|
|||||||
from ...models.tag import image_tag
|
from ...models.tag import image_tag
|
||||||
from .aliases import AliasService
|
from .aliases import AliasService
|
||||||
from .centroids import CentroidService
|
from .centroids import CentroidService
|
||||||
|
from .tag_name import normalize as normalize_tag_name
|
||||||
from .tagger import SURFACED_CATEGORIES
|
from .tagger import SURFACED_CATEGORIES
|
||||||
|
|
||||||
|
|
||||||
@@ -48,11 +49,11 @@ class SuggestionService:
|
|||||||
).scalar_one()
|
).scalar_one()
|
||||||
|
|
||||||
def _threshold_for(self, s: MLSettings, category: str) -> float:
|
def _threshold_for(self, s: MLSettings, category: str) -> float:
|
||||||
# 'artist' intentionally absent (FC-2d-vii-c) — falls through to
|
# 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired;
|
||||||
# the 1.01 "never surfaces" default like any unsurfaced category.
|
# both fall through to the 1.01 "never surfaces" default like any
|
||||||
|
# unsurfaced category.
|
||||||
return {
|
return {
|
||||||
"character": s.suggestion_threshold_character,
|
"character": s.suggestion_threshold_character,
|
||||||
"copyright": s.suggestion_threshold_copyright,
|
|
||||||
"general": s.suggestion_threshold_general,
|
"general": s.suggestion_threshold_general,
|
||||||
}.get(category, 1.01)
|
}.get(category, 1.01)
|
||||||
|
|
||||||
@@ -84,7 +85,12 @@ class SuggestionService:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# --- Camie predictions ---
|
# --- Camie predictions ---
|
||||||
candidates: list[tuple[str, str, float]] = []
|
# candidates carry (raw_name, display_name, category, confidence).
|
||||||
|
# raw_name = the booru-formatted vocab key, kept for alias_map
|
||||||
|
# lookup since alias rows are hand-curated against raw keys.
|
||||||
|
# display_name = normalize_tag_name(raw_name) — what the operator
|
||||||
|
# sees AND what gets written to tag.name on Accept.
|
||||||
|
candidates: list[tuple[str, str, str, float]] = []
|
||||||
for name, p in predictions.items():
|
for name, p in predictions.items():
|
||||||
category = p.get("category", "general")
|
category = p.get("category", "general")
|
||||||
if category not in SURFACED_CATEGORIES:
|
if category not in SURFACED_CATEGORIES:
|
||||||
@@ -92,10 +98,14 @@ class SuggestionService:
|
|||||||
conf = float(p.get("confidence", 0.0))
|
conf = float(p.get("confidence", 0.0))
|
||||||
if conf < self._threshold_for(settings, category):
|
if conf < self._threshold_for(settings, category):
|
||||||
continue
|
continue
|
||||||
candidates.append((name, category, conf))
|
display = normalize_tag_name(name)
|
||||||
|
if display is None:
|
||||||
|
# emoticon / pure-punctuation vocab entry — drop entirely
|
||||||
|
continue
|
||||||
|
candidates.append((name, display, category, conf))
|
||||||
|
|
||||||
alias_map = await self.aliases.resolve_many(
|
alias_map = await self.aliases.resolve_many(
|
||||||
[(n, c) for n, c, _ in candidates]
|
[(raw, c) for raw, _disp, c, _conf in candidates]
|
||||||
)
|
)
|
||||||
|
|
||||||
merged: dict[object, Suggestion] = {}
|
merged: dict[object, Suggestion] = {}
|
||||||
@@ -116,8 +126,8 @@ class SuggestionService:
|
|||||||
creates_new_tag=existing.creates_new_tag,
|
creates_new_tag=existing.creates_new_tag,
|
||||||
)
|
)
|
||||||
|
|
||||||
for name, category, conf in candidates:
|
for raw, display, category, conf in candidates:
|
||||||
canonical = alias_map.get((name, category))
|
canonical = alias_map.get((raw, category))
|
||||||
if canonical is not None:
|
if canonical is not None:
|
||||||
if canonical.id in applied or canonical.id in rejected:
|
if canonical.id in applied or canonical.id in rejected:
|
||||||
continue
|
continue
|
||||||
@@ -133,9 +143,17 @@ class SuggestionService:
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
# Case-insensitive match on BOTH the raw camie key AND
|
||||||
|
# the normalized form — covers legacy underscore-named
|
||||||
|
# Tag rows accepted before normalization shipped, AND
|
||||||
|
# any tag the operator created with the human form.
|
||||||
existing_tag = (
|
existing_tag = (
|
||||||
await self.session.execute(
|
await self.session.execute(
|
||||||
select(Tag).where(Tag.name == name)
|
select(Tag).where(
|
||||||
|
func.lower(Tag.name).in_(
|
||||||
|
[raw.lower(), display.lower()]
|
||||||
|
)
|
||||||
|
)
|
||||||
)
|
)
|
||||||
).scalars().first()
|
).scalars().first()
|
||||||
if existing_tag is not None:
|
if existing_tag is not None:
|
||||||
@@ -157,10 +175,10 @@ class SuggestionService:
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
_merge(
|
_merge(
|
||||||
f"raw:{name}:{category}",
|
f"raw:{display}:{category}",
|
||||||
Suggestion(
|
Suggestion(
|
||||||
canonical_tag_id=None,
|
canonical_tag_id=None,
|
||||||
display_name=name,
|
display_name=display,
|
||||||
category=category,
|
category=category,
|
||||||
score=conf,
|
score=conf,
|
||||||
source="tagger",
|
source="tagger",
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
"""Camie vocabulary -> human-readable tag-name normalization.
|
||||||
|
|
||||||
|
Camie v2's ~57k tag vocabulary is booru-derived and arrives as raw
|
||||||
|
strings like `uchiha_sasuke_(naruto)`, `#unicus_(idolmaster)`,
|
||||||
|
`1000-nen_ikiteru_(vocaloid)`, or `:/`. We want the operator to see
|
||||||
|
"Uchiha Sasuke", "Unicus", "1000-Nen Ikiteru", or to never see the
|
||||||
|
emoticon at all — and we want the same clean string to be what lands
|
||||||
|
in `tag.name` when the suggestion is accepted, so Accept matches the
|
||||||
|
existing-tag convention (`tag_service.find_or_create`).
|
||||||
|
|
||||||
|
Rules (operator-approved 2026-06-03):
|
||||||
|
1. Strip leading junk chars (#, ., +, ;, ~, _, whitespace)
|
||||||
|
2. Drop trailing `_(disambiguator)` block(s), iteratively
|
||||||
|
3. Strip wrapping single/double quotes (after disambig removal so
|
||||||
|
`"foo_em_up"_(series)` -> `"foo_em_up"` -> `foo_em_up`)
|
||||||
|
4. Replace remaining `_` with space; collapse runs of whitespace
|
||||||
|
5. Add a space after any `:` (namespace:tag -> namespace: tag)
|
||||||
|
6. Preserve hyphens (booru hyphens often carry meaning)
|
||||||
|
7. Title-case each space-separated word (first character only —
|
||||||
|
apostrophes, digits, hyphens stay)
|
||||||
|
8. If no letters AND no digits remain, return None (drops emoticons
|
||||||
|
like `:/` or `^_^`; preserves bare digit tags like `2005`)
|
||||||
|
9. No surname/givenname swap — no reliable signal in the vocab
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
_LEADING_JUNK = re.compile(r"^[#.+;~_\s]+")
|
||||||
|
_TRAILING_DISAMBIG = re.compile(r"_\([^)]*\)\s*$")
|
||||||
|
_MULTISPACE = re.compile(r"\s+")
|
||||||
|
_COLON_NOSPACE = re.compile(r":(?=\S)")
|
||||||
|
_HAS_ALPHANUMERIC = re.compile(r"[A-Za-z0-9]")
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_wrapping_quotes(s: str) -> str:
|
||||||
|
if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"):
|
||||||
|
return s[1:-1]
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
|
def _title_word(w: str) -> str:
|
||||||
|
return w[:1].upper() + w[1:] if w else w
|
||||||
|
|
||||||
|
|
||||||
|
def normalize(raw: str) -> str | None:
|
||||||
|
"""Return the human-readable form of a raw Camie tag, or None if the
|
||||||
|
string is junk (emoticon, empty after stripping)."""
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
s = _LEADING_JUNK.sub("", raw)
|
||||||
|
while True:
|
||||||
|
new = _TRAILING_DISAMBIG.sub("", s)
|
||||||
|
if new == s:
|
||||||
|
break
|
||||||
|
s = new
|
||||||
|
s = _strip_wrapping_quotes(s)
|
||||||
|
s = s.replace("_", " ")
|
||||||
|
s = _COLON_NOSPACE.sub(": ", s)
|
||||||
|
s = _MULTISPACE.sub(" ", s).strip()
|
||||||
|
if not s or not _HAS_ALPHANUMERIC.search(s):
|
||||||
|
return None
|
||||||
|
return " ".join(_title_word(w) for w in s.split(" "))
|
||||||
@@ -38,10 +38,13 @@ STORE_FLOOR = float(os.environ.get("TAGGER_STORE_FLOOR", "0.05"))
|
|||||||
|
|
||||||
# The categories FC-2b surfaces in the UI. Others (meta/rating/year) are
|
# The categories FC-2b surfaces in the UI. Others (meta/rating/year) are
|
||||||
# still stored but the suggestion service filters them out.
|
# still stored but the suggestion service filters them out.
|
||||||
# FC-2d-vii-c: 'artist' retired — artist identity is acquisition-derived
|
# 'artist' retired in FC-2d-vii-c — artist identity is acquisition-derived
|
||||||
# (image_record.artist_id), never ML-inferred. Raw predictions are still
|
# (image_record.artist_id), never ML-inferred. 'copyright' retired
|
||||||
# stored at STORE_FLOOR but artist never surfaces.
|
# 2026-06-01 — operator doesn't use the copyright tag-kind; fandom is
|
||||||
SURFACED_CATEGORIES = {"character", "copyright", "general"}
|
# this app's franchise/series concept (per TagsView.vue's doc comment).
|
||||||
|
# Raw predictions for both categories still get stored at STORE_FLOOR but
|
||||||
|
# don't surface in suggestions.
|
||||||
|
SURFACED_CATEGORIES = {"character", "general"}
|
||||||
|
|
||||||
# ImageNet preprocessing constants (per Camie v2 onnx_inference.py).
|
# ImageNet preprocessing constants (per Camie v2 onnx_inference.py).
|
||||||
_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
|
_IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)
|
||||||
|
|||||||
@@ -96,10 +96,12 @@ def str_field(v) -> str | None:
|
|||||||
# Shared gallery-dl invocation defaults. Embedded in each platform's
|
# Shared gallery-dl invocation defaults. Embedded in each platform's
|
||||||
# default_config (with platform-specific overrides) so per-platform
|
# default_config (with platform-specific overrides) so per-platform
|
||||||
# choices stay explicit.
|
# choices stay explicit.
|
||||||
|
# Note: the gallery-dl `skip` value (tick "exit:20" vs backfill True) is
|
||||||
|
# NOT here — it's derived from Source.backfill_runs_remaining at download
|
||||||
|
# time. See plan #544 / gallery_dl.TICK_SKIP_VALUE,BACKFILL_SKIP_VALUE.
|
||||||
GD_DEFAULTS = {
|
GD_DEFAULTS = {
|
||||||
"sleep": 3.0,
|
"sleep": 3.0,
|
||||||
"sleep_request": 1.5,
|
"sleep_request": 1.5,
|
||||||
"skip_existing": True,
|
|
||||||
"save_metadata": True,
|
"save_metadata": True,
|
||||||
"timeout": 3600,
|
"timeout": 3600,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,13 +69,19 @@ class PostFeedService:
|
|||||||
raise ValueError("direction must be 'older' or 'newer'")
|
raise ValueError("direction must be 'older' or 'newer'")
|
||||||
|
|
||||||
sort_key = _sort_key()
|
sort_key = _sort_key()
|
||||||
|
# Artist via the denormalized Post.artist_id (alembic 0030);
|
||||||
|
# Source via LEFT JOIN since post.source_id can now be NULL for
|
||||||
|
# filesystem-imported posts with no live subscription. A
|
||||||
|
# platform= filter implicitly excludes NULL-source posts (they
|
||||||
|
# have no platform); an artist_id= filter still surfaces them
|
||||||
|
# because Post.artist_id is always set.
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Post, Artist, Source)
|
select(Post, Artist, Source)
|
||||||
.join(Source, Post.source_id == Source.id)
|
.join(Artist, Post.artist_id == Artist.id)
|
||||||
.join(Artist, Source.artist_id == Artist.id)
|
.outerjoin(Source, Post.source_id == Source.id)
|
||||||
)
|
)
|
||||||
if artist_id is not None:
|
if artist_id is not None:
|
||||||
stmt = stmt.where(Source.artist_id == artist_id)
|
stmt = stmt.where(Post.artist_id == artist_id)
|
||||||
if platform is not None:
|
if platform is not None:
|
||||||
stmt = stmt.where(Source.platform == platform)
|
stmt = stmt.where(Source.platform == platform)
|
||||||
if cursor:
|
if cursor:
|
||||||
@@ -135,8 +141,8 @@ class PostFeedService:
|
|||||||
cursor for each end. Returns None if the post doesn't exist."""
|
cursor for each end. Returns None if the post doesn't exist."""
|
||||||
anchor = (await self.session.execute(
|
anchor = (await self.session.execute(
|
||||||
select(Post, Artist, Source)
|
select(Post, Artist, Source)
|
||||||
.join(Source, Post.source_id == Source.id)
|
.join(Artist, Post.artist_id == Artist.id)
|
||||||
.join(Artist, Source.artist_id == Artist.id)
|
.outerjoin(Source, Post.source_id == Source.id)
|
||||||
.where(Post.id == post_id)
|
.where(Post.id == post_id)
|
||||||
)).one_or_none()
|
)).one_or_none()
|
||||||
if anchor is None:
|
if anchor is None:
|
||||||
@@ -168,8 +174,8 @@ class PostFeedService:
|
|||||||
async def get_post(self, post_id: int) -> dict | None:
|
async def get_post(self, post_id: int) -> dict | None:
|
||||||
row = (await self.session.execute(
|
row = (await self.session.execute(
|
||||||
select(Post, Artist, Source)
|
select(Post, Artist, Source)
|
||||||
.join(Source, Post.source_id == Source.id)
|
.join(Artist, Post.artist_id == Artist.id)
|
||||||
.join(Artist, Source.artist_id == Artist.id)
|
.outerjoin(Source, Post.source_id == Source.id)
|
||||||
.where(Post.id == post_id)
|
.where(Post.id == post_id)
|
||||||
)).one_or_none()
|
)).one_or_none()
|
||||||
if row is None:
|
if row is None:
|
||||||
@@ -260,7 +266,7 @@ class PostFeedService:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
def _to_dict(
|
def _to_dict(
|
||||||
self, post: Post, artist: Artist, source: Source,
|
self, post: Post, artist: Artist, source: Source | None,
|
||||||
thumbs_map: dict, atts_map: dict,
|
thumbs_map: dict, atts_map: dict,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
plain_full = html_to_plain(post.description) if post.description else None
|
plain_full = html_to_plain(post.description) if post.description else None
|
||||||
@@ -269,6 +275,9 @@ class PostFeedService:
|
|||||||
else:
|
else:
|
||||||
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
||||||
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
||||||
|
# `source` is null for filesystem-imported posts with no live
|
||||||
|
# subscription (alembic 0030). Frontend renders that as a
|
||||||
|
# "filesystem import" affordance instead of a platform chip.
|
||||||
return {
|
return {
|
||||||
"id": post.id,
|
"id": post.id,
|
||||||
"external_post_id": post.external_post_id,
|
"external_post_id": post.external_post_id,
|
||||||
@@ -279,7 +288,10 @@ class PostFeedService:
|
|||||||
"description_plain": description_plain,
|
"description_plain": description_plain,
|
||||||
"description_truncated": truncated,
|
"description_truncated": truncated,
|
||||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||||
"source": {"id": source.id, "platform": source.platform},
|
"source": (
|
||||||
|
{"id": source.id, "platform": source.platform}
|
||||||
|
if source is not None else None
|
||||||
|
),
|
||||||
"thumbnails": thumbs_entry["thumbs"],
|
"thumbnails": thumbs_entry["thumbs"],
|
||||||
"thumbnails_more": thumbs_entry["more"],
|
"thumbnails_more": thumbs_entry["more"],
|
||||||
"attachments": atts_map.get(post.id, []),
|
"attachments": atts_map.get(post.id, []),
|
||||||
|
|||||||
@@ -70,11 +70,15 @@ class ProvenanceService:
|
|||||||
rec = await self.session.get(ImageRecord, image_id)
|
rec = await self.session.get(ImageRecord, image_id)
|
||||||
if rec is None:
|
if rec is None:
|
||||||
return None
|
return None
|
||||||
|
# Artist via Post.artist_id (alembic 0030); Source via LEFT JOIN
|
||||||
|
# since both Post.source_id and ImageProvenance.source_id can be
|
||||||
|
# NULL for filesystem-imported content. Frontend renders source=
|
||||||
|
# null as "filesystem import."
|
||||||
stmt = (
|
stmt = (
|
||||||
select(ImageProvenance, Post, Source, Artist)
|
select(ImageProvenance, Post, Source, Artist)
|
||||||
.join(Post, Post.id == ImageProvenance.post_id)
|
.join(Post, Post.id == ImageProvenance.post_id)
|
||||||
.join(Source, Source.id == ImageProvenance.source_id)
|
.join(Artist, Artist.id == Post.artist_id)
|
||||||
.join(Artist, Artist.id == Source.artist_id)
|
.outerjoin(Source, Source.id == ImageProvenance.source_id)
|
||||||
.where(ImageProvenance.image_record_id == image_id)
|
.where(ImageProvenance.image_record_id == image_id)
|
||||||
.order_by(ImageProvenance.captured_at.asc(),
|
.order_by(ImageProvenance.captured_at.asc(),
|
||||||
ImageProvenance.id.asc())
|
ImageProvenance.id.asc())
|
||||||
@@ -90,7 +94,7 @@ class ProvenanceService:
|
|||||||
"captured_at": ip.captured_at.isoformat()
|
"captured_at": ip.captured_at.isoformat()
|
||||||
if ip.captured_at else None,
|
if ip.captured_at else None,
|
||||||
"post": _post_dict(post),
|
"post": _post_dict(post),
|
||||||
"source": _source_dict(src),
|
"source": _source_dict(src) if src is not None else None,
|
||||||
"artist": _artist_dict(art),
|
"artist": _artist_dict(art),
|
||||||
}
|
}
|
||||||
for ip, post, src, art in rows
|
for ip, post, src, art in rows
|
||||||
@@ -99,10 +103,12 @@ class ProvenanceService:
|
|||||||
}
|
}
|
||||||
|
|
||||||
async def for_post(self, post_id: int) -> dict | None:
|
async def for_post(self, post_id: int) -> dict | None:
|
||||||
|
# Same LEFT JOIN to Source — get_post must succeed for a
|
||||||
|
# NULL-source post.
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Post, Source, Artist)
|
select(Post, Source, Artist)
|
||||||
.join(Source, Source.id == Post.source_id)
|
.join(Artist, Artist.id == Post.artist_id)
|
||||||
.join(Artist, Artist.id == Source.artist_id)
|
.outerjoin(Source, Source.id == Post.source_id)
|
||||||
.where(Post.id == post_id)
|
.where(Post.id == post_id)
|
||||||
)
|
)
|
||||||
row = (await self.session.execute(stmt)).first()
|
row = (await self.session.execute(stmt)).first()
|
||||||
@@ -111,7 +117,7 @@ class ProvenanceService:
|
|||||||
post, src, art = row
|
post, src, art = row
|
||||||
return {
|
return {
|
||||||
"post": _post_dict(post),
|
"post": _post_dict(post),
|
||||||
"source": _source_dict(src),
|
"source": _source_dict(src) if src is not None else None,
|
||||||
"artist": _artist_dict(art),
|
"artist": _artist_dict(art),
|
||||||
"attachments": await self._attachments_for_posts([post.id]),
|
"attachments": await self._attachments_for_posts([post.id]),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -86,7 +86,7 @@ def attempt_refetch(
|
|||||||
if src is None:
|
if src is None:
|
||||||
return {"status": "no_source"}
|
return {"status": "no_source"}
|
||||||
|
|
||||||
# Remove the bad copy so gallery-dl (skip_existing) re-fetches it on
|
# Remove the bad copy so gallery-dl's archive-skip re-fetches it on
|
||||||
# the source re-check instead of skipping the still-present corrupt
|
# the source re-check instead of skipping the still-present corrupt
|
||||||
# file.
|
# file.
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -59,9 +59,11 @@ class SourceRecord:
|
|||||||
config_overrides: dict | None
|
config_overrides: dict | None
|
||||||
last_checked_at: str | None
|
last_checked_at: str | None
|
||||||
last_error: str | None
|
last_error: str | None
|
||||||
|
error_type: str | None
|
||||||
check_interval_override: int | None
|
check_interval_override: int | None
|
||||||
consecutive_failures: int
|
consecutive_failures: int
|
||||||
next_check_at: str | None
|
next_check_at: str | None
|
||||||
|
backfill_runs_remaining: int
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
def to_dict(self) -> dict:
|
||||||
return {
|
return {
|
||||||
@@ -75,9 +77,11 @@ class SourceRecord:
|
|||||||
"config_overrides": self.config_overrides,
|
"config_overrides": self.config_overrides,
|
||||||
"last_checked_at": self.last_checked_at,
|
"last_checked_at": self.last_checked_at,
|
||||||
"last_error": self.last_error,
|
"last_error": self.last_error,
|
||||||
|
"error_type": self.error_type,
|
||||||
"check_interval_override": self.check_interval_override,
|
"check_interval_override": self.check_interval_override,
|
||||||
"consecutive_failures": self.consecutive_failures,
|
"consecutive_failures": self.consecutive_failures,
|
||||||
"next_check_at": self.next_check_at,
|
"next_check_at": self.next_check_at,
|
||||||
|
"backfill_runs_remaining": self.backfill_runs_remaining,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -85,6 +89,11 @@ class SourceRecord:
|
|||||||
|
|
||||||
_EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"}
|
_EDITABLE = {"enabled", "url", "config_overrides", "check_interval_override", "platform"}
|
||||||
|
|
||||||
|
# Plan #544 follow-up: newly created enabled sources pre-arm backfill so
|
||||||
|
# their first N polls walk gallery-dl's full post history with the longer
|
||||||
|
# timeout (matches the manual "Deep scan" button's default).
|
||||||
|
NEW_SOURCE_BACKFILL_RUNS = 3
|
||||||
|
|
||||||
|
|
||||||
class SourceService:
|
class SourceService:
|
||||||
def __init__(self, session: AsyncSession):
|
def __init__(self, session: AsyncSession):
|
||||||
@@ -137,9 +146,11 @@ class SourceService:
|
|||||||
config_overrides=source.config_overrides,
|
config_overrides=source.config_overrides,
|
||||||
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
||||||
last_error=source.last_error,
|
last_error=source.last_error,
|
||||||
|
error_type=source.error_type,
|
||||||
check_interval_override=source.check_interval_override,
|
check_interval_override=source.check_interval_override,
|
||||||
consecutive_failures=source.consecutive_failures or 0,
|
consecutive_failures=source.consecutive_failures or 0,
|
||||||
next_check_at=nxt.isoformat() if nxt else None,
|
next_check_at=nxt.isoformat() if nxt else None,
|
||||||
|
backfill_runs_remaining=source.backfill_runs_remaining or 0,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _row_to_record(self, source: Source) -> SourceRecord:
|
async def _row_to_record(self, source: Source) -> SourceRecord:
|
||||||
@@ -157,7 +168,7 @@ class SourceService:
|
|||||||
if artist_id is not None:
|
if artist_id is not None:
|
||||||
stmt = stmt.where(Source.artist_id == artist_id)
|
stmt = stmt.where(Source.artist_id == artist_id)
|
||||||
if not include_synthetic:
|
if not include_synthetic:
|
||||||
# Filesystem-import sidecar anchors (importer._source_for_sidecar)
|
# Pre-alembic-0030 sidecar synthetic anchors
|
||||||
# have url='sidecar:<platform>:<slug>' and exist only to give
|
# have url='sidecar:<platform>:<slug>' and exist only to give
|
||||||
# imported Posts a NOT-NULL Source FK. They aren't pollable
|
# imported Posts a NOT-NULL Source FK. They aren't pollable
|
||||||
# feeds; the Subscriptions UI used to render them as phantom
|
# feeds; the Subscriptions UI used to render them as phantom
|
||||||
@@ -201,10 +212,21 @@ class SourceService:
|
|||||||
select(func.count(Source.id)).where(Source.artist_id == artist_id)
|
select(func.count(Source.id)).where(Source.artist_id == artist_id)
|
||||||
)).scalar_one()
|
)).scalar_one()
|
||||||
|
|
||||||
|
# Plan #544 follow-up: a freshly added subscription has no archive
|
||||||
|
# yet, so the first few polls would walk the full post history in
|
||||||
|
# tick mode and trip exit:20 after ~20 contiguous archive hits —
|
||||||
|
# except there are none yet, so tick mode would walk forever and
|
||||||
|
# blow the wall-clock cap. Pre-arm backfill so the initial syncs
|
||||||
|
# use the longer timeout + skip:True walk. Tick mode resumes once
|
||||||
|
# the budget is spent or the queue drains.
|
||||||
|
# Disabled sources (incl. sidecar synthetics, url='sidecar:...')
|
||||||
|
# are never polled, so leave their counter at 0.
|
||||||
|
backfill_runs = NEW_SOURCE_BACKFILL_RUNS if enabled else 0
|
||||||
source = Source(
|
source = Source(
|
||||||
artist_id=artist_id, platform=platform, url=url,
|
artist_id=artist_id, platform=platform, url=url,
|
||||||
enabled=enabled, config_overrides=config_overrides,
|
enabled=enabled, config_overrides=config_overrides,
|
||||||
check_interval_override=check_interval_override,
|
check_interval_override=check_interval_override,
|
||||||
|
backfill_runs_remaining=backfill_runs,
|
||||||
)
|
)
|
||||||
self.session.add(source)
|
self.session.add(source)
|
||||||
try:
|
try:
|
||||||
@@ -264,6 +286,25 @@ class SourceService:
|
|||||||
await self.session.commit()
|
await self.session.commit()
|
||||||
return await self._row_to_record(source)
|
return await self._row_to_record(source)
|
||||||
|
|
||||||
|
async def set_backfill_runs(
|
||||||
|
self, source_id: int, runs: int,
|
||||||
|
) -> SourceRecord:
|
||||||
|
"""Plan #544: arm a source for backfill mode. The next `runs`
|
||||||
|
download runs will use gallery-dl's full-walk config (skip: True
|
||||||
|
+ 30-min timeout) instead of the catch-up default. Runs must be
|
||||||
|
1..10 — bigger is rejected to keep the operator from accidentally
|
||||||
|
setting a runaway budget."""
|
||||||
|
if not isinstance(runs, int) or runs < 1 or runs > 10:
|
||||||
|
raise ValueError("runs must be an integer in [1, 10]")
|
||||||
|
source = (await self.session.execute(
|
||||||
|
select(Source).where(Source.id == source_id)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
if source is None:
|
||||||
|
raise LookupError(f"source id={source_id} not found")
|
||||||
|
source.backfill_runs_remaining = runs
|
||||||
|
await self.session.commit()
|
||||||
|
return await self._row_to_record(source)
|
||||||
|
|
||||||
async def delete(self, source_id: int) -> None:
|
async def delete(self, source_id: int) -> None:
|
||||||
source = (await self.session.execute(
|
source = (await self.session.execute(
|
||||||
select(Source).where(Source.id == source_id)
|
select(Source).where(Source.id == source_id)
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from dataclasses import dataclass
|
|||||||
|
|
||||||
from sqlalchemy import and_, case, exists, func, select, text, update
|
from sqlalchemy import and_, case, exists, func, select, text, update
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ..models import Tag, TagKind, image_tag
|
from ..models import Tag, TagKind, image_tag
|
||||||
@@ -86,9 +87,12 @@ class TagService:
|
|||||||
f"fandom_id {fandom_id} does not reference a fandom tag"
|
f"fandom_id {fandom_id} does not reference a fandom tag"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Upsert via INSERT ... ON CONFLICT DO NOTHING. We can't use the
|
# Audit 2026-06-02: race-safe upsert via savepoint +
|
||||||
# uniqueness index name directly (it's a partial coalesce-based
|
# IntegrityError recovery. The partial uniqueness index on
|
||||||
# expression), so we re-select after insert.
|
# (name, kind, COALESCE(fandom_id, -1)) catches concurrent
|
||||||
|
# inserts; without the savepoint the outer transaction would
|
||||||
|
# poison and the calling request crashes. Mirrors
|
||||||
|
# importer._get_or_create.
|
||||||
stmt = (
|
stmt = (
|
||||||
select(Tag)
|
select(Tag)
|
||||||
.where(Tag.name == name)
|
.where(Tag.name == name)
|
||||||
@@ -101,10 +105,16 @@ class TagService:
|
|||||||
if existing:
|
if existing:
|
||||||
return existing
|
return existing
|
||||||
|
|
||||||
new_tag = Tag(name=name, kind=kind, fandom_id=fandom_id)
|
sp = await self.session.begin_nested()
|
||||||
self.session.add(new_tag)
|
try:
|
||||||
await self.session.flush()
|
new_tag = Tag(name=name, kind=kind, fandom_id=fandom_id)
|
||||||
return new_tag
|
self.session.add(new_tag)
|
||||||
|
await self.session.flush()
|
||||||
|
await sp.commit()
|
||||||
|
return new_tag
|
||||||
|
except IntegrityError:
|
||||||
|
await sp.rollback()
|
||||||
|
return (await self.session.execute(stmt)).scalar_one()
|
||||||
|
|
||||||
async def autocomplete(
|
async def autocomplete(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -240,8 +240,11 @@ def prune_backups() -> dict:
|
|||||||
|
|
||||||
Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
|
Returns {"db_deleted": N, "images_deleted": M, "files_unlinked": K}.
|
||||||
Tagged rows (tag IS NOT NULL) are never pruned.
|
Tagged rows (tag IS NOT NULL) are never pruned.
|
||||||
Status='running' / 'restoring' rows are never pruned (recovery
|
Status='running' / 'restoring' rows are never pruned — the
|
||||||
sweep from FC-3i handles those via task_run).
|
recover_stalled_backup_runs sweep flips truly-stuck ones to
|
||||||
|
'error' first. (Earlier docstring claimed the FC-3i TaskRun sweep
|
||||||
|
handled those, but TaskRun cleanup never touched BackupRun rows.
|
||||||
|
Audit 2026-06-02 added the dedicated sweep.)
|
||||||
"""
|
"""
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
counts = {"db_deleted": 0, "images_deleted": 0, "files_unlinked": 0}
|
||||||
|
|||||||
@@ -7,14 +7,17 @@ from datetime import UTC, datetime, timedelta
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from sqlalchemy import and_, delete, or_, select, update
|
from sqlalchemy import Integer, and_, cast, delete, func, or_, select, update
|
||||||
|
|
||||||
from ..celery_app import celery
|
from ..celery_app import celery
|
||||||
from ..models import (
|
from ..models import (
|
||||||
|
BackupRun,
|
||||||
DownloadEvent,
|
DownloadEvent,
|
||||||
ImageRecord,
|
ImageRecord,
|
||||||
|
ImportBatch,
|
||||||
ImportSettings,
|
ImportSettings,
|
||||||
ImportTask,
|
ImportTask,
|
||||||
|
LibraryAuditRun,
|
||||||
Source,
|
Source,
|
||||||
TaskRun,
|
TaskRun,
|
||||||
)
|
)
|
||||||
@@ -55,6 +58,29 @@ FFPROBE_TIMEOUT_SECONDS = 10
|
|||||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
|
|
||||||
|
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
||||||
|
# > the entity's longest legitimate runtime (its task's time_limit + a
|
||||||
|
# small buffer) so the sweep never flags in-flight work.
|
||||||
|
#
|
||||||
|
# Backups: images backup has time_limit=23400s (6.5h). 7h covers it
|
||||||
|
# with a 30-min buffer; db backup at 12 min hard limit fits trivially.
|
||||||
|
BACKUP_STALL_THRESHOLD_MINUTES = 7 * 60
|
||||||
|
# Library audit: scan_library_for_rule has time_limit=7500s (2h5m).
|
||||||
|
# 2h15m gives a 10-min buffer.
|
||||||
|
LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES = 135
|
||||||
|
# Import batches finalize only after every child ImportTask hits a
|
||||||
|
# terminal state. The recovery sweep targets the case where every
|
||||||
|
# task is done but the batch never got its closing UPDATE
|
||||||
|
# (orchestrator crashed at the wrong instant). 2h is well past any
|
||||||
|
# realistic single-batch import.
|
||||||
|
IMPORT_BATCH_STALL_THRESHOLD_MINUTES = 120
|
||||||
|
|
||||||
|
# Retention windows (terminal rows older than these get deleted by
|
||||||
|
# the daily prune sweeps). 30 days = operator-flagged "useful for
|
||||||
|
# triage for a few weeks, then noise."
|
||||||
|
LIBRARY_AUDIT_KEEP_DAYS = 30
|
||||||
|
IMPORT_BATCH_KEEP_DAYS = 30
|
||||||
|
|
||||||
# Overrides for recover_stalled_task_runs (the TaskRun 'running' sweep).
|
# Overrides for recover_stalled_task_runs (the TaskRun 'running' sweep).
|
||||||
# Tasks/queues that legitimately run longer than the default 5-min
|
# Tasks/queues that legitimately run longer than the default 5-min
|
||||||
# threshold need their own larger value, else the sweep marks in-flight
|
# threshold need their own larger value, else the sweep marks in-flight
|
||||||
@@ -69,9 +95,24 @@ TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
|||||||
# files); time_limit=2100.
|
# files); time_limit=2100.
|
||||||
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
"ml": 25,
|
"ml": 25,
|
||||||
|
# Audit 2026-06-02 — maintenance/scan queues run tasks that
|
||||||
|
# legitimately exceed the 5-min default (verify_integrity at 70m
|
||||||
|
# hard, scan_directory at 70m hard, apply_allowlist_tags /
|
||||||
|
# recompute_centroids / backfill_phash at 35m hard). 75 min lives
|
||||||
|
# above the longest of those and the per-task overrides below
|
||||||
|
# cover the outliers (backups, library audit).
|
||||||
|
"maintenance": 75,
|
||||||
|
"scan": 75,
|
||||||
}
|
}
|
||||||
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
"backend.app.tasks.import_file.import_archive_file": 40,
|
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||||
|
# Backup images runs hours, not minutes (6.5h hard limit). The
|
||||||
|
# task-name override beats the queue's 75-min default so a
|
||||||
|
# legitimately-running backup isn't flagged.
|
||||||
|
"backend.app.tasks.backup.backup_images_task": 420,
|
||||||
|
"backend.app.tasks.backup.restore_images_task": 420,
|
||||||
|
# Library audit scans the full library — 2h hard limit.
|
||||||
|
"backend.app.tasks.library_audit.scan_library_for_rule": 130,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -182,6 +223,11 @@ def recover_interrupted_tasks() -> int:
|
|||||||
.where(ImportTask.created_at < orphan_cutoff)
|
.where(ImportTask.created_at < orphan_cutoff)
|
||||||
.values(
|
.values(
|
||||||
status="failed",
|
status="failed",
|
||||||
|
# Without finished_at, cleanup_old_tasks (`WHERE
|
||||||
|
# finished_at < cutoff`) never reaps these rows —
|
||||||
|
# orphan-swept rows would become permanent table
|
||||||
|
# tenants. Audit 2026-06-02.
|
||||||
|
finished_at=now,
|
||||||
error=(
|
error=(
|
||||||
"orphan pending/queued swept by recover_interrupted_tasks "
|
"orphan pending/queued swept by recover_interrupted_tasks "
|
||||||
"(scanner likely crashed mid-enqueue); retry via "
|
"(scanner likely crashed mid-enqueue); retry via "
|
||||||
@@ -278,6 +324,14 @@ def recover_stalled_task_runs() -> int:
|
|||||||
f"no completion signal received within {minutes} min"
|
f"no completion signal received within {minutes} min"
|
||||||
),
|
),
|
||||||
finished_at=now,
|
finished_at=now,
|
||||||
|
# Matches celery_signals.finalize's
|
||||||
|
# int((now - started_at).total_seconds() * 1000)
|
||||||
|
# — sweep-closed rows now carry duration like
|
||||||
|
# normally-finalized rows. Audit 2026-06-02.
|
||||||
|
duration_ms=cast(
|
||||||
|
func.extract("epoch", now - TaskRun.started_at) * 1000,
|
||||||
|
Integer,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
for w in extra_where:
|
for w in extra_where:
|
||||||
@@ -347,7 +401,12 @@ def prune_task_runs() -> dict:
|
|||||||
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.backfill_phash")
|
@celery.task(
|
||||||
|
name="backend.app.tasks.maintenance.backfill_phash",
|
||||||
|
# Audit 2026-06-02 — keyset-paginated phash recompute over the whole
|
||||||
|
# library; legitimately runs >5 min on large libraries.
|
||||||
|
soft_time_limit=1800, time_limit=2100,
|
||||||
|
)
|
||||||
def backfill_phash() -> int:
|
def backfill_phash() -> int:
|
||||||
"""Recompute phash for stored images that have none (imported before
|
"""Recompute phash for stored images that have none (imported before
|
||||||
FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill,
|
FC-2d-i+ii). Keyset-paginated by id (restart-safe), NULL-only fill,
|
||||||
@@ -425,7 +484,13 @@ def _verify_one(path: Path, expected_sha: str, mime: str, sha_fn) -> str:
|
|||||||
return "failed_verification"
|
return "failed_verification"
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.verify_integrity")
|
@celery.task(
|
||||||
|
name="backend.app.tasks.maintenance.verify_integrity",
|
||||||
|
# Audit 2026-06-02 — full library sha256 + decode probe; on 100k-image
|
||||||
|
# libraries this runs an hour or more. Match the maintenance queue's
|
||||||
|
# recovery threshold (75 min) with 30s buffer below.
|
||||||
|
soft_time_limit=3600, time_limit=4200,
|
||||||
|
)
|
||||||
def verify_integrity() -> int:
|
def verify_integrity() -> int:
|
||||||
"""Verify every ImageRecord file: sha256 recompute + decode/probe
|
"""Verify every ImageRecord file: sha256 recompute + decode/probe
|
||||||
(PIL for images; ffprobe for videos). Writes integrity_status
|
(PIL for images; ffprobe for videos). Writes integrity_status
|
||||||
@@ -521,6 +586,156 @@ def recover_stalled_download_events() -> int:
|
|||||||
return events_recovered
|
return events_recovered
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
|
||||||
|
def recover_stalled_backup_runs() -> int:
|
||||||
|
"""Flip BackupRun rows stuck in running/restoring past the hard limit
|
||||||
|
to error. Audit 2026-06-02.
|
||||||
|
|
||||||
|
prune_backups (FC-3h) used to claim the FC-3i task_run sweep handled
|
||||||
|
these — but that sweep only flips TaskRun rows, not the BackupRun
|
||||||
|
artifact rows. A SIGKILL'd backup left BackupRun stuck forever
|
||||||
|
(dashboard showed phantom in-flight backups, keep_last_n offset
|
||||||
|
arithmetic skewed because zombies sat outside the ok/error window).
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
cutoff = now - timedelta(minutes=BACKUP_STALL_THRESHOLD_MINUTES)
|
||||||
|
msg = (
|
||||||
|
f"stranded by recovery sweep (no terminal status after "
|
||||||
|
f"{BACKUP_STALL_THRESHOLD_MINUTES // 60}h)"
|
||||||
|
)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
result = session.execute(
|
||||||
|
update(BackupRun)
|
||||||
|
.where(BackupRun.status.in_(["running", "restoring"]))
|
||||||
|
.where(BackupRun.started_at < cutoff)
|
||||||
|
.values(status="error", finished_at=now, error=msg)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
recovered = result.rowcount or 0
|
||||||
|
if recovered:
|
||||||
|
log.info("recover_stalled_backup_runs: recovered %d rows", recovered)
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_library_audit_runs")
|
||||||
|
def recover_stalled_library_audit_runs() -> int:
|
||||||
|
"""Flip LibraryAuditRun rows stuck in running past the hard limit
|
||||||
|
to error. Audit 2026-06-02.
|
||||||
|
|
||||||
|
LibraryAuditRun.status='running' was protected by an exclusive
|
||||||
|
guard in start_audit_run — a SIGKILL'd run would block all future
|
||||||
|
audits until manual DB surgery. (The guard is now age-aware, but
|
||||||
|
this sweep is what makes that work in practice.)
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
cutoff = now - timedelta(minutes=LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES)
|
||||||
|
msg = (
|
||||||
|
f"stranded by recovery sweep (no terminal status after "
|
||||||
|
f"{LIBRARY_AUDIT_STALL_THRESHOLD_MINUTES} min)"
|
||||||
|
)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
result = session.execute(
|
||||||
|
update(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.status == "running")
|
||||||
|
.where(LibraryAuditRun.started_at < cutoff)
|
||||||
|
.values(status="error", finished_at=now, error=msg)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
recovered = result.rowcount or 0
|
||||||
|
if recovered:
|
||||||
|
log.info(
|
||||||
|
"recover_stalled_library_audit_runs: recovered %d rows", recovered,
|
||||||
|
)
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_import_batches")
|
||||||
|
def recover_stalled_import_batches() -> int:
|
||||||
|
"""Finalize ImportBatch rows stuck in running past the hard limit
|
||||||
|
when NO outstanding ImportTask remains. Audit 2026-06-02.
|
||||||
|
|
||||||
|
A batch row finalizes only after every child task hits a terminal
|
||||||
|
state. The orphan case: scanner crashed between the last task's
|
||||||
|
completion and the batch's closing UPDATE. The
|
||||||
|
`/api/import/status` route then surfaces the batch as 'active'
|
||||||
|
indefinitely while `/api/system/stats` (which uses the same
|
||||||
|
EXISTS predicate we apply below) correctly returns null.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
cutoff = now - timedelta(minutes=IMPORT_BATCH_STALL_THRESHOLD_MINUTES)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
# Batches still 'running' past the cutoff whose tasks are all
|
||||||
|
# terminal — there's no outstanding work, so flip the batch
|
||||||
|
# too. Mirrors the EXISTS predicate the active-batch surfaces use.
|
||||||
|
result = session.execute(
|
||||||
|
update(ImportBatch)
|
||||||
|
.where(ImportBatch.status == "running")
|
||||||
|
.where(ImportBatch.started_at < cutoff)
|
||||||
|
.where(
|
||||||
|
~select(ImportTask.id)
|
||||||
|
.where(
|
||||||
|
ImportTask.batch_id == ImportBatch.id,
|
||||||
|
ImportTask.status.in_(["pending", "queued", "processing"]),
|
||||||
|
)
|
||||||
|
.exists()
|
||||||
|
)
|
||||||
|
.values(status="complete", finished_at=now)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
recovered = result.rowcount or 0
|
||||||
|
if recovered:
|
||||||
|
log.info(
|
||||||
|
"recover_stalled_import_batches: finalized %d zombie batches",
|
||||||
|
recovered,
|
||||||
|
)
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.prune_library_audit_runs")
|
||||||
|
def prune_library_audit_runs() -> int:
|
||||||
|
"""Daily retention: delete terminal LibraryAuditRun rows older than
|
||||||
|
LIBRARY_AUDIT_KEEP_DAYS. Never touches 'running'. Audit 2026-06-02.
|
||||||
|
|
||||||
|
Audit rows carry matched_ids JSONB blobs that can hold tens of
|
||||||
|
thousands of ids; without retention these accumulate.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cutoff = datetime.now(UTC) - timedelta(days=LIBRARY_AUDIT_KEEP_DAYS)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
result = session.execute(
|
||||||
|
delete(LibraryAuditRun)
|
||||||
|
.where(LibraryAuditRun.status.in_(["ready", "applied", "cancelled", "error"]))
|
||||||
|
.where(LibraryAuditRun.finished_at < cutoff)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(name="backend.app.tasks.maintenance.prune_import_batches")
|
||||||
|
def prune_import_batches() -> int:
|
||||||
|
"""Daily retention: delete terminal ImportBatch rows older than
|
||||||
|
IMPORT_BATCH_KEEP_DAYS. Cascade-deletes child ImportTask rows via
|
||||||
|
the model relationship. Never touches 'running'. Audit 2026-06-02.
|
||||||
|
"""
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
cutoff = datetime.now(UTC) - timedelta(days=IMPORT_BATCH_KEEP_DAYS)
|
||||||
|
with SessionLocal() as session:
|
||||||
|
# ORM-level delete here (not Core delete) so the
|
||||||
|
# ImportBatch->tasks cascade fires; Core delete would skip it.
|
||||||
|
old_batches = session.execute(
|
||||||
|
select(ImportBatch)
|
||||||
|
.where(ImportBatch.status.in_(["complete", "cancelled"]))
|
||||||
|
.where(ImportBatch.finished_at < cutoff)
|
||||||
|
).scalars().all()
|
||||||
|
for batch in old_batches:
|
||||||
|
session.delete(batch)
|
||||||
|
session.commit()
|
||||||
|
return len(old_batches)
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events")
|
@celery.task(name="backend.app.tasks.maintenance.cleanup_old_download_events")
|
||||||
def cleanup_old_download_events() -> int:
|
def cleanup_old_download_events() -> int:
|
||||||
"""FC-3d: delete terminal DownloadEvent rows older than the configured
|
"""FC-3d: delete terminal DownloadEvent rows older than the configured
|
||||||
|
|||||||
+15
-2
@@ -212,7 +212,14 @@ def backfill(self) -> int:
|
|||||||
return enqueued
|
return enqueued
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.ml.apply_allowlist_tags", bind=True)
|
@celery.task(
|
||||||
|
name="backend.app.tasks.ml.apply_allowlist_tags",
|
||||||
|
bind=True,
|
||||||
|
# Audit 2026-06-02 — the full-sweep mode (neither tag_id nor image_id)
|
||||||
|
# is O(images × allowlist) and legitimately runs >5 min on large
|
||||||
|
# libraries. Cap matches the maintenance queue's recovery threshold.
|
||||||
|
soft_time_limit=1800, time_limit=2100,
|
||||||
|
)
|
||||||
def apply_allowlist_tags(self, tag_id: int | None = None,
|
def apply_allowlist_tags(self, tag_id: int | None = None,
|
||||||
image_id: int | None = None) -> int:
|
image_id: int | None = None) -> int:
|
||||||
"""Retroactively apply allowlisted tags.
|
"""Retroactively apply allowlisted tags.
|
||||||
@@ -341,7 +348,13 @@ def recompute_centroid(self, tag_id: int) -> bool:
|
|||||||
return asyncio.run(_run())
|
return asyncio.run(_run())
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.ml.recompute_centroids", bind=True)
|
@celery.task(
|
||||||
|
name="backend.app.tasks.ml.recompute_centroids",
|
||||||
|
bind=True,
|
||||||
|
# Audit 2026-06-02 — drifted-centroid rebuild over potentially
|
||||||
|
# hundreds of tags.
|
||||||
|
soft_time_limit=1800, time_limit=2100,
|
||||||
|
)
|
||||||
def recompute_centroids(self) -> int:
|
def recompute_centroids(self) -> int:
|
||||||
"""Daily: find drifted centroids, enqueue recompute_centroid for each."""
|
"""Daily: find drifted centroids, enqueue recompute_centroid for each."""
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|||||||
@@ -35,7 +35,15 @@ def _iter_import_files(import_root: Path):
|
|||||||
yield entry
|
yield entry
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.scan.scan_directory", bind=True)
|
@celery.task(
|
||||||
|
name="backend.app.tasks.scan.scan_directory",
|
||||||
|
bind=True,
|
||||||
|
# Audit 2026-06-02 — large libraries make the scan legitimately long.
|
||||||
|
# Hard cap at 70 min so the corresponding QUEUE_STUCK_THRESHOLD_MINUTES
|
||||||
|
# ("scan") of 75 min always wins; soft limit gives the task a clean
|
||||||
|
# exit window before SIGKILL.
|
||||||
|
soft_time_limit=3600, time_limit=4200,
|
||||||
|
)
|
||||||
def scan_directory(self, triggered_by: str = "manual",
|
def scan_directory(self, triggered_by: str = "manual",
|
||||||
mode: str = "quick") -> int:
|
mode: str = "quick") -> int:
|
||||||
"""Walks the import root and creates ImportTasks. `mode` is 'quick'
|
"""Walks the import root and creates ImportTasks. `mode` is 'quick'
|
||||||
|
|||||||
@@ -20,14 +20,32 @@ IMAGES_ROOT = Path("/images")
|
|||||||
THUMB_MAGIC_JPEG = b"\xff\xd8\xff"
|
THUMB_MAGIC_JPEG = b"\xff\xd8\xff"
|
||||||
THUMB_MAGIC_PNG = b"\x89PNG\r\n\x1a\n"
|
THUMB_MAGIC_PNG = b"\x89PNG\r\n\x1a\n"
|
||||||
|
|
||||||
|
# Minimum file size for a thumbnail to count as valid. Anything smaller
|
||||||
|
# is almost certainly truncated/corrupt — a legitimate 400×400 JPEG@85
|
||||||
|
# bottoms out around 2KB even on a solid-color image; 400×400 PNG starts
|
||||||
|
# around 1KB. 256 bytes is well below any real thumbnail and well above
|
||||||
|
# header-only corrupt files (~8-12 bytes). Operator-flagged 2026-06-01:
|
||||||
|
# header-only corrupt files were silently passing the magic-byte check
|
||||||
|
# and backfill counted them as "ok" — so broken-image tiles in the UI
|
||||||
|
# never got regenerated even after running backfill.
|
||||||
|
MIN_THUMB_BYTES = 256
|
||||||
|
|
||||||
|
|
||||||
def _thumb_is_valid(path: Path) -> bool:
|
def _thumb_is_valid(path: Path) -> bool:
|
||||||
"""Return True iff `path` exists and starts with a JPEG or PNG magic header.
|
"""Return True iff `path` exists, starts with a JPEG or PNG magic
|
||||||
|
header, AND is at least MIN_THUMB_BYTES on disk.
|
||||||
|
|
||||||
The on-disk thumbnail format is set by services/thumbnailer.py — JPEG for
|
The on-disk thumbnail format is set by services/thumbnailer.py — JPEG
|
||||||
opaque sources, PNG for alpha sources. Anything else (missing file, OSError,
|
for opaque sources, PNG for alpha sources. Anything else (missing
|
||||||
truncated, wrong magic) is invalid.
|
file, OSError, truncated below the size floor, wrong magic) is
|
||||||
|
invalid and gets re-enqueued.
|
||||||
"""
|
"""
|
||||||
|
try:
|
||||||
|
size = path.stat().st_size
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
if size < MIN_THUMB_BYTES:
|
||||||
|
return False
|
||||||
try:
|
try:
|
||||||
with path.open("rb") as f:
|
with path.open("rb") as f:
|
||||||
head = f.read(12)
|
head = f.read(12)
|
||||||
@@ -42,6 +60,59 @@ def _thumb_is_valid(path: Path) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _run_backfill_scan() -> dict:
|
||||||
|
"""Synchronous scan logic shared by the Celery task and the API
|
||||||
|
endpoint. Returns {enqueued, ok, regenerated, scanned}.
|
||||||
|
|
||||||
|
Operator-flagged 2026-06-01: the original task was fire-and-forget,
|
||||||
|
so the admin UI couldn't show what backfill actually found —
|
||||||
|
operator saw \"Enqueued.\" with no counts and assumed nothing was
|
||||||
|
happening. Now the API runs this synchronously and returns the
|
||||||
|
real numbers; the periodic Celery task wraps it too."""
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
enqueued = 0
|
||||||
|
ok = 0
|
||||||
|
regenerated = 0
|
||||||
|
scanned = 0
|
||||||
|
last_id = 0
|
||||||
|
with SessionLocal() as session:
|
||||||
|
while True:
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord.id, ImageRecord.thumbnail_path)
|
||||||
|
.where(ImageRecord.id > last_id)
|
||||||
|
.order_by(ImageRecord.id.asc())
|
||||||
|
.limit(500)
|
||||||
|
).all()
|
||||||
|
if not rows:
|
||||||
|
break
|
||||||
|
scanned += len(rows)
|
||||||
|
for image_id, thumb_path in rows:
|
||||||
|
if thumb_path is None:
|
||||||
|
generate_thumbnail.delay(image_id)
|
||||||
|
enqueued += 1
|
||||||
|
elif _thumb_is_valid(Path(thumb_path)):
|
||||||
|
ok += 1
|
||||||
|
else:
|
||||||
|
session.execute(
|
||||||
|
update(ImageRecord)
|
||||||
|
.where(ImageRecord.id == image_id)
|
||||||
|
.values(thumbnail_path=None)
|
||||||
|
)
|
||||||
|
generate_thumbnail.delay(image_id)
|
||||||
|
enqueued += 1
|
||||||
|
regenerated += 1
|
||||||
|
session.commit()
|
||||||
|
last_id = rows[-1][0]
|
||||||
|
return {
|
||||||
|
"scanned": scanned,
|
||||||
|
"enqueued": enqueued,
|
||||||
|
"ok": ok,
|
||||||
|
"regenerated": regenerated,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
@celery.task(
|
||||||
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
name="backend.app.tasks.thumbnail.generate_thumbnail",
|
||||||
bind=True,
|
bind=True,
|
||||||
@@ -84,48 +155,15 @@ def backfill_thumbnails(self) -> dict:
|
|||||||
"""Scan ImageRecord and enqueue generate_thumbnail for rows whose
|
"""Scan ImageRecord and enqueue generate_thumbnail for rows whose
|
||||||
thumbnail is missing, gone from disk, or has wrong magic bytes.
|
thumbnail is missing, gone from disk, or has wrong magic bytes.
|
||||||
|
|
||||||
Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path for
|
Keyset paginates by id ASC, page size 500. NULLs out thumbnail_path
|
||||||
rows that point at a missing or corrupt file before enqueueing — keeps
|
for rows that point at a missing or corrupt file before enqueueing —
|
||||||
the DB self-consistent on partial runs and makes re-runs safe.
|
keeps the DB self-consistent on partial runs and makes re-runs safe.
|
||||||
|
|
||||||
Returns {"enqueued": N, "ok": M, "regenerated": K} where:
|
Returns {scanned, enqueued, ok, regenerated} where:
|
||||||
- enqueued = total generate_thumbnail.delay() calls
|
- scanned = total rows examined
|
||||||
- ok = rows whose existing thumbnail file is valid (skipped)
|
- enqueued = total generate_thumbnail.delay() calls
|
||||||
- regenerated = subset of enqueued that had a non-NULL thumbnail_path
|
- ok = rows whose existing thumbnail file is valid (skipped)
|
||||||
cleared (i.e. missing + corrupt)
|
- regenerated = subset of enqueued that had a non-NULL
|
||||||
|
thumbnail_path cleared (i.e. missing + corrupt)
|
||||||
"""
|
"""
|
||||||
from sqlalchemy import select, update
|
return _run_backfill_scan()
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
enqueued = 0
|
|
||||||
ok = 0
|
|
||||||
regenerated = 0
|
|
||||||
last_id = 0
|
|
||||||
with SessionLocal() as session:
|
|
||||||
while True:
|
|
||||||
rows = session.execute(
|
|
||||||
select(ImageRecord.id, ImageRecord.thumbnail_path)
|
|
||||||
.where(ImageRecord.id > last_id)
|
|
||||||
.order_by(ImageRecord.id.asc())
|
|
||||||
.limit(500)
|
|
||||||
).all()
|
|
||||||
if not rows:
|
|
||||||
break
|
|
||||||
for image_id, thumb_path in rows:
|
|
||||||
if thumb_path is None:
|
|
||||||
generate_thumbnail.delay(image_id)
|
|
||||||
enqueued += 1
|
|
||||||
elif _thumb_is_valid(Path(thumb_path)):
|
|
||||||
ok += 1
|
|
||||||
else:
|
|
||||||
session.execute(
|
|
||||||
update(ImageRecord)
|
|
||||||
.where(ImageRecord.id == image_id)
|
|
||||||
.values(thumbnail_path=None)
|
|
||||||
)
|
|
||||||
generate_thumbnail.delay(image_id)
|
|
||||||
enqueued += 1
|
|
||||||
regenerated += 1
|
|
||||||
session.commit()
|
|
||||||
last_id = rows[-1][0]
|
|
||||||
return {"enqueued": enqueued, "ok": ok, "regenerated": regenerated}
|
|
||||||
|
|||||||
@@ -194,9 +194,20 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
if (platform.authType === 'cookies') {
|
if (platform.authType === 'cookies') {
|
||||||
const cookies = await extractCookiesForPlatform(key);
|
const cookies = await extractCookiesForPlatform(key);
|
||||||
if (cookies.length === 0) return { error: 'No cookies found — log in first.' };
|
if (cookies.length === 0) return { error: 'No cookies found — log in first.' };
|
||||||
|
// Verify the captured cookies are actually live BEFORE
|
||||||
|
// uploading. Skips upload on confirmed-stale sessions so we
|
||||||
|
// don't overwrite FC-side credentials with garbage. Platforms
|
||||||
|
// without a verify config (verify.ok === null) fall through
|
||||||
|
// to upload as before.
|
||||||
|
const v = await verifyCookiesForPlatform(key);
|
||||||
|
if (v.ok === false) {
|
||||||
|
return {
|
||||||
|
error: `Captured ${cookies.length} ${platform.name} cookies but they don't appear authenticated (${v.reason}). Log in again in this browser, then retry.`,
|
||||||
|
};
|
||||||
|
}
|
||||||
const data = toNetscapeFormat(cookies);
|
const data = toNetscapeFormat(cookies);
|
||||||
await api.uploadCredentials(key, 'cookies', data);
|
await api.uploadCredentials(key, 'cookies', data);
|
||||||
return { success: true, cookieCount: cookies.length };
|
return { success: true, cookieCount: cookies.length, verified: v.ok === true };
|
||||||
}
|
}
|
||||||
if (key === 'discord') {
|
if (key === 'discord') {
|
||||||
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
||||||
@@ -229,8 +240,13 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
results[key] = { skipped: true, reason: 'no cookies' };
|
results[key] = { skipped: true, reason: 'no cookies' };
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
const v = await verifyCookiesForPlatform(key);
|
||||||
|
if (v.ok === false) {
|
||||||
|
results[key] = { error: `verify failed: ${v.reason}` };
|
||||||
|
continue;
|
||||||
|
}
|
||||||
await api.uploadCredentials(key, 'cookies', toNetscapeFormat(cookies));
|
await api.uploadCredentials(key, 'cookies', toNetscapeFormat(cookies));
|
||||||
results[key] = { success: true, cookieCount: cookies.length };
|
results[key] = { success: true, cookieCount: cookies.length, verified: v.ok === true };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
results[key] = { error: e.message };
|
results[key] = { error: e.message };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -76,3 +76,38 @@ async function getCookieCount(platformKey) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Verify cookies are live by hitting an authenticated endpoint with the
|
||||||
|
* browser's current cookie jar. Returns:
|
||||||
|
* { ok: true, status } — verified
|
||||||
|
* { ok: false, status, reason } — endpoint said we're not logged in
|
||||||
|
* { ok: null, reason } — no verify config for this platform; caller
|
||||||
|
* should treat as "verify not available,
|
||||||
|
* proceed with upload"
|
||||||
|
*
|
||||||
|
* Implementation note: extensions with `host_permissions` for the target
|
||||||
|
* domain get the user's cookies auto-attached to fetch() — same set
|
||||||
|
* gallery-dl will later use on the backend.
|
||||||
|
*/
|
||||||
|
async function verifyCookiesForPlatform(platformKey) {
|
||||||
|
const platform = PLATFORMS[platformKey];
|
||||||
|
if (!platform) return { ok: false, reason: `Unknown platform: ${platformKey}` };
|
||||||
|
if (!platform.verify) return { ok: null, reason: 'verify-not-configured' };
|
||||||
|
|
||||||
|
const { url, method, okStatuses } = platform.verify;
|
||||||
|
let resp;
|
||||||
|
try {
|
||||||
|
resp = await fetch(url, { method, credentials: 'include', cache: 'no-store' });
|
||||||
|
} catch (e) {
|
||||||
|
return { ok: false, reason: `Verify request failed: ${e.message}` };
|
||||||
|
}
|
||||||
|
if (okStatuses.includes(resp.status)) {
|
||||||
|
return { ok: true, status: resp.status };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: resp.status,
|
||||||
|
reason: `${url} returned HTTP ${resp.status} — session looks stale or logged out`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,6 +13,13 @@ const PLATFORMS = {
|
|||||||
authType: 'cookies',
|
authType: 'cookies',
|
||||||
color: '#FF424D',
|
color: '#FF424D',
|
||||||
urlPattern: /^https?:\/\/(www\.)?patreon\.com/,
|
urlPattern: /^https?:\/\/(www\.)?patreon\.com/,
|
||||||
|
// Patreon's `/api/current_user` returns 200 + the logged-in user
|
||||||
|
// when authenticated, 401 otherwise. Cheapest definitive check.
|
||||||
|
verify: {
|
||||||
|
url: 'https://www.patreon.com/api/current_user',
|
||||||
|
method: 'GET',
|
||||||
|
okStatuses: [200],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
subscribestar: {
|
subscribestar: {
|
||||||
name: 'SubscribeStar',
|
name: 'SubscribeStar',
|
||||||
@@ -26,6 +33,9 @@ const PLATFORMS = {
|
|||||||
authType: 'cookies',
|
authType: 'cookies',
|
||||||
color: '#FFD700',
|
color: '#FFD700',
|
||||||
urlPattern: /^https?:\/\/(www\.)?subscribestar\.(com|adult)/,
|
urlPattern: /^https?:\/\/(www\.)?subscribestar\.(com|adult)/,
|
||||||
|
// No known stable auth-required endpoint that returns a definitive
|
||||||
|
// status code; skipping verify so we don't false-positive-fail
|
||||||
|
// good cookies. Operator can add later if a clean endpoint surfaces.
|
||||||
},
|
},
|
||||||
hentaifoundry: {
|
hentaifoundry: {
|
||||||
name: 'Hentai Foundry',
|
name: 'Hentai Foundry',
|
||||||
@@ -33,6 +43,14 @@ const PLATFORMS = {
|
|||||||
authType: 'cookies',
|
authType: 'cookies',
|
||||||
color: '#9C27B0',
|
color: '#9C27B0',
|
||||||
urlPattern: /^https?:\/\/(www\.)?hentai-foundry\.com/,
|
urlPattern: /^https?:\/\/(www\.)?hentai-foundry\.com/,
|
||||||
|
// Mirror gallery-dl's _init_site_filters: HEAD on `?enterAgree=1`.
|
||||||
|
// Logged in → 200, logged out → 401. Catches the exact failure mode
|
||||||
|
// the backend extractor would hit later.
|
||||||
|
verify: {
|
||||||
|
url: 'https://www.hentai-foundry.com/?enterAgree=1',
|
||||||
|
method: 'HEAD',
|
||||||
|
okStatuses: [200],
|
||||||
|
},
|
||||||
},
|
},
|
||||||
discord: {
|
discord: {
|
||||||
name: 'Discord',
|
name: 'Discord',
|
||||||
@@ -56,6 +74,9 @@ const PLATFORMS = {
|
|||||||
authType: 'cookies',
|
authType: 'cookies',
|
||||||
color: '#05CC47',
|
color: '#05CC47',
|
||||||
urlPattern: /^https?:\/\/(www\.)?deviantart\.com/,
|
urlPattern: /^https?:\/\/(www\.)?deviantart\.com/,
|
||||||
|
// DA's logged-in-only endpoints sit behind their internal _napi
|
||||||
|
// namespace which shifts; skipping verify until a stable check
|
||||||
|
// surfaces. Same posture as SubscribeStar.
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest_version": 3,
|
"manifest_version": 3,
|
||||||
"name": "FabledCurator",
|
"name": "FabledCurator",
|
||||||
"version": "1.0.6",
|
"version": "1.0.7",
|
||||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||||
|
|
||||||
"browser_specific_settings": {
|
"browser_specific_settings": {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "fabledcurator-extension",
|
"name": "fabledcurator-extension",
|
||||||
"version": "1.0.6",
|
"version": "1.0.7",
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "Firefox extension for FabledCurator",
|
"description": "Firefox extension for FabledCurator",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
|
|||||||
@@ -132,8 +132,9 @@ async function exportPlatformCookies(key, card) {
|
|||||||
if (r.error) showError(r.error);
|
if (r.error) showError(r.error);
|
||||||
else {
|
else {
|
||||||
const n = r.cookieCount ?? null;
|
const n = r.cookieCount ?? null;
|
||||||
|
const verifiedSuffix = r.verified ? ' (verified ✓)' : '';
|
||||||
const msg = n !== null
|
const msg = n !== null
|
||||||
? `${PLATFORMS[key].name}: ${n} cookies exported`
|
? `${PLATFORMS[key].name}: ${n} cookies exported${verifiedSuffix}`
|
||||||
: `${PLATFORMS[key].name}: token exported`;
|
: `${PLATFORMS[key].name}: token exported`;
|
||||||
showSuccess(msg);
|
showSuccess(msg);
|
||||||
await loadPlatformStatus();
|
await loadPlatformStatus();
|
||||||
|
|||||||
+14
-1
@@ -9,7 +9,9 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, ref, watch } from 'vue'
|
||||||
|
import { useRoute } from 'vue-router'
|
||||||
|
|
||||||
import AppShell from './components/AppShell.vue'
|
import AppShell from './components/AppShell.vue'
|
||||||
import AppSnackbar from './components/AppSnackbar.vue'
|
import AppSnackbar from './components/AppSnackbar.vue'
|
||||||
import ImageViewer from './components/modal/ImageViewer.vue'
|
import ImageViewer from './components/modal/ImageViewer.vue'
|
||||||
@@ -17,9 +19,20 @@ import { useModalStore } from './stores/modal.js'
|
|||||||
|
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const snackbar = ref(null)
|
const snackbar = ref(null)
|
||||||
|
const route = useRoute()
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
// Expose snackbar via a simple global so stores can call it without props.
|
// Expose snackbar via a simple global so stores can call it without props.
|
||||||
window.__fcToast = (opts) => snackbar.value?.open(opts)
|
window.__fcToast = (opts) => snackbar.value?.open(opts)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Audit 2026-06-02: the modal is an overlay, not a page. When the
|
||||||
|
// route changes (RouterLink inside the modal, history back/forward,
|
||||||
|
// programmatic push from any view), close the modal so it doesn't
|
||||||
|
// hover over a different route. Watching route.name (not the path)
|
||||||
|
// keeps within-route nav like /artist/foo → /artist/bar from
|
||||||
|
// dismissing the modal mid-browse.
|
||||||
|
watch(() => route.name, () => {
|
||||||
|
if (modal.isOpen) modal.close()
|
||||||
|
})
|
||||||
</script>
|
</script>
|
||||||
|
|||||||
@@ -11,9 +11,6 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { onMounted, watch } from 'vue'
|
|
||||||
import { useRoute, useRouter } from 'vue-router'
|
|
||||||
|
|
||||||
import { useArtistStore } from '../../stores/artist.js'
|
import { useArtistStore } from '../../stores/artist.js'
|
||||||
import { useModalStore } from '../../stores/modal.js'
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import MasonryGrid from '../discovery/MasonryGrid.vue'
|
import MasonryGrid from '../discovery/MasonryGrid.vue'
|
||||||
@@ -24,22 +21,9 @@ const props = defineProps({
|
|||||||
|
|
||||||
const store = useArtistStore()
|
const store = useArtistStore()
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const route = useRoute()
|
|
||||||
const router = useRouter()
|
|
||||||
|
|
||||||
onMounted(() => {
|
|
||||||
const initial = parseInt(route.query.image, 10)
|
|
||||||
if (!isNaN(initial)) modal.open(initial)
|
|
||||||
})
|
|
||||||
|
|
||||||
watch(() => route.query.image, (q) => {
|
|
||||||
const id = parseInt(q, 10)
|
|
||||||
if (!isNaN(id) && id !== modal.currentImageId) modal.open(id)
|
|
||||||
else if (isNaN(id) && modal.currentImageId !== null) modal.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
function openImage (id) {
|
function openImage (id) {
|
||||||
router.push({ query: { ...route.query, image: id } })
|
modal.open(id)
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -124,10 +124,16 @@ async function onRetry() {
|
|||||||
if (!props.event.source_id) return
|
if (!props.event.source_id) return
|
||||||
retrying.value = true
|
retrying.value = true
|
||||||
try {
|
try {
|
||||||
await sourcesStore.checkNow(props.event.source_id)
|
const body = await sourcesStore.checkNow(props.event.source_id)
|
||||||
toast({
|
// Audit 2026-06-02: the previous handler unconditionally toasted
|
||||||
text: `Source check re-queued`, type: 'success',
|
// "re-queued" even when the platform was in cooldown (202 +
|
||||||
})
|
// status='deferred'). Operator thought work was in flight when
|
||||||
|
// nothing was actually enqueued.
|
||||||
|
if (body?.status === 'deferred') {
|
||||||
|
toast({ text: 'Retry deferred — platform in cooldown', type: 'info' })
|
||||||
|
} else {
|
||||||
|
toast({ text: 'Source check re-queued', type: 'success' })
|
||||||
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
const isInFlight = !!e?.body?.download_event_id
|
const isInFlight = !!e?.body?.download_event_id
|
||||||
toast({
|
toast({
|
||||||
|
|||||||
@@ -48,10 +48,11 @@ function onSearch(q) {
|
|||||||
if (!query) { results.value = []; return }
|
if (!query) { results.value = []; return }
|
||||||
loading.value = true
|
loading.value = true
|
||||||
try {
|
try {
|
||||||
// Scope the autocomplete to the prediction's category where it maps
|
// Scope the autocomplete to the prediction's category where it
|
||||||
// to a tag kind. 'copyright' has no tag kind; search unscoped there.
|
// maps to a tag kind. Only 'character' surfaces as both a
|
||||||
const kind = ['artist', 'character'].includes(props.category)
|
// suggestion category and a tag kind now ('artist' + 'copyright'
|
||||||
? props.category : null
|
// retired); other categories search unscoped.
|
||||||
|
const kind = props.category === 'character' ? 'character' : null
|
||||||
const params = { q: query, limit: 20 }
|
const params = { q: query, limit: 20 }
|
||||||
if (kind) params.kind = kind
|
if (kind) params.kind = kind
|
||||||
results.value = await api.get('/api/tags/autocomplete', { params })
|
results.value = await api.get('/api/tags/autocomplete', { params })
|
||||||
|
|||||||
@@ -92,7 +92,16 @@ let prevBodyOverflow = null
|
|||||||
// own keystrokes.
|
// own keystrokes.
|
||||||
function onKeyDown(ev) {
|
function onKeyDown(ev) {
|
||||||
if (ev.key === 'Escape') {
|
if (ev.key === 'Escape') {
|
||||||
if (isTextEntry(ev.target)) return
|
// Escape closes the modal even from inside a text input — that's
|
||||||
|
// the universal "get me out of here" expectation, and the
|
||||||
|
// autofocused tag-entry field would otherwise trap focus with no
|
||||||
|
// visible escape (operator-flagged 2026-06-01). EXCEPTION: when a
|
||||||
|
// nested Vuetify overlay is open (v-menu autocomplete dropdown,
|
||||||
|
// FandomPicker v-dialog, per-suggestion 3-dot menu), let that
|
||||||
|
// overlay's own Esc handling fire instead of closing the whole
|
||||||
|
// modal mid-interaction. Vuetify marks open overlays with
|
||||||
|
// `.v-overlay--active`.
|
||||||
|
if (document.querySelector('.v-overlay--active')) return
|
||||||
ev.preventDefault()
|
ev.preventDefault()
|
||||||
emit('close')
|
emit('close')
|
||||||
} else if (ev.key === 'ArrowLeft') {
|
} else if (ev.key === 'ArrowLeft') {
|
||||||
|
|||||||
@@ -10,17 +10,30 @@
|
|||||||
density="compact"
|
density="compact"
|
||||||
>{{ state.error }}</v-alert>
|
>{{ state.error }}</v-alert>
|
||||||
|
|
||||||
<template v-else>
|
<!-- Cards scroll independently of the section title + attachments
|
||||||
|
below them. Cap at ~2.5 cards visible (operator-asked 2026-06-01:
|
||||||
|
keeps the Tags section anchored below at a consistent position;
|
||||||
|
the half-visible third card hints there's more). -->
|
||||||
|
<div v-else class="fc-prov__cards">
|
||||||
<article
|
<article
|
||||||
v-for="e in state.entries" :key="e.provenance_id" class="fc-prov__card"
|
v-for="e in state.entries" :key="e.provenance_id" class="fc-prov__card"
|
||||||
>
|
>
|
||||||
<div class="fc-prov__head">
|
<div class="fc-prov__head">
|
||||||
<span class="fc-prov__platform">{{ e.source.platform }}</span>
|
<!-- Posts with no live subscription have source=null (alembic
|
||||||
|
0030); render an explicit "filesystem import" affordance
|
||||||
|
instead of a platform chip. -->
|
||||||
|
<span class="fc-prov__platform">
|
||||||
|
{{ e.source?.platform ?? 'filesystem import' }}
|
||||||
|
</span>
|
||||||
<span v-if="postDate(e)" class="fc-prov__date">{{ postDate(e) }}</span>
|
<span v-if="postDate(e)" class="fc-prov__date">{{ postDate(e) }}</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="fc-prov__post">
|
<button
|
||||||
|
type="button" class="fc-prov__post"
|
||||||
|
:title="`Open ${postTitle(e)} in the posts feed for ${e.artist.name}`"
|
||||||
|
@click="openPost(e.post.id, e.artist.id)"
|
||||||
|
>
|
||||||
{{ postTitle(e) }}
|
{{ postTitle(e) }}
|
||||||
</div>
|
</button>
|
||||||
<div class="fc-prov__meta">
|
<div class="fc-prov__meta">
|
||||||
<RouterLink :to="`/artist/${e.artist.slug}`">
|
<RouterLink :to="`/artist/${e.artist.slug}`">
|
||||||
by {{ e.artist.name }}
|
by {{ e.artist.name }}
|
||||||
@@ -29,12 +42,8 @@
|
|||||||
· {{ e.post.attachment_count }} files
|
· {{ e.post.attachment_count }} files
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="fc-prov__actions">
|
<div v-if="e.post.description_html" class="fc-prov__actions">
|
||||||
<a href="#" @click.prevent="openPost(e.post.id)">
|
|
||||||
View post
|
|
||||||
</a>
|
|
||||||
<a
|
<a
|
||||||
v-if="e.post.description_html"
|
|
||||||
href="#" @click.prevent="toggleDesc(e.provenance_id)"
|
href="#" @click.prevent="toggleDesc(e.provenance_id)"
|
||||||
>{{ expanded[e.provenance_id] ? 'Hide description ▴' : 'Show description ▾' }}</a>
|
>{{ expanded[e.provenance_id] ? 'Hide description ▴' : 'Show description ▾' }}</a>
|
||||||
</div>
|
</div>
|
||||||
@@ -58,7 +67,7 @@
|
|||||||
</RouterLink>
|
</RouterLink>
|
||||||
</div>
|
</div>
|
||||||
</article>
|
</article>
|
||||||
</template>
|
</div>
|
||||||
|
|
||||||
<div v-if="attachments.length" class="fc-prov__attach">
|
<div v-if="attachments.length" class="fc-prov__attach">
|
||||||
<h4 class="fc-prov__attach-title">Attachments</h4>
|
<h4 class="fc-prov__attach-title">Attachments</h4>
|
||||||
@@ -137,10 +146,16 @@ function postTitle(e) {
|
|||||||
return toPlainText(e.post.title) || `Post ${e.post.external_post_id}`
|
return toPlainText(e.post.title) || `Post ${e.post.external_post_id}`
|
||||||
}
|
}
|
||||||
|
|
||||||
function openPost(postId) {
|
function openPost(postId, artistId) {
|
||||||
// Land on the post in the posts feed (in context), not the gallery
|
// Land on the post in the posts feed (in context), not the gallery
|
||||||
// image grid. Operator-flagged 2026-05-28.
|
// image grid. Scope the feed to this artist so the user lands in
|
||||||
router.push({ path: '/posts', query: { post_id: postId } })
|
// that creator's stream, not the global one — operator-flagged
|
||||||
|
// 2026-06-01. PostsView reads `artist_id` from the query string
|
||||||
|
// (PostsView.vue line ~92) and filters via post_feed_service.
|
||||||
|
router.push({
|
||||||
|
path: '/posts',
|
||||||
|
query: { post_id: postId, artist_id: artistId },
|
||||||
|
})
|
||||||
modal.close()
|
modal.close()
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
@@ -153,6 +168,18 @@ function openPost(postId) {
|
|||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
margin-bottom: 12px;
|
margin-bottom: 12px;
|
||||||
}
|
}
|
||||||
|
.fc-prov__cards {
|
||||||
|
/* 2.5 cards-worth at the typical collapsed card height (~108px each
|
||||||
|
incl. 10px gap). Slightly under to ensure the third card's bottom
|
||||||
|
edge is clipped — the visual cue that there's more below. */
|
||||||
|
max-height: 270px;
|
||||||
|
overflow-y: auto;
|
||||||
|
/* Hairline scrollbar that doesn't compete with content. */
|
||||||
|
scrollbar-width: thin;
|
||||||
|
scrollbar-color: rgb(var(--v-theme-surface-light)) transparent;
|
||||||
|
/* Pad-right so the scrollbar gutter doesn't squeeze card borders. */
|
||||||
|
padding-right: 4px;
|
||||||
|
}
|
||||||
.fc-prov__card {
|
.fc-prov__card {
|
||||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||||
border-radius: 6px; padding: 10px 12px; margin-bottom: 10px;
|
border-radius: 6px; padding: 10px 12px; margin-bottom: 10px;
|
||||||
@@ -163,8 +190,21 @@ function openPost(postId) {
|
|||||||
text-transform: lowercase;
|
text-transform: lowercase;
|
||||||
}
|
}
|
||||||
.fc-prov__post {
|
.fc-prov__post {
|
||||||
font-weight: 700; margin: 4px 0;
|
/* Clickable title — opens the post in the artist-scoped feed
|
||||||
color: rgb(var(--v-theme-on-surface));
|
(operator-flagged 2026-06-01: title IS the primary action, the
|
||||||
|
prior "View post" link was redundant). Styled as a button-link:
|
||||||
|
accent color, underline on hover, focus ring for keyboard nav. */
|
||||||
|
display: block; width: 100%; text-align: left;
|
||||||
|
background: none; border: none; padding: 0;
|
||||||
|
font: inherit; font-weight: 700;
|
||||||
|
margin: 4px 0;
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.fc-prov__post:hover { text-decoration: underline; }
|
||||||
|
.fc-prov__post:focus-visible {
|
||||||
|
outline: 2px solid rgb(var(--v-theme-accent));
|
||||||
|
outline-offset: 2px; border-radius: 3px;
|
||||||
}
|
}
|
||||||
.fc-prov__meta {
|
.fc-prov__meta {
|
||||||
font-size: 13px; color: rgb(var(--v-theme-on-surface-variant));
|
font-size: 13px; color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
|||||||
@@ -1,29 +1,52 @@
|
|||||||
<template>
|
<template>
|
||||||
|
<!-- Chip-card row: visible border + hover/focus state unifies the
|
||||||
|
name, score, and action buttons as one "object" (operator-asked
|
||||||
|
2026-06-01). The row itself is informational; the explicit
|
||||||
|
Accept button + 3-dot menu are the action affordances. -->
|
||||||
<div class="fc-suggestion">
|
<div class="fc-suggestion">
|
||||||
<span class="fc-suggestion__name">
|
<span class="fc-suggestion__name">
|
||||||
{{ suggestion.display_name }}
|
{{ suggestion.display_name }}
|
||||||
<span v-if="suggestion.creates_new_tag" class="fc-suggestion__new"
|
<span v-if="suggestion.creates_new_tag" class="fc-suggestion__new"
|
||||||
title="No matching tag yet — accepting creates it">+new</span>
|
title="No matching tag yet — accepting creates it">+ new</span>
|
||||||
</span>
|
</span>
|
||||||
<span class="fc-suggestion__score">{{ scorePct }}</span>
|
<span class="fc-suggestion__score">{{ scorePct }}</span>
|
||||||
<v-btn
|
<v-btn
|
||||||
icon="mdi-plus" size="x-small" variant="text" color="accent"
|
class="fc-suggestion__accept"
|
||||||
|
size="small" variant="tonal" color="accent"
|
||||||
|
density="compact" rounded="pill"
|
||||||
:aria-label="`Accept ${suggestion.display_name}`"
|
:aria-label="`Accept ${suggestion.display_name}`"
|
||||||
@click="$emit('accept', suggestion)"
|
@click="$emit('accept', suggestion)"
|
||||||
/>
|
>
|
||||||
<v-menu>
|
Accept
|
||||||
<template #activator="{ props }">
|
</v-btn>
|
||||||
<v-btn icon="mdi-dots-vertical" size="x-small" variant="text" v-bind="props" />
|
<!-- Operator-flagged 2026-06-02: the kebab menu wasn't opening.
|
||||||
</template>
|
Wrapping in a <span @click.stop> matches the TagPanel chip
|
||||||
<v-list density="compact">
|
fix — even though there's no parent click capture here today,
|
||||||
<v-list-item @click="$emit('alias', suggestion)">
|
the wrap is harmless and keeps both kebabs on the same
|
||||||
<v-list-item-title>Treat as alias for…</v-list-item-title>
|
pattern. Click bubbles from the v-btn → opens menu via
|
||||||
</v-list-item>
|
activator props → bubble continues to span → stopPropagation
|
||||||
<v-list-item @click="$emit('dismiss', suggestion)">
|
halts it. -->
|
||||||
<v-list-item-title>Dismiss for this image</v-list-item-title>
|
<span class="fc-suggestion__menu-wrap" @click.stop>
|
||||||
</v-list-item>
|
<v-menu>
|
||||||
</v-list>
|
<template #activator="{ props }">
|
||||||
</v-menu>
|
<v-btn
|
||||||
|
class="fc-suggestion__menu"
|
||||||
|
icon="mdi-dots-vertical" size="small"
|
||||||
|
variant="outlined" density="compact"
|
||||||
|
:aria-label="`More actions for ${suggestion.display_name}`"
|
||||||
|
v-bind="props"
|
||||||
|
/>
|
||||||
|
</template>
|
||||||
|
<v-list density="compact">
|
||||||
|
<v-list-item @click="$emit('alias', suggestion)">
|
||||||
|
<v-list-item-title>Treat as alias for…</v-list-item-title>
|
||||||
|
</v-list-item>
|
||||||
|
<v-list-item @click="$emit('dismiss', suggestion)">
|
||||||
|
<v-list-item-title>Dismiss for this image</v-list-item-title>
|
||||||
|
</v-list-item>
|
||||||
|
</v-list>
|
||||||
|
</v-menu>
|
||||||
|
</span>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
@@ -38,17 +61,50 @@ const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
|||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
.fc-suggestion {
|
.fc-suggestion {
|
||||||
display: flex; align-items: center; gap: 6px;
|
display: flex; align-items: center; gap: 8px;
|
||||||
padding: 2px 0;
|
padding: 6px 10px; margin-bottom: 4px;
|
||||||
|
background: rgb(var(--v-theme-surface));
|
||||||
|
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||||
|
border-radius: 6px;
|
||||||
|
transition: background 120ms ease, border-color 120ms ease;
|
||||||
|
}
|
||||||
|
.fc-suggestion:hover {
|
||||||
|
background: rgb(var(--v-theme-surface-light));
|
||||||
|
border-color: rgb(var(--v-theme-accent), 0.4);
|
||||||
|
}
|
||||||
|
.fc-suggestion__name {
|
||||||
|
flex: 1; min-width: 0;
|
||||||
|
font-size: 14px;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
}
|
}
|
||||||
.fc-suggestion__name { flex: 1; min-width: 0; }
|
|
||||||
.fc-suggestion__new {
|
.fc-suggestion__new {
|
||||||
font-size: 10px; color: rgb(var(--v-theme-accent));
|
display: inline-block;
|
||||||
margin-left: 4px;
|
font-size: 10px; font-weight: 600;
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
background: rgba(var(--v-theme-accent), 0.12);
|
||||||
|
border: 1px solid rgb(var(--v-theme-accent), 0.4);
|
||||||
|
padding: 1px 6px; border-radius: 999px;
|
||||||
|
margin-left: 6px;
|
||||||
|
text-transform: uppercase; letter-spacing: 0.04em;
|
||||||
}
|
}
|
||||||
.fc-suggestion__score {
|
.fc-suggestion__score {
|
||||||
|
flex: 0 0 auto; min-width: 38px; text-align: right;
|
||||||
font-size: 11px;
|
font-size: 11px;
|
||||||
color: rgb(var(--v-theme-on-surface-variant, var(--v-theme-on-surface)));
|
color: rgb(var(--v-theme-on-surface-variant, var(--v-theme-on-surface)));
|
||||||
font-family: 'JetBrains Mono', monospace;
|
font-family: 'JetBrains Mono', monospace;
|
||||||
}
|
}
|
||||||
|
/* Vuetify's compact density doesn't shrink the tonal button enough
|
||||||
|
for a tight row; clamp the min-width so Accept stays compact. */
|
||||||
|
.fc-suggestion__accept :deep(.v-btn__content) {
|
||||||
|
font-size: 12px; letter-spacing: 0.02em;
|
||||||
|
}
|
||||||
|
.fc-suggestion__menu-wrap {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
.fc-suggestion__menu {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -22,7 +22,7 @@
|
|||||||
<SuggestionsCategoryGroup
|
<SuggestionsCategoryGroup
|
||||||
v-if="store.byCategory.general && store.byCategory.general.length"
|
v-if="store.byCategory.general && store.byCategory.general.length"
|
||||||
label="General" :items="store.byCategory.general"
|
label="General" :items="store.byCategory.general"
|
||||||
collapsible :default-open="false"
|
collapsible :default-open="true"
|
||||||
@accept="onAccept" @alias="onAlias" @dismiss="store.dismiss"
|
@accept="onAccept" @alias="onAlias" @dismiss="store.dismiss"
|
||||||
/>
|
/>
|
||||||
</template>
|
</template>
|
||||||
@@ -41,13 +41,18 @@
|
|||||||
import { toast } from '../../utils/toast.js'
|
import { toast } from '../../utils/toast.js'
|
||||||
import { computed, ref, watch } from 'vue'
|
import { computed, ref, watch } from 'vue'
|
||||||
import { useSuggestionsStore, CATEGORY_LABELS } from '../../stores/suggestions.js'
|
import { useSuggestionsStore, CATEGORY_LABELS } from '../../stores/suggestions.js'
|
||||||
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import SuggestionsCategoryGroup from './SuggestionsCategoryGroup.vue'
|
import SuggestionsCategoryGroup from './SuggestionsCategoryGroup.vue'
|
||||||
import AliasPickerDialog from './AliasPickerDialog.vue'
|
import AliasPickerDialog from './AliasPickerDialog.vue'
|
||||||
|
|
||||||
const props = defineProps({ imageId: { type: Number, required: true } })
|
const props = defineProps({ imageId: { type: Number, required: true } })
|
||||||
const store = useSuggestionsStore()
|
const store = useSuggestionsStore()
|
||||||
|
const modal = useModalStore()
|
||||||
|
|
||||||
const peopleCats = ['artist', 'character', 'copyright']
|
// 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired as
|
||||||
|
// suggestion categories. Only 'character' remains as a people-style
|
||||||
|
// category alongside the general bucket.
|
||||||
|
const peopleCats = ['character']
|
||||||
function labelFor(c) { return CATEGORY_LABELS[c] || c }
|
function labelFor(c) { return CATEGORY_LABELS[c] || c }
|
||||||
|
|
||||||
const isEmpty = computed(() =>
|
const isEmpty = computed(() =>
|
||||||
@@ -56,9 +61,20 @@ const isEmpty = computed(() =>
|
|||||||
|
|
||||||
watch(() => props.imageId, (id) => { if (id != null) store.load(id) }, { immediate: true })
|
watch(() => props.imageId, (id) => { if (id != null) store.load(id) }, { immediate: true })
|
||||||
|
|
||||||
|
// After a successful accept/alias-accept, refresh the modal's current
|
||||||
|
// tag list so TagPanel's chip rail reflects the newly-attached tag.
|
||||||
|
// Operator-flagged 2026-06-01: the suggestion store dropped the
|
||||||
|
// suggestion (correct) but didn't propagate the new tag back into the
|
||||||
|
// modal store, so the chip rail looked unchanged even though the
|
||||||
|
// backend had recorded the application. Mirrors the addExistingTag /
|
||||||
|
// createAndAdd flows which already call reloadTags() after applying.
|
||||||
async function onAccept(s) {
|
async function onAccept(s) {
|
||||||
try { await store.accept(s) }
|
try {
|
||||||
catch (e) { toast({ text: `Accept failed: ${e.message}`, type: 'error' }) }
|
await store.accept(s)
|
||||||
|
await modal.reloadTags()
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Accept failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const aliasDialog = ref(false)
|
const aliasDialog = ref(false)
|
||||||
@@ -68,6 +84,7 @@ async function onAliasConfirm(canonicalTagId) {
|
|||||||
try {
|
try {
|
||||||
await store.aliasAccept(aliasTarget.value, canonicalTagId)
|
await store.aliasAccept(aliasTarget.value, canonicalTagId)
|
||||||
aliasDialog.value = false
|
aliasDialog.value = false
|
||||||
|
await modal.reloadTags()
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
toast({ text: `Alias failed: ${e.message}`, type: 'error' })
|
toast({ text: `Alias failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-tag-autocomplete">
|
<div class="fc-tag-autocomplete">
|
||||||
<v-text-field
|
<v-text-field
|
||||||
|
ref="inputRef"
|
||||||
v-model="query"
|
v-model="query"
|
||||||
placeholder="Add tag (or kind:name — character/fandom/series)"
|
placeholder="Add tag (or kind:name — character/fandom/series)"
|
||||||
density="compact" hide-details
|
density="compact" hide-details
|
||||||
@@ -52,13 +53,21 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, ref, watch } from 'vue'
|
import { computed, nextTick, onMounted, ref, watch } from 'vue'
|
||||||
import { useTagStore } from '../../stores/tags.js'
|
import { useTagStore } from '../../stores/tags.js'
|
||||||
import FandomPicker from './FandomPicker.vue'
|
import FandomPicker from './FandomPicker.vue'
|
||||||
|
|
||||||
const emit = defineEmits(['pick-existing', 'pick-new', 'cancel'])
|
const emit = defineEmits(['pick-existing', 'pick-new', 'cancel'])
|
||||||
const store = useTagStore()
|
const store = useTagStore()
|
||||||
|
|
||||||
|
// Autofocus on modal open so the operator can type the moment the view
|
||||||
|
// modal renders, no extra click required (operator-asked 2026-06-01).
|
||||||
|
// Vuetify's v-text-field exposes .focus() on the component instance;
|
||||||
|
// nextTick waits for the modal's mount to finish so the inner <input>
|
||||||
|
// element exists.
|
||||||
|
const inputRef = ref(null)
|
||||||
|
onMounted(() => { nextTick(() => inputRef.value?.focus?.()) })
|
||||||
|
|
||||||
// Single text input; no kind dropdown. Client-side mirror of the
|
// Single text input; no kind dropdown. Client-side mirror of the
|
||||||
// backend's parse_kind_prefix lives below — kept in sync with
|
// backend's parse_kind_prefix lives below — kept in sync with
|
||||||
// KNOWN_KINDS in backend/app/utils/tag_prefix.py. The backend is the
|
// KNOWN_KINDS in backend/app/utils/tag_prefix.py. The backend is the
|
||||||
|
|||||||
@@ -10,19 +10,27 @@
|
|||||||
>
|
>
|
||||||
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
||||||
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
{{ tag.name }}<span v-if="tag.fandom_id">→</span>
|
||||||
<v-menu>
|
<!-- Operator-flagged 2026-06-02: the previous activator had
|
||||||
<template #activator="{ props: mp }">
|
`@click.stop` directly on the v-icon, which silently
|
||||||
<v-icon
|
overrode Vuetify's onClick from `v-bind="mp"` — the menu
|
||||||
v-bind="mp" size="x-small" class="ml-1"
|
never opened. Now the v-icon receives the activator
|
||||||
icon="mdi-dots-vertical" @click.stop
|
onClick cleanly, and the wrapping span absorbs the
|
||||||
/>
|
bubbled click so the chip's close button isn't tripped. -->
|
||||||
</template>
|
<span class="kebab-wrap" @click.stop>
|
||||||
<v-list density="compact">
|
<v-menu>
|
||||||
<v-list-item @click="openRename(tag)">
|
<template #activator="{ props: mp }">
|
||||||
<v-list-item-title>Rename…</v-list-item-title>
|
<v-icon
|
||||||
</v-list-item>
|
v-bind="mp" size="x-small" class="ml-1"
|
||||||
</v-list>
|
icon="mdi-dots-vertical"
|
||||||
</v-menu>
|
/>
|
||||||
|
</template>
|
||||||
|
<v-list density="compact">
|
||||||
|
<v-list-item @click="openRename(tag)">
|
||||||
|
<v-list-item-title>Rename…</v-list-item-title>
|
||||||
|
</v-list-item>
|
||||||
|
</v-list>
|
||||||
|
</v-menu>
|
||||||
|
</span>
|
||||||
</v-chip>
|
</v-chip>
|
||||||
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
<span v-if="!modal.current?.tags?.length" class="text-caption">No tags yet.</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -113,4 +121,5 @@ async function onRenamed() {
|
|||||||
margin-bottom: 12px;
|
margin-bottom: 12px;
|
||||||
}
|
}
|
||||||
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
.fc-tag-panel__chips { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||||
|
.kebab-wrap { display: inline-flex; align-items: center; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -7,7 +7,12 @@
|
|||||||
@keydown.enter="onCardClick"
|
@keydown.enter="onCardClick"
|
||||||
>
|
>
|
||||||
<div class="fc-post-card__head">
|
<div class="fc-post-card__head">
|
||||||
<v-chip size="x-small" variant="tonal">{{ post.source.platform }}</v-chip>
|
<!-- Posts with no live subscription have source=null (alembic
|
||||||
|
0030); show a "filesystem import" affordance instead of a
|
||||||
|
platform chip. -->
|
||||||
|
<v-chip size="x-small" variant="tonal">
|
||||||
|
{{ post.source?.platform ?? 'filesystem import' }}
|
||||||
|
</v-chip>
|
||||||
<RouterLink
|
<RouterLink
|
||||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||||
class="fc-post-card__artist"
|
class="fc-post-card__artist"
|
||||||
@@ -25,7 +30,7 @@
|
|||||||
v-if="post.post_url"
|
v-if="post.post_url"
|
||||||
:href="post.post_url" target="_blank" rel="noopener"
|
:href="post.post_url" target="_blank" rel="noopener"
|
||||||
icon="mdi-open-in-new" size="x-small" variant="text"
|
icon="mdi-open-in-new" size="x-small" variant="text"
|
||||||
:aria-label="`open original post on ${post.source.platform}`"
|
:aria-label="`open original post on ${post.source?.platform ?? 'web'}`"
|
||||||
@click.stop
|
@click.stop
|
||||||
/>
|
/>
|
||||||
<v-btn
|
<v-btn
|
||||||
|
|||||||
@@ -22,10 +22,10 @@ import { reactive, watch } from 'vue'
|
|||||||
import { useMLStore } from '../../stores/ml.js'
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
|
|
||||||
const store = useMLStore()
|
const store = useMLStore()
|
||||||
|
// 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired as
|
||||||
|
// suggestion categories; their threshold rows are gone.
|
||||||
const fields = [
|
const fields = [
|
||||||
{ key: 'suggestion_threshold_artist', label: 'Artist' },
|
|
||||||
{ key: 'suggestion_threshold_character', label: 'Character' },
|
{ key: 'suggestion_threshold_character', label: 'Character' },
|
||||||
{ key: 'suggestion_threshold_copyright', label: 'Copyright' },
|
|
||||||
{ key: 'suggestion_threshold_general', label: 'General' },
|
{ key: 'suggestion_threshold_general', label: 'General' },
|
||||||
{ key: 'centroid_similarity_threshold', label: 'Centroid similarity' }
|
{ key: 'centroid_similarity_threshold', label: 'Centroid similarity' }
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ import { computed } from 'vue'
|
|||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
queues: { type: Object, default: null }, // store.queues
|
queues: { type: Object, default: null }, // store.queues
|
||||||
workers: { type: Object, default: null }, // store.workers
|
workers: { type: Object, default: null }, // store.workers
|
||||||
recentMinute: { type: Array, default: () => [] }, // store.recentMinute
|
recentRuns: { type: Array, default: () => [] }, // store.recentRuns
|
||||||
compact: { type: Boolean, default: false },
|
compact: { type: Boolean, default: false },
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -80,7 +80,7 @@ function activeCount(name) {
|
|||||||
|
|
||||||
const recentByQueue = computed(() => {
|
const recentByQueue = computed(() => {
|
||||||
const out = {}
|
const out = {}
|
||||||
for (const r of props.recentMinute) {
|
for (const r of props.recentRuns) {
|
||||||
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
|
if (!out[r.queue]) out[r.queue] = { ok: 0, err: 0 }
|
||||||
if (r.status === 'ok') out[r.queue].ok++
|
if (r.status === 'ok') out[r.queue].ok++
|
||||||
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
|
else if (r.status === 'error' || r.status === 'timeout') out[r.queue].err++
|
||||||
|
|||||||
@@ -24,7 +24,7 @@
|
|||||||
<QueuesTable
|
<QueuesTable
|
||||||
:queues="store.queues"
|
:queues="store.queues"
|
||||||
:workers="store.workers"
|
:workers="store.workers"
|
||||||
:recent-minute="store.recentMinute"
|
:recent-runs="store.recentRuns"
|
||||||
compact
|
compact
|
||||||
/>
|
/>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
@@ -47,7 +47,7 @@ function pollOnce() {
|
|||||||
if (document.hidden) return
|
if (document.hidden) return
|
||||||
store.loadQueues()
|
store.loadQueues()
|
||||||
store.loadWorkers()
|
store.loadWorkers()
|
||||||
store.loadRecentMinute()
|
store.loadRecentRuns()
|
||||||
}
|
}
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
|
|||||||
@@ -14,7 +14,7 @@
|
|||||||
<QueuesTable
|
<QueuesTable
|
||||||
:queues="store.queues"
|
:queues="store.queues"
|
||||||
:workers="store.workers"
|
:workers="store.workers"
|
||||||
:recent-minute="store.recentMinute"
|
:recent-runs="store.recentRuns"
|
||||||
/>
|
/>
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
@@ -202,7 +202,7 @@ function pollQueues() {
|
|||||||
if (document.hidden) return
|
if (document.hidden) return
|
||||||
store.loadQueues()
|
store.loadQueues()
|
||||||
store.loadWorkers()
|
store.loadWorkers()
|
||||||
store.loadRecentMinute()
|
store.loadRecentRuns()
|
||||||
}
|
}
|
||||||
function pollFailures() {
|
function pollFailures() {
|
||||||
if (document.hidden) return
|
if (document.hidden) return
|
||||||
|
|||||||
@@ -10,7 +10,14 @@
|
|||||||
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
<v-btn color="primary" rounded="pill" :loading="busy" @click="run">
|
||||||
<v-icon start>mdi-image-refresh</v-icon> Run backfill now
|
<v-icon start>mdi-image-refresh</v-icon> Run backfill now
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<span v-if="done" class="ml-3 text-caption">Enqueued.</span>
|
<span v-if="result" class="ml-3 text-caption">
|
||||||
|
Scanned <strong>{{ result.scanned }}</strong> · enqueued
|
||||||
|
<strong>{{ result.enqueued }}</strong>
|
||||||
|
<span v-if="result.regenerated > 0">
|
||||||
|
({{ result.regenerated }} regenerated)
|
||||||
|
</span>
|
||||||
|
· {{ result.ok }} ok
|
||||||
|
</span>
|
||||||
<QueueStatusBar queue="thumbnail" queue-label="Thumbnail" />
|
<QueueStatusBar queue="thumbnail" queue-label="Thumbnail" />
|
||||||
</v-card-text>
|
</v-card-text>
|
||||||
</v-card>
|
</v-card>
|
||||||
@@ -23,10 +30,11 @@ import { useThumbnailsStore } from '../../stores/thumbnails.js'
|
|||||||
import QueueStatusBar from './QueueStatusBar.vue'
|
import QueueStatusBar from './QueueStatusBar.vue'
|
||||||
const store = useThumbnailsStore()
|
const store = useThumbnailsStore()
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
const done = ref(false)
|
const result = ref(null)
|
||||||
async function run () {
|
async function run () {
|
||||||
busy.value = true
|
busy.value = true
|
||||||
try { await store.triggerBackfill(); done.value = true }
|
result.value = null
|
||||||
|
try { result.value = await store.triggerBackfill() }
|
||||||
catch (e) { toast({ text: e.message, type: 'error' }) }
|
catch (e) { toast({ text: e.message, type: 'error' }) }
|
||||||
finally { busy.value = false }
|
finally { busy.value = false }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,6 +58,7 @@
|
|||||||
:retrying-all="retryingAll"
|
:retrying-all="retryingAll"
|
||||||
@retry="onRetrySource"
|
@retry="onRetrySource"
|
||||||
@retry-all="onRetryAll"
|
@retry-all="onRetryAll"
|
||||||
|
@view-logs="onViewFailingLogs"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<div v-if="store.loading && store.events.length === 0" class="fc-dl__loading">
|
<div v-if="store.loading && store.events.length === 0" class="fc-dl__loading">
|
||||||
@@ -364,6 +365,23 @@ watch(filterModel, async (m) => {
|
|||||||
async function openDetail(id) {
|
async function openDetail(id) {
|
||||||
await store.loadOne(id)
|
await store.loadOne(id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function onViewFailingLogs(source) {
|
||||||
|
// Find and open the most recent DownloadEvent for this source.
|
||||||
|
// Reuses the existing DownloadDetailModal — same stdout/stderr/error
|
||||||
|
// surface the row-click in the events feed shows.
|
||||||
|
try {
|
||||||
|
const ev = await store.loadLastForSource(source.id)
|
||||||
|
if (!ev) {
|
||||||
|
toast({
|
||||||
|
text: `No download events recorded for ${source.artist_name || source.platform} yet.`,
|
||||||
|
type: 'warning',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Failed to load logs: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -25,10 +25,27 @@
|
|||||||
<v-chip size="x-small" color="error" variant="flat" label class="fc-fail__count">
|
<v-chip size="x-small" color="error" variant="flat" label class="fc-fail__count">
|
||||||
{{ s.consecutive_failures }}× failed
|
{{ s.consecutive_failures }}× failed
|
||||||
</v-chip>
|
</v-chip>
|
||||||
|
<v-chip
|
||||||
|
v-if="s.error_type"
|
||||||
|
size="x-small" variant="outlined" label
|
||||||
|
:color="errorTypeColor(s.error_type)"
|
||||||
|
class="fc-fail__class"
|
||||||
|
:title="errorTypeHint(s.error_type)"
|
||||||
|
>
|
||||||
|
{{ s.error_type }}
|
||||||
|
</v-chip>
|
||||||
<span class="fc-fail__err" :title="s.last_error || ''">
|
<span class="fc-fail__err" :title="s.last_error || ''">
|
||||||
{{ s.last_error || 'no error message recorded' }}
|
{{ s.last_error || 'no error message recorded' }}
|
||||||
</span>
|
</span>
|
||||||
<v-spacer />
|
<v-spacer />
|
||||||
|
<v-btn
|
||||||
|
size="x-small" variant="text" prepend-icon="mdi-text-box-search-outline"
|
||||||
|
:loading="logLoadingIds.has(s.id)"
|
||||||
|
@click="onViewLogs(s)"
|
||||||
|
title="Show the most recent download event's stdout/stderr/error"
|
||||||
|
>
|
||||||
|
Logs
|
||||||
|
</v-btn>
|
||||||
<v-btn
|
<v-btn
|
||||||
size="x-small" variant="text" prepend-icon="mdi-refresh"
|
size="x-small" variant="text" prepend-icon="mdi-refresh"
|
||||||
:loading="retryingIds.has(s.id)"
|
:loading="retryingIds.has(s.id)"
|
||||||
@@ -52,9 +69,59 @@ defineProps({
|
|||||||
retryingIds: { type: Set, default: () => new Set() },
|
retryingIds: { type: Set, default: () => new Set() },
|
||||||
retryingAll: { type: Boolean, default: false },
|
retryingAll: { type: Boolean, default: false },
|
||||||
})
|
})
|
||||||
defineEmits(['retry', 'retry-all'])
|
const emit = defineEmits(['retry', 'retry-all', 'view-logs'])
|
||||||
|
|
||||||
const open = ref(true)
|
const open = ref(true)
|
||||||
|
// Per-row loading flag so the spinner lives on the row whose Logs
|
||||||
|
// button was clicked, not on every row.
|
||||||
|
const logLoadingIds = ref(new Set())
|
||||||
|
|
||||||
|
// Audit 2026-06-02: surface the ErrorType taxonomy as a colored chip
|
||||||
|
// next to the consecutive-failures count so operators can bulk-triage
|
||||||
|
// by error class. Color reflects "what to do next":
|
||||||
|
// warning (yellow) — auth/cookie issue: operator should rotate
|
||||||
|
// info (blue) — backend-paced (cooldown / rate limit / timeout)
|
||||||
|
// error (red) — likely terminal without operator intervention
|
||||||
|
const ERROR_TYPE_COLOR = {
|
||||||
|
auth_error: 'warning',
|
||||||
|
rate_limited: 'info',
|
||||||
|
timeout: 'info',
|
||||||
|
network_error: 'info',
|
||||||
|
not_found: 'error',
|
||||||
|
access_denied: 'error',
|
||||||
|
validation_failed: 'error',
|
||||||
|
unsupported_url: 'error',
|
||||||
|
http_error: 'error',
|
||||||
|
unknown_error: 'error',
|
||||||
|
partial: 'info',
|
||||||
|
tier_limited: 'info',
|
||||||
|
no_new_content: 'info',
|
||||||
|
}
|
||||||
|
const ERROR_TYPE_HINT = {
|
||||||
|
auth_error: 'Cookies likely expired — re-upload in Credentials.',
|
||||||
|
rate_limited: 'Platform-wide cooldown active. Will retry after it expires.',
|
||||||
|
timeout: 'Subprocess exceeded its time budget. Often retries cleanly.',
|
||||||
|
network_error: 'Transient network issue. Will retry on next tick.',
|
||||||
|
not_found: 'URL 404 — creator may have renamed or deleted.',
|
||||||
|
access_denied: 'Subscription tier may not grant this content.',
|
||||||
|
validation_failed: 'Downloaded files were quarantined by the validator.',
|
||||||
|
http_error: 'Generic HTTP error — see Logs.',
|
||||||
|
unsupported_url: 'gallery-dl does not support this URL pattern.',
|
||||||
|
unknown_error: 'Could not classify — see Logs.',
|
||||||
|
}
|
||||||
|
function errorTypeColor(t) { return ERROR_TYPE_COLOR[t] || 'error' }
|
||||||
|
function errorTypeHint(t) { return ERROR_TYPE_HINT[t] || '' }
|
||||||
|
async function onViewLogs(s) {
|
||||||
|
if (logLoadingIds.value.has(s.id)) return
|
||||||
|
logLoadingIds.value = new Set(logLoadingIds.value).add(s.id)
|
||||||
|
try {
|
||||||
|
await emit('view-logs', s)
|
||||||
|
} finally {
|
||||||
|
const next = new Set(logLoadingIds.value)
|
||||||
|
next.delete(s.id)
|
||||||
|
logLoadingIds.value = next
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
@@ -68,16 +135,32 @@ const open = ref(true)
|
|||||||
.fc-fail__title { font-weight: 600; }
|
.fc-fail__title { font-weight: 600; }
|
||||||
.fc-fail__body {
|
.fc-fail__body {
|
||||||
padding: 0 14px 10px;
|
padding: 0 14px 10px;
|
||||||
display: flex; flex-direction: column; gap: 2px;
|
display: flex; flex-direction: column;
|
||||||
|
/* Borders, not gap, so the row separators are visible inside the
|
||||||
|
* tonal error card (where surface tints get washed out and gap is
|
||||||
|
* just empty space). */
|
||||||
}
|
}
|
||||||
.fc-fail__row {
|
.fc-fail__row {
|
||||||
display: flex; align-items: center; gap: 10px;
|
display: flex; align-items: center; gap: 10px;
|
||||||
padding: 6px 8px;
|
padding: 8px 10px;
|
||||||
border-radius: 4px;
|
border-radius: 4px;
|
||||||
background: rgb(var(--v-theme-surface) / 0.4);
|
/* Hover-darken gives the eye a horizontal track from the artist
|
||||||
|
* name on the left to the Logs/Retry buttons on the right.
|
||||||
|
* Bottom border replaces the previous-too-subtle zebra striping —
|
||||||
|
* inside the tonal error card, surface-tint contrast was negligible.
|
||||||
|
* Operator-flagged 2026-06-01 (twice). */
|
||||||
|
border-bottom: 1px solid rgb(255 255 255 / 0.08);
|
||||||
|
transition: background 80ms ease;
|
||||||
|
}
|
||||||
|
.fc-fail__row:last-child {
|
||||||
|
border-bottom: none;
|
||||||
|
}
|
||||||
|
.fc-fail__row:hover {
|
||||||
|
background: rgb(0 0 0 / 0.25);
|
||||||
}
|
}
|
||||||
.fc-fail__artist { font-weight: 600; white-space: nowrap; }
|
.fc-fail__artist { font-weight: 600; white-space: nowrap; }
|
||||||
.fc-fail__count { flex: 0 0 auto; }
|
.fc-fail__count { flex: 0 0 auto; }
|
||||||
|
.fc-fail__class { flex: 0 0 auto; }
|
||||||
.fc-fail__err {
|
.fc-fail__err {
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
font-size: 0.8rem;
|
font-size: 0.8rem;
|
||||||
|
|||||||
@@ -31,6 +31,12 @@
|
|||||||
v-if="(source.consecutive_failures || 0) > 0"
|
v-if="(source.consecutive_failures || 0) > 0"
|
||||||
size="x-small" color="error" variant="tonal" label
|
size="x-small" color="error" variant="tonal" label
|
||||||
>{{ source.consecutive_failures }}</v-chip>
|
>{{ source.consecutive_failures }}</v-chip>
|
||||||
|
<v-chip
|
||||||
|
v-else-if="(source.backfill_runs_remaining || 0) > 0"
|
||||||
|
size="x-small" color="info" variant="tonal" label
|
||||||
|
>
|
||||||
|
backfill ({{ source.backfill_runs_remaining }}×)
|
||||||
|
</v-chip>
|
||||||
<span v-else class="fc-source-row__zero">0</span>
|
<span v-else class="fc-source-row__zero">0</span>
|
||||||
</td>
|
</td>
|
||||||
<td class="fc-source-row__actions">
|
<td class="fc-source-row__actions">
|
||||||
@@ -42,6 +48,16 @@
|
|||||||
<v-icon>mdi-play</v-icon>
|
<v-icon>mdi-play</v-icon>
|
||||||
<v-tooltip activator="parent" location="top">Check now</v-tooltip>
|
<v-tooltip activator="parent" location="top">Check now</v-tooltip>
|
||||||
</v-btn>
|
</v-btn>
|
||||||
|
<v-btn
|
||||||
|
icon="mdi-magnify-scan" size="x-small" variant="text"
|
||||||
|
:disabled="(source.backfill_runs_remaining || 0) > 0"
|
||||||
|
@click.stop="$emit('backfill', source)"
|
||||||
|
>
|
||||||
|
<v-icon>mdi-magnify-scan</v-icon>
|
||||||
|
<v-tooltip activator="parent" location="top">
|
||||||
|
Deep scan — walk full history for next few runs
|
||||||
|
</v-tooltip>
|
||||||
|
</v-btn>
|
||||||
<v-btn
|
<v-btn
|
||||||
icon="mdi-pencil" size="x-small" variant="text"
|
icon="mdi-pencil" size="x-small" variant="text"
|
||||||
@click.stop="$emit('edit', source)"
|
@click.stop="$emit('edit', source)"
|
||||||
@@ -69,7 +85,7 @@ const props = defineProps({
|
|||||||
checking: { type: Boolean, default: false },
|
checking: { type: Boolean, default: false },
|
||||||
warningThreshold: { type: Number, default: 5 },
|
warningThreshold: { type: Number, default: 5 },
|
||||||
})
|
})
|
||||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check'])
|
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill'])
|
||||||
|
|
||||||
function onToggleEnabled(value) {
|
function onToggleEnabled(value) {
|
||||||
emit('toggle', { source: props.source, enabled: value })
|
emit('toggle', { source: props.source, enabled: value })
|
||||||
|
|||||||
@@ -174,6 +174,7 @@
|
|||||||
@remove="removeSource"
|
@remove="removeSource"
|
||||||
@toggle="toggleSourceEnabled"
|
@toggle="toggleSourceEnabled"
|
||||||
@check="onCheck"
|
@check="onCheck"
|
||||||
|
@backfill="onBackfill"
|
||||||
/>
|
/>
|
||||||
<tr v-if="item.sources.length === 0">
|
<tr v-if="item.sources.length === 0">
|
||||||
<td colspan="8" class="fc-subs__sources-empty">
|
<td colspan="8" class="fc-subs__sources-empty">
|
||||||
@@ -411,6 +412,17 @@ function onArtistCreated(artist) {
|
|||||||
async function onCheck(source) {
|
async function onCheck(source) {
|
||||||
try {
|
try {
|
||||||
const body = await store.checkNow(source.id)
|
const body = await store.checkNow(source.id)
|
||||||
|
// Audit 2026-06-02: /api/sources/<id>/check returns 202 with
|
||||||
|
// `{status:'deferred', cooldown_until}` when the platform is in
|
||||||
|
// cooldown — the previous handler treated this as success and
|
||||||
|
// toasted "event #undefined", masking that nothing was enqueued.
|
||||||
|
if (body?.status === 'deferred') {
|
||||||
|
toast({
|
||||||
|
text: 'Check deferred — platform in cooldown',
|
||||||
|
type: 'info',
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
toast({
|
toast({
|
||||||
text: `Check enqueued (event #${body.download_event_id})`,
|
text: `Check enqueued (event #${body.download_event_id})`,
|
||||||
type: 'success',
|
type: 'success',
|
||||||
@@ -431,20 +443,56 @@ async function onCheck(source) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Plan #544: arm a source for backfill mode (gallery-dl walks the full
|
||||||
|
// post history) for the next N download runs. Default 3 — enough budget
|
||||||
|
// to finish a deep creator without re-prompting the operator across
|
||||||
|
// timeout boundaries. The chip on the row reflects the remaining count.
|
||||||
|
async function onBackfill(source) {
|
||||||
|
const raw = globalThis.window?.prompt(
|
||||||
|
`Deep scan "${source.artist_name} (${source.platform})" — walk full history for the next how many download runs? (1–10, default 3)`,
|
||||||
|
'3',
|
||||||
|
)
|
||||||
|
if (raw == null) return
|
||||||
|
const runs = parseInt(raw, 10)
|
||||||
|
if (!Number.isFinite(runs) || runs < 1 || runs > 10) {
|
||||||
|
toast({ text: 'Deep scan: runs must be 1–10', type: 'error' })
|
||||||
|
return
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await store.setBackfill(source.id, runs, source.artist_id)
|
||||||
|
toast({
|
||||||
|
text: `Deep scan armed for ${runs} run${runs === 1 ? '' : 's'}`,
|
||||||
|
type: 'success',
|
||||||
|
})
|
||||||
|
await store.loadAll()
|
||||||
|
} catch (e) {
|
||||||
|
toast({
|
||||||
|
text: `Deep scan failed: ${e?.detail || e?.message || e}`,
|
||||||
|
type: 'error',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function checkAll(group) {
|
async function checkAll(group) {
|
||||||
let ok = 0
|
let ok = 0
|
||||||
let conflict = 0
|
let conflict = 0
|
||||||
|
let deferred = 0
|
||||||
for (const s of group.sources) {
|
for (const s of group.sources) {
|
||||||
if (!s.enabled) continue
|
if (!s.enabled) continue
|
||||||
try {
|
try {
|
||||||
await store.checkNow(s.id)
|
const body = await store.checkNow(s.id)
|
||||||
ok += 1
|
// Audit 2026-06-02: deferred (202 + cooldown_until) used to be
|
||||||
|
// counted as queued, inflating the success tally and hiding
|
||||||
|
// that the cooldown actually held the work back.
|
||||||
|
if (body?.status === 'deferred') deferred += 1
|
||||||
|
else ok += 1
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (e?.body?.download_event_id) conflict += 1
|
if (e?.body?.download_event_id) conflict += 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const parts = []
|
const parts = []
|
||||||
if (ok) parts.push(`${ok} queued`)
|
if (ok) parts.push(`${ok} queued`)
|
||||||
|
if (deferred) parts.push(`${deferred} deferred (cooldown)`)
|
||||||
if (conflict) parts.push(`${conflict} already running`)
|
if (conflict) parts.push(`${conflict} already running`)
|
||||||
toast({
|
toast({
|
||||||
text: parts.join(', ') || 'Nothing to check (no enabled sources)',
|
text: parts.join(', ') || 'Nothing to check (no enabled sources)',
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
// Inflight-token guard for stores whose async loads can be re-triggered
|
||||||
|
// by rapid filter/navigation changes. Without this, late responses
|
||||||
|
// from a prior load overwrite the store with stale data
|
||||||
|
// (last-writer-wins, not request-order-wins). gallery.js had a
|
||||||
|
// hand-rolled `inflightId` of the same shape; this composable
|
||||||
|
// extracts it so every store can use the same pattern.
|
||||||
|
//
|
||||||
|
// Audit 2026-06-02 (workflow wf_bbe3fdb1-e62) found this missing in
|
||||||
|
// modal/suggestions/artist/downloads/directory/posts. The two most
|
||||||
|
// operator-impacting consequences: (1) modal tag mutations could
|
||||||
|
// land DELETE/POST on the wrong image when the user navigated mid-
|
||||||
|
// flight; (2) suggestions accept could push a tag to the wrong
|
||||||
|
// image AND add it to the allowlist.
|
||||||
|
//
|
||||||
|
// Usage:
|
||||||
|
// const inflight = useInflightToken()
|
||||||
|
//
|
||||||
|
// async function load() {
|
||||||
|
// const t = inflight.claim()
|
||||||
|
// const body = await api.get(...)
|
||||||
|
// if (!t.isCurrent()) return // stale — abort write
|
||||||
|
// items.value = body.items // safe to commit
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// function setFilter(f) {
|
||||||
|
// inflight.cancel() // any in-flight token is now stale
|
||||||
|
// filter.value = f
|
||||||
|
// load() // claims a fresh token
|
||||||
|
// }
|
||||||
|
//
|
||||||
|
// For multi-await flows (POST then GET, optimistic mutation then
|
||||||
|
// reconcile), check isCurrent() after EACH await — any intervening
|
||||||
|
// claim() or cancel() invalidates the prior token.
|
||||||
|
export function useInflightToken() {
|
||||||
|
let _seq = 0
|
||||||
|
let _current = 0
|
||||||
|
|
||||||
|
function claim() {
|
||||||
|
_current = ++_seq
|
||||||
|
const id = _current
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
isCurrent: () => _current === id,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function cancel() {
|
||||||
|
_current = ++_seq
|
||||||
|
}
|
||||||
|
|
||||||
|
return { claim, cancel }
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import { defineStore } from 'pinia'
|
import { defineStore } from 'pinia'
|
||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
import { usePostsStore } from './posts.js'
|
import { usePostsStore } from './posts.js'
|
||||||
|
|
||||||
const PAGE = 60
|
const PAGE = 60
|
||||||
@@ -15,12 +16,17 @@ export const useArtistStore = defineStore('artist', () => {
|
|||||||
const error = ref(null)
|
const error = ref(null)
|
||||||
const notFound = ref(false)
|
const notFound = ref(false)
|
||||||
let started = false
|
let started = false
|
||||||
|
// Rapid artist-to-artist navigation used to render the previous
|
||||||
|
// artist's overview/images briefly when the second load resolved
|
||||||
|
// after the third. Audit 2026-06-02.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
async function load (slug) {
|
async function load (slug) {
|
||||||
// Cross-artist reset: clear this store AND the posts store so the new
|
// Cross-artist reset: clear this store AND the posts store so the new
|
||||||
// artist doesn't briefly render with the previous artist's content
|
// artist doesn't briefly render with the previous artist's content
|
||||||
// when the user is on the Posts tab. (Gallery tab uses this artist
|
// when the user is on the Posts tab. (Gallery tab uses this artist
|
||||||
// store's own images list — cleared above.)
|
// store's own images list — cleared above.)
|
||||||
|
inflight.cancel()
|
||||||
overview.value = null
|
overview.value = null
|
||||||
images.value = []
|
images.value = []
|
||||||
nextCursor.value = null
|
nextCursor.value = null
|
||||||
@@ -29,14 +35,18 @@ export const useArtistStore = defineStore('artist', () => {
|
|||||||
error.value = null
|
error.value = null
|
||||||
loading.value = true
|
loading.value = true
|
||||||
usePostsStore().$reset?.()
|
usePostsStore().$reset?.()
|
||||||
|
const t = inflight.claim()
|
||||||
try {
|
try {
|
||||||
overview.value = await api.get(`/api/artist/${encodeURIComponent(slug)}`)
|
const body = await api.get(`/api/artist/${encodeURIComponent(slug)}`)
|
||||||
|
if (!t.isCurrent()) return
|
||||||
|
overview.value = body
|
||||||
await loadMoreImages(slug)
|
await loadMoreImages(slug)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (!t.isCurrent()) return
|
||||||
if (e.status === 404) notFound.value = true
|
if (e.status === 404) notFound.value = true
|
||||||
else error.value = e.message
|
else error.value = e.message
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
if (t.isCurrent()) loading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -44,19 +54,22 @@ export const useArtistStore = defineStore('artist', () => {
|
|||||||
if (imagesLoading.value) return
|
if (imagesLoading.value) return
|
||||||
if (started && nextCursor.value === null) return
|
if (started && nextCursor.value === null) return
|
||||||
imagesLoading.value = true
|
imagesLoading.value = true
|
||||||
|
const t = inflight.claim()
|
||||||
try {
|
try {
|
||||||
const params = { limit: PAGE }
|
const params = { limit: PAGE }
|
||||||
if (nextCursor.value) params.cursor = nextCursor.value
|
if (nextCursor.value) params.cursor = nextCursor.value
|
||||||
const body = await api.get(
|
const body = await api.get(
|
||||||
`/api/artist/${encodeURIComponent(slug)}/images`, { params }
|
`/api/artist/${encodeURIComponent(slug)}/images`, { params }
|
||||||
)
|
)
|
||||||
|
if (!t.isCurrent()) return
|
||||||
images.value.push(...body.images)
|
images.value.push(...body.images)
|
||||||
nextCursor.value = body.next_cursor
|
nextCursor.value = body.next_cursor
|
||||||
started = true
|
started = true
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (!t.isCurrent()) return
|
||||||
error.value = e.message
|
error.value = e.message
|
||||||
} finally {
|
} finally {
|
||||||
imagesLoading.value = false
|
if (t.isCurrent()) imagesLoading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { defineStore } from 'pinia'
|
|||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
|
|
||||||
const PAGE = 60
|
const PAGE = 60
|
||||||
|
|
||||||
@@ -13,16 +14,23 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
|||||||
const q = ref('')
|
const q = ref('')
|
||||||
const platform = ref(null)
|
const platform = ref(null)
|
||||||
let started = false
|
let started = false
|
||||||
|
// Typed "alice" then "alice bob" used to drop the second fetch
|
||||||
|
// entirely (loading flag still true from the first), so the UI
|
||||||
|
// showed alice results while the input said "alice bob". Inflight
|
||||||
|
// token + reset() cancelling in-flight requests fixes both: the
|
||||||
|
// first response is discarded, the second is fetched. Audit 2026-06-02.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
async function loadMore() {
|
async function loadMore() {
|
||||||
if (loading.value) return
|
|
||||||
if (started && nextCursor.value === null) return
|
if (started && nextCursor.value === null) return
|
||||||
|
const t = inflight.claim()
|
||||||
await run(async () => {
|
await run(async () => {
|
||||||
const params = { limit: PAGE }
|
const params = { limit: PAGE }
|
||||||
if (q.value) params.q = q.value
|
if (q.value) params.q = q.value
|
||||||
if (platform.value) params.platform = platform.value
|
if (platform.value) params.platform = platform.value
|
||||||
if (nextCursor.value) params.cursor = nextCursor.value
|
if (nextCursor.value) params.cursor = nextCursor.value
|
||||||
const body = await api.get('/api/artists/directory', { params })
|
const body = await api.get('/api/artists/directory', { params })
|
||||||
|
if (!t.isCurrent()) return
|
||||||
cards.value.push(...body.cards)
|
cards.value.push(...body.cards)
|
||||||
nextCursor.value = body.next_cursor
|
nextCursor.value = body.next_cursor
|
||||||
started = true
|
started = true
|
||||||
@@ -30,6 +38,7 @@ export const useArtistDirectoryStore = defineStore('artistDirectory', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function reset() {
|
async function reset() {
|
||||||
|
inflight.cancel()
|
||||||
cards.value = []
|
cards.value = []
|
||||||
nextCursor.value = null
|
nextCursor.value = null
|
||||||
started = false
|
started = false
|
||||||
|
|||||||
@@ -23,7 +23,11 @@ export const useCredentialsStore = defineStore('credentials', () => {
|
|||||||
const rec = await api.post('/api/credentials', {
|
const rec = await api.post('/api/credentials', {
|
||||||
body: { platform, credential_type, data },
|
body: { platform, credential_type, data },
|
||||||
})
|
})
|
||||||
byPlatform.value.delete(platform)
|
// Reflect the returned record immediately — the previous .delete()
|
||||||
|
// call left the card rendering "no credential" for the gap between
|
||||||
|
// upload completion and the caller's follow-up loadAll(). Audit
|
||||||
|
// 2026-06-02.
|
||||||
|
byPlatform.value.set(platform, rec)
|
||||||
return rec
|
return rec
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { defineStore } from 'pinia'
|
|||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
|
|
||||||
export const useDownloadsStore = defineStore('downloads', () => {
|
export const useDownloadsStore = defineStore('downloads', () => {
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
@@ -22,6 +23,10 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
|||||||
// the "active now" panel always reflects what's happening regardless of
|
// the "active now" panel always reflects what's happening regardless of
|
||||||
// how the operator has filtered the historical list below.
|
// how the operator has filtered the historical list below.
|
||||||
const activeEvents = ref([])
|
const activeEvents = ref([])
|
||||||
|
// Filter changes (applyFilter) and rapid pagination can interleave
|
||||||
|
// responses; without an inflight guard the late response from a
|
||||||
|
// prior filter overwrites the current view. Audit 2026-06-02.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
function _params(extra = {}) {
|
function _params(extra = {}) {
|
||||||
const out = { limit: 50, ...extra }
|
const out = { limit: 50, ...extra }
|
||||||
@@ -32,8 +37,10 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function loadFirst() {
|
async function loadFirst() {
|
||||||
|
const t = inflight.claim()
|
||||||
await run(async () => {
|
await run(async () => {
|
||||||
const body = await api.get('/api/downloads', { params: _params() })
|
const body = await api.get('/api/downloads', { params: _params() })
|
||||||
|
if (!t.isCurrent()) return
|
||||||
events.value = body
|
events.value = body
|
||||||
cursor.value = body.length ? body[body.length - 1].id : null
|
cursor.value = body.length ? body[body.length - 1].id : null
|
||||||
hasMore.value = body.length === 50
|
hasMore.value = body.length === 50
|
||||||
@@ -42,8 +49,10 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
|||||||
|
|
||||||
async function loadMore() {
|
async function loadMore() {
|
||||||
if (!hasMore.value || cursor.value == null) return
|
if (!hasMore.value || cursor.value == null) return
|
||||||
|
const t = inflight.claim()
|
||||||
await run(async () => {
|
await run(async () => {
|
||||||
const body = await api.get('/api/downloads', { params: _params({ before: cursor.value }) })
|
const body = await api.get('/api/downloads', { params: _params({ before: cursor.value }) })
|
||||||
|
if (!t.isCurrent()) return
|
||||||
events.value.push(...body)
|
events.value.push(...body)
|
||||||
cursor.value = body.length ? body[body.length - 1].id : cursor.value
|
cursor.value = body.length ? body[body.length - 1].id : cursor.value
|
||||||
hasMore.value = body.length === 50
|
hasMore.value = body.length === 50
|
||||||
@@ -55,7 +64,25 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
|||||||
return selected.value
|
return selected.value
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Open the detail modal for the most recent DownloadEvent of a given
|
||||||
|
// source. Used by the failing-sources rollup's "Logs" button so the
|
||||||
|
// operator can troubleshoot without leaving the Downloads tab to find
|
||||||
|
// the row (operator-flagged 2026-06-01).
|
||||||
|
async function loadLastForSource(sourceId) {
|
||||||
|
const events = await api.get('/api/downloads', {
|
||||||
|
params: { source_id: sourceId, limit: 1 },
|
||||||
|
})
|
||||||
|
if (!events.length) {
|
||||||
|
selected.value = null
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
return await loadOne(events[0].id)
|
||||||
|
}
|
||||||
|
|
||||||
async function applyFilter(patch) {
|
async function applyFilter(patch) {
|
||||||
|
// Drop any in-flight loadFirst/loadMore from the previous filter
|
||||||
|
// so its late response doesn't overwrite this filter's results.
|
||||||
|
inflight.cancel()
|
||||||
filter.value = { ...filter.value, ...patch }
|
filter.value = { ...filter.value, ...patch }
|
||||||
await loadFirst()
|
await loadFirst()
|
||||||
}
|
}
|
||||||
@@ -98,7 +125,8 @@ export const useDownloadsStore = defineStore('downloads', () => {
|
|||||||
return {
|
return {
|
||||||
events, cursor, hasMore, filter, selected, loading, error, stats,
|
events, cursor, hasMore, filter, selected, loading, error, stats,
|
||||||
activity, failing, activeEvents,
|
activity, failing, activeEvents,
|
||||||
loadFirst, loadMore, loadOne, applyFilter, closeDetail, loadStats,
|
loadFirst, loadMore, loadOne, loadLastForSource, applyFilter,
|
||||||
|
closeDetail, loadStats,
|
||||||
loadActivity, loadFailing, loadActive, recoverStalled,
|
loadActivity, loadFailing, loadActive, recoverStalled,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { defineStore } from 'pinia'
|
import { defineStore } from 'pinia'
|
||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
|
|
||||||
// Operator-confirmed 2026-05-30: fetch PAGE-sized chunks instead of one
|
// Operator-confirmed 2026-05-30: fetch PAGE-sized chunks instead of one
|
||||||
// 50-item request so items render as each batch lands. Total initial
|
// 50-item request so items render as each batch lands. Total initial
|
||||||
@@ -22,9 +23,12 @@ export const useGalleryStore = defineStore('gallery', () => {
|
|||||||
const timelineBuckets = ref([])
|
const timelineBuckets = ref([])
|
||||||
const timelineLoading = ref(false)
|
const timelineLoading = ref(false)
|
||||||
|
|
||||||
let inflightId = 0
|
// Was a hand-rolled inflightId counter; the audit-2026-06-02 fan-out
|
||||||
|
// moved this pattern into useInflightToken so every store can share it.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
async function loadInitial() {
|
async function loadInitial() {
|
||||||
|
inflight.cancel()
|
||||||
images.value = []
|
images.value = []
|
||||||
dateGroups.value = []
|
dateGroups.value = []
|
||||||
nextCursor.value = null
|
nextCursor.value = null
|
||||||
@@ -41,19 +45,19 @@ export const useGalleryStore = defineStore('gallery', () => {
|
|||||||
if (loading.value) return
|
if (loading.value) return
|
||||||
loading.value = true
|
loading.value = true
|
||||||
error.value = null
|
error.value = null
|
||||||
const myId = ++inflightId
|
const t = inflight.claim()
|
||||||
try {
|
try {
|
||||||
const params = { limit: PAGE, ...activeFilterParam() }
|
const params = { limit: PAGE, ...activeFilterParam() }
|
||||||
if (nextCursor.value) params.cursor = nextCursor.value
|
if (nextCursor.value) params.cursor = nextCursor.value
|
||||||
const body = await api.get('/api/gallery/scroll', { params })
|
const body = await api.get('/api/gallery/scroll', { params })
|
||||||
if (myId !== inflightId) return // stale response
|
if (!t.isCurrent()) return
|
||||||
images.value.push(...body.images)
|
images.value.push(...body.images)
|
||||||
dateGroups.value = mergeGroups(dateGroups.value, body.date_groups)
|
dateGroups.value = mergeGroups(dateGroups.value, body.date_groups)
|
||||||
nextCursor.value = body.next_cursor
|
nextCursor.value = body.next_cursor
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
error.value = e.message
|
error.value = e.message
|
||||||
} finally {
|
} finally {
|
||||||
if (myId === inflightId) loading.value = false
|
if (t.isCurrent()) loading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -68,8 +72,14 @@ export const useGalleryStore = defineStore('gallery', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function jumpTo(year, month) {
|
async function jumpTo(year, month) {
|
||||||
|
// Rapid timeline-jump clicks need the same race guard as
|
||||||
|
// loadMore — first jump's late body could clobber the second
|
||||||
|
// jump's already-applied state.
|
||||||
|
inflight.cancel()
|
||||||
|
const t = inflight.claim()
|
||||||
const params = { year, month, ...activeFilterParam() }
|
const params = { year, month, ...activeFilterParam() }
|
||||||
const body = await api.get('/api/gallery/jump', { params })
|
const body = await api.get('/api/gallery/jump', { params })
|
||||||
|
if (!t.isCurrent()) return
|
||||||
if (body.cursor) {
|
if (body.cursor) {
|
||||||
images.value = []
|
images.value = []
|
||||||
dateGroups.value = []
|
dateGroups.value = []
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { toast } from '../utils/toast.js'
|
|||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
|
|
||||||
export const useModalStore = defineStore('modal', () => {
|
export const useModalStore = defineStore('modal', () => {
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
@@ -10,6 +11,11 @@ export const useModalStore = defineStore('modal', () => {
|
|||||||
const currentImageId = ref(null)
|
const currentImageId = ref(null)
|
||||||
const current = ref(null)
|
const current = ref(null)
|
||||||
const { loading, error, run } = useAsyncAction({ errorAs: 'message' })
|
const { loading, error, run } = useAsyncAction({ errorAs: 'message' })
|
||||||
|
// Tag mutations interpolate the image id into the URL after an
|
||||||
|
// await; without an inflight token, a fast prev/next can route the
|
||||||
|
// DELETE/POST to the wrong image AND the response to the wrong
|
||||||
|
// chip rail. Audit 2026-06-02.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
// Post-scoped cycle. When set, prev/next cycles within this array
|
// Post-scoped cycle. When set, prev/next cycles within this array
|
||||||
// (used by PostCard's expanded-mosaic PostImageGrid clicks). When
|
// (used by PostCard's expanded-mosaic PostImageGrid clicks). When
|
||||||
@@ -19,6 +25,9 @@ export const useModalStore = defineStore('modal', () => {
|
|||||||
const postImageIndex = ref(0)
|
const postImageIndex = ref(0)
|
||||||
|
|
||||||
async function open (id, opts = {}) {
|
async function open (id, opts = {}) {
|
||||||
|
// Cancel any in-flight tag mutation or reloadTags from the
|
||||||
|
// previous image so its late response can't apply to this one.
|
||||||
|
inflight.cancel()
|
||||||
currentImageId.value = id
|
currentImageId.value = id
|
||||||
current.value = null // cleared upfront so it stays null on error
|
current.value = null // cleared upfront so it stays null on error
|
||||||
// Update post-scoped state if caller passed it; otherwise clear so
|
// Update post-scoped state if caller passed it; otherwise clear so
|
||||||
@@ -31,12 +40,16 @@ export const useModalStore = defineStore('modal', () => {
|
|||||||
postImageIds.value = null
|
postImageIds.value = null
|
||||||
postImageIndex.value = 0
|
postImageIndex.value = 0
|
||||||
}
|
}
|
||||||
|
const t = inflight.claim()
|
||||||
await run(async () => {
|
await run(async () => {
|
||||||
current.value = await api.get(`/api/gallery/image/${id}`)
|
const body = await api.get(`/api/gallery/image/${id}`)
|
||||||
|
if (!t.isCurrent()) return
|
||||||
|
current.value = body
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
async function close () {
|
async function close () {
|
||||||
|
inflight.cancel()
|
||||||
currentImageId.value = null
|
currentImageId.value = null
|
||||||
current.value = null
|
current.value = null
|
||||||
error.value = null
|
error.value = null
|
||||||
@@ -75,37 +88,87 @@ export const useModalStore = defineStore('modal', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function reloadTags () {
|
async function reloadTags () {
|
||||||
if (!currentImageId.value) return
|
// Capture image id at call-time. After an await, currentImageId
|
||||||
const tags = await api.get(`/api/images/${currentImageId.value}/tags`)
|
// may have advanced via prev/next navigation; without capture, the
|
||||||
if (current.value) current.value.tags = tags
|
// GET would target the new image and write its tags onto a chip
|
||||||
|
// rail the user didn't open. Audit 2026-06-02.
|
||||||
|
const imageId = currentImageId.value
|
||||||
|
if (!imageId) return
|
||||||
|
const t = inflight.claim()
|
||||||
|
const tags = await api.get(`/api/images/${imageId}/tags`)
|
||||||
|
if (!t.isCurrent()) return
|
||||||
|
// Only commit if the modal is still showing this image — guards
|
||||||
|
// against close() / nav clearing current between the await and now.
|
||||||
|
if (current.value && currentImageId.value === imageId) {
|
||||||
|
current.value.tags = tags
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function removeTag (tagId) {
|
async function removeTag (tagId) {
|
||||||
if (!currentImageId.value) return
|
const imageId = currentImageId.value
|
||||||
|
if (!imageId) return
|
||||||
const prev = current.value.tags
|
const prev = current.value.tags
|
||||||
|
// Optimistic UI: drop the chip immediately.
|
||||||
current.value.tags = current.value.tags.filter(t => t.id !== tagId)
|
current.value.tags = current.value.tags.filter(t => t.id !== tagId)
|
||||||
|
// Split the two POSTs so a dismiss failure (secondary side-effect)
|
||||||
|
// doesn't roll back the successful DELETE — previously the catch
|
||||||
|
// unconditionally restored the chip rail even when only the
|
||||||
|
// dismiss had failed, so the UI lied until refresh. Audit 2026-06-02.
|
||||||
try {
|
try {
|
||||||
await api.delete(`/api/images/${currentImageId.value}/tags/${tagId}`)
|
await api.delete(`/api/images/${imageId}/tags/${tagId}`)
|
||||||
await api.post(`/api/images/${currentImageId.value}/suggestions/dismiss`, {
|
} catch (e) {
|
||||||
|
// Real failure: roll back, surface, rethrow.
|
||||||
|
if (current.value && currentImageId.value === imageId) {
|
||||||
|
current.value.tags = prev
|
||||||
|
}
|
||||||
|
toast({ text: `Failed to remove tag: ${e.message}`, type: 'error' })
|
||||||
|
throw e
|
||||||
|
}
|
||||||
|
// DELETE landed. The dismiss is fire-and-best-effort — log on
|
||||||
|
// failure but DON'T roll back the chip rail; the tag is gone
|
||||||
|
// server-side regardless.
|
||||||
|
try {
|
||||||
|
await api.post(`/api/images/${imageId}/suggestions/dismiss`, {
|
||||||
body: { tag_id: tagId },
|
body: { tag_id: tagId },
|
||||||
})
|
})
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
current.value.tags = prev
|
toast({
|
||||||
toast({ text: `Failed to remove tag: ${e.message}`, type: 'error' })
|
text: `Tag removed, but failed to dismiss suggestion: ${e.message}`,
|
||||||
throw e
|
type: 'warning',
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function addExistingTag (tagId) {
|
async function addExistingTag (tagId) {
|
||||||
if (!currentImageId.value) return
|
const imageId = currentImageId.value
|
||||||
await api.post(`/api/images/${currentImageId.value}/tags`, {
|
if (!imageId) return
|
||||||
|
await api.post(`/api/images/${imageId}/tags`, {
|
||||||
body: { tag_id: tagId, source: 'manual' },
|
body: { tag_id: tagId, source: 'manual' },
|
||||||
})
|
})
|
||||||
await reloadTags()
|
await reloadTags()
|
||||||
}
|
}
|
||||||
|
|
||||||
async function createAndAdd ({ name, kind, fandom_id = null }) {
|
async function createAndAdd ({ name, kind, fandom_id = null }) {
|
||||||
|
// Capture imageId so the post-create reloadTags / addExistingTag
|
||||||
|
// flow stays bound to the image the user clicked on, even if
|
||||||
|
// they navigated during the /api/tags POST.
|
||||||
|
const imageId = currentImageId.value
|
||||||
|
if (!imageId) return
|
||||||
const tag = await api.post('/api/tags', { body: { name, kind, fandom_id } })
|
const tag = await api.post('/api/tags', { body: { name, kind, fandom_id } })
|
||||||
|
// Audit 2026-06-02: a kind='fandom' created here used to be
|
||||||
|
// invisible to FandomPicker until a full page reload — its load
|
||||||
|
// gates on fandomCache.length, so a non-empty cache skips the
|
||||||
|
// refetch and the new fandom never appears. Push it into the
|
||||||
|
// cache directly so the next open sees it.
|
||||||
|
if (kind === 'fandom') {
|
||||||
|
const { useTagStore } = await import('./tags.js')
|
||||||
|
const tagStore = useTagStore()
|
||||||
|
tagStore.fandomCache.push({
|
||||||
|
id: tag.id, name: tag.name, kind: 'fandom',
|
||||||
|
fandom_id: null, fandom_name: null, image_count: 0,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if (currentImageId.value !== imageId) return // navigated away
|
||||||
await addExistingTag(tag.id)
|
await addExistingTag(tag.id)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { defineStore } from 'pinia'
|
|||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
|
|
||||||
export const usePostsStore = defineStore('posts', () => {
|
export const usePostsStore = defineStore('posts', () => {
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
@@ -19,6 +20,12 @@ export const usePostsStore = defineStore('posts', () => {
|
|||||||
const doneOlder = ref(false)
|
const doneOlder = ref(false)
|
||||||
const doneNewer = ref(false)
|
const doneNewer = ref(false)
|
||||||
const anchorId = ref(null)
|
const anchorId = ref(null)
|
||||||
|
// loadInitial, loadMore, loadAround, loadOlder, loadNewer all share
|
||||||
|
// one `loading` flag and previously had no inflight guard. A filter
|
||||||
|
// change (loadInitial) racing a still-in-flight loadMore would
|
||||||
|
// append the prior filter's items into the new filter's feed.
|
||||||
|
// Audit 2026-06-02.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
function _qs() {
|
function _qs() {
|
||||||
const q = {}
|
const q = {}
|
||||||
@@ -36,6 +43,7 @@ export const usePostsStore = defineStore('posts', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function loadInitial(newFilters) {
|
async function loadInitial(newFilters) {
|
||||||
|
inflight.cancel()
|
||||||
filters.value = {
|
filters.value = {
|
||||||
artist_id: newFilters?.artist_id ?? null,
|
artist_id: newFilters?.artist_id ?? null,
|
||||||
platform: newFilters?.platform ?? null,
|
platform: newFilters?.platform ?? null,
|
||||||
@@ -46,8 +54,10 @@ export const usePostsStore = defineStore('posts', () => {
|
|||||||
|
|
||||||
async function loadMore() {
|
async function loadMore() {
|
||||||
if (loading.value || done.value) return
|
if (loading.value || done.value) return
|
||||||
|
const t = inflight.claim()
|
||||||
await run(async () => {
|
await run(async () => {
|
||||||
const body = await api.get('/api/posts', { params: _qs() })
|
const body = await api.get('/api/posts', { params: _qs() })
|
||||||
|
if (!t.isCurrent()) return
|
||||||
items.value.push(...body.items)
|
items.value.push(...body.items)
|
||||||
cursor.value = body.next_cursor
|
cursor.value = body.next_cursor
|
||||||
if (body.next_cursor == null) done.value = true
|
if (body.next_cursor == null) done.value = true
|
||||||
@@ -59,14 +69,40 @@ export const usePostsStore = defineStore('posts', () => {
|
|||||||
return await api.get(`/api/posts/${id}`)
|
return await api.get(`/api/posts/${id}`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Filter overlay for the around/older/newer (in-context anchored)
|
||||||
|
// path. Keep this distinct from `filters.value` (the down-only feed)
|
||||||
|
// so a normal-feed filter change doesn't leak into an active anchored
|
||||||
|
// view (or vice versa). Caller of loadAround passes the snapshot; the
|
||||||
|
// subsequent loadOlder/loadNewer use it verbatim.
|
||||||
|
function _aroundParams(extra) {
|
||||||
|
const p = { ...extra }
|
||||||
|
if (filters.value.artist_id != null) p.artist_id = filters.value.artist_id
|
||||||
|
if (filters.value.platform) p.platform = filters.value.platform
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
// Load a window centered on `postId`: newer posts above, the post, older
|
// Load a window centered on `postId`: newer posts above, the post, older
|
||||||
// posts below. Sets both directional cursors for subsequent scrolling.
|
// posts below. Sets both directional cursors for subsequent scrolling.
|
||||||
async function loadAround(postId) {
|
// Accepts the same filter shape as loadInitial so the anchored view
|
||||||
|
// stays artist/platform-scoped (operator-flagged 2026-06-01: clicking a
|
||||||
|
// post title from the modal's Provenance card opens the post in the
|
||||||
|
// posts feed; without this the older/newer scroll loaded unfiltered
|
||||||
|
// global posts instead of staying in the artist's stream).
|
||||||
|
async function loadAround(postId, newFilters) {
|
||||||
|
inflight.cancel()
|
||||||
|
filters.value = {
|
||||||
|
artist_id: newFilters?.artist_id ?? null,
|
||||||
|
platform: newFilters?.platform ?? null,
|
||||||
|
}
|
||||||
loading.value = true
|
loading.value = true
|
||||||
error.value = null
|
error.value = null
|
||||||
anchorId.value = null
|
anchorId.value = null
|
||||||
|
const t = inflight.claim()
|
||||||
try {
|
try {
|
||||||
const body = await api.get('/api/posts', { params: { around: postId } })
|
const body = await api.get('/api/posts', {
|
||||||
|
params: _aroundParams({ around: postId }),
|
||||||
|
})
|
||||||
|
if (!t.isCurrent()) return
|
||||||
items.value = body.items
|
items.value = body.items
|
||||||
cursorOlder.value = body.cursor_older
|
cursorOlder.value = body.cursor_older
|
||||||
cursorNewer.value = body.cursor_newer
|
cursorNewer.value = body.cursor_newer
|
||||||
@@ -74,43 +110,54 @@ export const usePostsStore = defineStore('posts', () => {
|
|||||||
doneNewer.value = body.cursor_newer == null
|
doneNewer.value = body.cursor_newer == null
|
||||||
anchorId.value = body.anchor_id
|
anchorId.value = body.anchor_id
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (!t.isCurrent()) return
|
||||||
error.value = e
|
error.value = e
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
if (t.isCurrent()) loading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadOlder() {
|
async function loadOlder() {
|
||||||
if (loading.value || doneOlder.value || cursorOlder.value == null) return
|
if (loading.value || doneOlder.value || cursorOlder.value == null) return
|
||||||
loading.value = true
|
loading.value = true
|
||||||
|
const t = inflight.claim()
|
||||||
try {
|
try {
|
||||||
const body = await api.get('/api/posts', {
|
const body = await api.get('/api/posts', {
|
||||||
params: { cursor: cursorOlder.value, direction: 'older' },
|
params: _aroundParams({
|
||||||
|
cursor: cursorOlder.value, direction: 'older',
|
||||||
|
}),
|
||||||
})
|
})
|
||||||
|
if (!t.isCurrent()) return
|
||||||
items.value.push(...body.items)
|
items.value.push(...body.items)
|
||||||
cursorOlder.value = body.next_cursor
|
cursorOlder.value = body.next_cursor
|
||||||
if (body.next_cursor == null) doneOlder.value = true
|
if (body.next_cursor == null) doneOlder.value = true
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (!t.isCurrent()) return
|
||||||
error.value = e
|
error.value = e
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
if (t.isCurrent()) loading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadNewer() {
|
async function loadNewer() {
|
||||||
if (loading.value || doneNewer.value || cursorNewer.value == null) return
|
if (loading.value || doneNewer.value || cursorNewer.value == null) return
|
||||||
loading.value = true
|
loading.value = true
|
||||||
|
const t = inflight.claim()
|
||||||
try {
|
try {
|
||||||
const body = await api.get('/api/posts', {
|
const body = await api.get('/api/posts', {
|
||||||
params: { cursor: cursorNewer.value, direction: 'newer' },
|
params: _aroundParams({
|
||||||
|
cursor: cursorNewer.value, direction: 'newer',
|
||||||
|
}),
|
||||||
})
|
})
|
||||||
|
if (!t.isCurrent()) return
|
||||||
items.value.unshift(...body.items)
|
items.value.unshift(...body.items)
|
||||||
cursorNewer.value = body.next_cursor
|
cursorNewer.value = body.next_cursor
|
||||||
if (body.next_cursor == null) doneNewer.value = true
|
if (body.next_cursor == null) doneNewer.value = true
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
if (!t.isCurrent()) return
|
||||||
error.value = e
|
error.value = e
|
||||||
} finally {
|
} finally {
|
||||||
loading.value = false
|
if (t.isCurrent()) loading.value = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -19,6 +19,16 @@ import { useApi } from '../composables/useApi.js'
|
|||||||
const PAGE = 3
|
const PAGE = 3
|
||||||
const INITIAL_BATCHES = 20
|
const INITIAL_BATCHES = 20
|
||||||
const APPEND_DELAY_MS = 80 // ≈ the MasonryGrid stagger animation (70 ms)
|
const APPEND_DELAY_MS = 80 // ≈ the MasonryGrid stagger animation (70 ms)
|
||||||
|
// Operator-flagged 2026-06-01: scrolling the showcase eventually hit a
|
||||||
|
// premature "End." because /api/showcase returns a *random sample* and
|
||||||
|
// after enough scrolling the `seen` Set accumulated enough to fully
|
||||||
|
// collide with a 3-item batch. The showcase is supposed to be endless;
|
||||||
|
// only a genuinely empty API response (library has zero images) should
|
||||||
|
// mark it exhausted. Retry up to FETCH_RETRY_CAP times on all-dupe
|
||||||
|
// batches; only flip `exhausted` when the API returns 0 items OR every
|
||||||
|
// retry came back dupe-only (graceful fallback for tiny libraries
|
||||||
|
// where retries will keep returning the same handful of items).
|
||||||
|
const FETCH_RETRY_CAP = 8
|
||||||
|
|
||||||
|
|
||||||
function _sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
|
function _sleep(ms) { return new Promise(r => setTimeout(r, ms)) }
|
||||||
@@ -48,17 +58,32 @@ export const useShowcaseStore = defineStore('showcase', () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Single batch — used by infinite-scroll appends. Trickles its 5 items
|
// Single batch — used by infinite-scroll appends. Trickles its items
|
||||||
// in for the same one-at-a-time cadence as the initial load.
|
// in for the same one-at-a-time cadence as the initial load. Retries
|
||||||
|
// up to FETCH_RETRY_CAP times when the API's random sample comes back
|
||||||
|
// all-duplicates (the showcase is endless by design; only a genuinely
|
||||||
|
// empty API response should mark it exhausted, not an unlucky sample).
|
||||||
async function fetchPage() {
|
async function fetchPage() {
|
||||||
if (loading.value) return
|
if (loading.value) return
|
||||||
loading.value = true
|
loading.value = true
|
||||||
error.value = null
|
error.value = null
|
||||||
try {
|
try {
|
||||||
const body = await api.get('/api/showcase', { params: { limit: PAGE } })
|
for (let attempt = 0; attempt < FETCH_RETRY_CAP; attempt++) {
|
||||||
const fresh = (body.images || []).filter(i => !seen.has(i.id))
|
const body = await api.get('/api/showcase', { params: { limit: PAGE } })
|
||||||
if (fresh.length === 0) { exhausted.value = true; return }
|
const items = body.images || []
|
||||||
await _trickleAppend(fresh, _seq)
|
// API genuinely empty → library is empty / endpoint exhausted.
|
||||||
|
if (items.length === 0) { exhausted.value = true; return }
|
||||||
|
const fresh = items.filter(i => !seen.has(i.id))
|
||||||
|
if (fresh.length > 0) {
|
||||||
|
await _trickleAppend(fresh, _seq)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// All-dupes batch — keep trying. Showcase is endless by intent.
|
||||||
|
}
|
||||||
|
// Retry cap hit with zero fresh items: library is probably much
|
||||||
|
// smaller than the running `seen` set, fall back to exhausted so
|
||||||
|
// the UI stops trying. Operator can shuffle to reset `seen`.
|
||||||
|
exhausted.value = true
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
error.value = e.message || String(e)
|
error.value = e.message || String(e)
|
||||||
} finally {
|
} finally {
|
||||||
|
|||||||
@@ -85,6 +85,15 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Plan #544: arm a source for backfill mode. The next `runs` download
|
||||||
|
// runs (default 3) walk gallery-dl's full post history instead of
|
||||||
|
// exiting early at the first contiguous archived block.
|
||||||
|
async function setBackfill(id, runs = 3, artistIdHint = null) {
|
||||||
|
const body = await api.post(`/api/sources/${id}/backfill`, { body: { runs } })
|
||||||
|
_invalidate(artistIdHint ?? body.artist_id)
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
function sourcesByArtistGrouped() {
|
function sourcesByArtistGrouped() {
|
||||||
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
||||||
const arr = byArtist.value.get(null) ?? []
|
const arr = byArtist.value.get(null) ?? []
|
||||||
@@ -111,6 +120,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
loadAll, loadForArtist,
|
loadAll, loadForArtist,
|
||||||
create, update, remove,
|
create, update, remove,
|
||||||
checkNow,
|
checkNow,
|
||||||
|
setBackfill,
|
||||||
findOrCreateArtist, autocompleteArtist,
|
findOrCreateArtist, autocompleteArtist,
|
||||||
loadScheduleStatus,
|
loadScheduleStatus,
|
||||||
sourcesByArtistGrouped,
|
sourcesByArtistGrouped,
|
||||||
|
|||||||
@@ -3,13 +3,14 @@ import { toast } from '../utils/toast.js'
|
|||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
|
|
||||||
// Category display order: people/sources first, general last.
|
// Category display order: people first, general last.
|
||||||
export const CATEGORY_ORDER = ['artist', 'character', 'copyright', 'general']
|
// 'artist' (FC-2d-vii-c) and 'copyright' (2026-06-01) retired — only
|
||||||
|
// character and general surface as suggestion categories now.
|
||||||
|
export const CATEGORY_ORDER = ['character', 'general']
|
||||||
export const CATEGORY_LABELS = {
|
export const CATEGORY_LABELS = {
|
||||||
artist: 'Artist',
|
|
||||||
character: 'Character',
|
character: 'Character',
|
||||||
copyright: 'Copyright',
|
|
||||||
general: 'General'
|
general: 'General'
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -18,12 +19,25 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
|
|||||||
const byCategory = ref({}) // { category: [suggestion, ...] }
|
const byCategory = ref({}) // { category: [suggestion, ...] }
|
||||||
const { loading, error, run } = useAsyncAction({ errorAs: 'message' })
|
const { loading, error, run } = useAsyncAction({ errorAs: 'message' })
|
||||||
let currentImageId = null
|
let currentImageId = null
|
||||||
|
// Audit 2026-06-02: this store had no inflight guard — a late
|
||||||
|
// /suggestions response from a prior image could overwrite
|
||||||
|
// byCategory while currentImageId pointed at a new one, and
|
||||||
|
// accept() dereferenced currentImageId AFTER an awaited POST so
|
||||||
|
// the subsequent /suggestions/accept could apply A's chosen tag
|
||||||
|
// to image B (and push it to the allowlist). Both fixed below
|
||||||
|
// by capturing imageId at call-time and gating writes on the token.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
async function load(imageId) {
|
async function load(imageId) {
|
||||||
|
// Cancel any in-flight load from the previous image so its late
|
||||||
|
// response can't overwrite this image's byCategory.
|
||||||
|
inflight.cancel()
|
||||||
currentImageId = imageId
|
currentImageId = imageId
|
||||||
byCategory.value = {} // cleared upfront so it stays empty on error
|
byCategory.value = {} // cleared upfront so it stays empty on error
|
||||||
|
const t = inflight.claim()
|
||||||
await run(async () => {
|
await run(async () => {
|
||||||
const body = await api.get(`/api/images/${imageId}/suggestions`)
|
const body = await api.get(`/api/images/${imageId}/suggestions`)
|
||||||
|
if (!t.isCurrent()) return
|
||||||
byCategory.value = body.by_category || {}
|
byCategory.value = body.by_category || {}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -35,6 +49,11 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function accept(suggestion) {
|
async function accept(suggestion) {
|
||||||
|
// Capture imageId so a mid-flight prev/next can't reroute the
|
||||||
|
// accept POST to a different image AND push the tag to that
|
||||||
|
// image's allowlist.
|
||||||
|
const imageId = currentImageId
|
||||||
|
if (imageId == null) return
|
||||||
// Raw tags (creates_new_tag) have no canonical_tag_id; the backend's
|
// Raw tags (creates_new_tag) have no canonical_tag_id; the backend's
|
||||||
// accept endpoint needs a tag_id, so for raw tags we create the tag
|
// accept endpoint needs a tag_id, so for raw tags we create the tag
|
||||||
// first via the existing /api/tags endpoint, then accept by id.
|
// first via the existing /api/tags endpoint, then accept by id.
|
||||||
@@ -45,10 +64,14 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
|
|||||||
})
|
})
|
||||||
tagId = created.id
|
tagId = created.id
|
||||||
}
|
}
|
||||||
await api.post(`/api/images/${currentImageId}/suggestions/accept`, {
|
await api.post(`/api/images/${imageId}/suggestions/accept`, {
|
||||||
body: { tag_id: tagId }
|
body: { tag_id: tagId }
|
||||||
})
|
})
|
||||||
_drop(suggestion.category, s => s === suggestion)
|
// Only drop from THIS image's category list — if the user navigated,
|
||||||
|
// the new image has its own suggestions and this drop would corrupt them.
|
||||||
|
if (currentImageId === imageId) {
|
||||||
|
_drop(suggestion.category, s => s === suggestion)
|
||||||
|
}
|
||||||
toast({
|
toast({
|
||||||
text: `Tagged: ${suggestion.display_name}`,
|
text: `Tagged: ${suggestion.display_name}`,
|
||||||
type: 'success'
|
type: 'success'
|
||||||
@@ -56,14 +79,18 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function aliasAccept(suggestion, canonicalTagId) {
|
async function aliasAccept(suggestion, canonicalTagId) {
|
||||||
await api.post(`/api/images/${currentImageId}/suggestions/alias`, {
|
const imageId = currentImageId
|
||||||
|
if (imageId == null) return
|
||||||
|
await api.post(`/api/images/${imageId}/suggestions/alias`, {
|
||||||
body: {
|
body: {
|
||||||
alias_string: suggestion.display_name,
|
alias_string: suggestion.display_name,
|
||||||
alias_category: suggestion.category,
|
alias_category: suggestion.category,
|
||||||
canonical_tag_id: canonicalTagId
|
canonical_tag_id: canonicalTagId
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
_drop(suggestion.category, s => s === suggestion)
|
if (currentImageId === imageId) {
|
||||||
|
_drop(suggestion.category, s => s === suggestion)
|
||||||
|
}
|
||||||
toast({
|
toast({
|
||||||
text: `Aliased & tagged: ${suggestion.display_name}`,
|
text: `Aliased & tagged: ${suggestion.display_name}`,
|
||||||
type: 'success'
|
type: 'success'
|
||||||
@@ -71,15 +98,19 @@ export const useSuggestionsStore = defineStore('suggestions', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function dismiss(suggestion) {
|
async function dismiss(suggestion) {
|
||||||
|
const imageId = currentImageId
|
||||||
|
if (imageId == null) return
|
||||||
// Dismiss needs a tag_id; raw tags have none, so dismissing a raw
|
// Dismiss needs a tag_id; raw tags have none, so dismissing a raw
|
||||||
// suggestion just hides it client-side (nothing to persist a rejection
|
// suggestion just hides it client-side (nothing to persist a rejection
|
||||||
// against until the tag exists).
|
// against until the tag exists).
|
||||||
if (suggestion.canonical_tag_id != null) {
|
if (suggestion.canonical_tag_id != null) {
|
||||||
await api.post(`/api/images/${currentImageId}/suggestions/dismiss`, {
|
await api.post(`/api/images/${imageId}/suggestions/dismiss`, {
|
||||||
body: { tag_id: suggestion.canonical_tag_id }
|
body: { tag_id: suggestion.canonical_tag_id }
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
_drop(suggestion.category, s => s === suggestion)
|
if (currentImageId === imageId) {
|
||||||
|
_drop(suggestion.category, s => s === suggestion)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
|||||||
// Live polled state.
|
// Live polled state.
|
||||||
const queues = ref(null) // { queues: {name: depth|null}, fetched_at }
|
const queues = ref(null) // { queues: {name: depth|null}, fetched_at }
|
||||||
const workers = ref(null) // { workers: {hostname: {...}}, fetched_at }
|
const workers = ref(null) // { workers: {hostname: {...}}, fetched_at }
|
||||||
const recentMinute = ref([]) // last-60s rows (for Overview summary)
|
const recentRuns = ref([]) // last-60s rows (for Overview summary)
|
||||||
const failures = ref(null) // { recent, count_by_type, since }
|
const failures = ref(null) // { recent, count_by_type, since }
|
||||||
|
|
||||||
// Paginated runs (Activity tab "All recent activity" pane).
|
// Paginated runs (Activity tab "All recent activity" pane).
|
||||||
@@ -45,14 +45,14 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadRecentMinute() {
|
async function loadRecentRuns() {
|
||||||
// Used by the Overview summary card: pull last 60s of runs to compute
|
// Used by the Overview summary card: pull last 60s of runs to compute
|
||||||
// per-queue ok/err counts. One call covers all queues; UI groups.
|
// per-queue ok/err counts. One call covers all queues; UI groups.
|
||||||
try {
|
try {
|
||||||
const body = await api.get('/api/system/activity/runs', {
|
const body = await api.get('/api/system/activity/runs', {
|
||||||
params: { limit: 200 },
|
params: { limit: 200 },
|
||||||
})
|
})
|
||||||
recentMinute.value = body.runs || []
|
recentRuns.value = body.runs || []
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
lastError.value = e.message
|
lastError.value = e.message
|
||||||
}
|
}
|
||||||
@@ -106,10 +106,10 @@ export const useSystemActivityStore = defineStore('systemActivity', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
queues, workers, recentMinute, failures, summary,
|
queues, workers, recentRuns, failures, summary,
|
||||||
runs, runsCursor, runsHasMore, runsFilter,
|
runs, runsCursor, runsHasMore, runsFilter,
|
||||||
loading, lastError,
|
loading, lastError,
|
||||||
loadQueues, loadWorkers, loadRecentMinute,
|
loadQueues, loadWorkers, loadRecentRuns,
|
||||||
loadRuns, loadFailures, loadSummary, setFilter,
|
loadRuns, loadFailures, loadSummary, setFilter,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { defineStore } from 'pinia'
|
|||||||
import { ref, computed } from 'vue'
|
import { ref, computed } from 'vue'
|
||||||
import { useApi } from '../composables/useApi.js'
|
import { useApi } from '../composables/useApi.js'
|
||||||
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||||
|
import { useInflightToken } from '../composables/useInflightToken.js'
|
||||||
|
|
||||||
const PAGE = 60
|
const PAGE = 60
|
||||||
|
|
||||||
@@ -13,16 +14,21 @@ export const useTagDirectoryStore = defineStore('tagDirectory', () => {
|
|||||||
const kind = ref(null)
|
const kind = ref(null)
|
||||||
const q = ref('')
|
const q = ref('')
|
||||||
let started = false
|
let started = false
|
||||||
|
// Same shape as artistDirectory — rapid setQuery/setKind dropped
|
||||||
|
// the second fetch because loading was still true from the first.
|
||||||
|
// Audit 2026-06-02.
|
||||||
|
const inflight = useInflightToken()
|
||||||
|
|
||||||
async function loadMore() {
|
async function loadMore() {
|
||||||
if (loading.value) return
|
|
||||||
if (started && nextCursor.value === null) return
|
if (started && nextCursor.value === null) return
|
||||||
|
const t = inflight.claim()
|
||||||
await run(async () => {
|
await run(async () => {
|
||||||
const params = { limit: PAGE }
|
const params = { limit: PAGE }
|
||||||
if (kind.value) params.kind = kind.value
|
if (kind.value) params.kind = kind.value
|
||||||
if (q.value) params.q = q.value
|
if (q.value) params.q = q.value
|
||||||
if (nextCursor.value) params.cursor = nextCursor.value
|
if (nextCursor.value) params.cursor = nextCursor.value
|
||||||
const body = await api.get('/api/tags/directory', { params })
|
const body = await api.get('/api/tags/directory', { params })
|
||||||
|
if (!t.isCurrent()) return
|
||||||
cards.value.push(...body.cards)
|
cards.value.push(...body.cards)
|
||||||
nextCursor.value = body.next_cursor
|
nextCursor.value = body.next_cursor
|
||||||
started = true
|
started = true
|
||||||
@@ -30,6 +36,7 @@ export const useTagDirectoryStore = defineStore('tagDirectory', () => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function reset() {
|
async function reset() {
|
||||||
|
inflight.cancel()
|
||||||
cards.value = []
|
cards.value = []
|
||||||
nextCursor.value = null
|
nextCursor.value = null
|
||||||
started = false
|
started = false
|
||||||
|
|||||||
@@ -4,8 +4,11 @@ import { useApi } from '../composables/useApi.js'
|
|||||||
export const useThumbnailsStore = defineStore('thumbnails', () => {
|
export const useThumbnailsStore = defineStore('thumbnails', () => {
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
|
|
||||||
|
// Returns { scanned, enqueued, ok, regenerated } — the API now runs
|
||||||
|
// the scan synchronously so the operator gets immediate feedback
|
||||||
|
// instead of just a Celery task id with no visible outcome.
|
||||||
async function triggerBackfill () {
|
async function triggerBackfill () {
|
||||||
await api.post('/api/thumbnails/backfill')
|
return await api.post('/api/thumbnails/backfill')
|
||||||
}
|
}
|
||||||
|
|
||||||
return { triggerBackfill }
|
return { triggerBackfill }
|
||||||
|
|||||||
@@ -24,7 +24,7 @@
|
|||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { onMounted, watch } from 'vue'
|
import { onMounted, watch } from 'vue'
|
||||||
import { useRoute, useRouter } from 'vue-router'
|
import { useRoute } from 'vue-router'
|
||||||
import { useGalleryStore } from '../stores/gallery.js'
|
import { useGalleryStore } from '../stores/gallery.js'
|
||||||
import { useModalStore } from '../stores/modal.js'
|
import { useModalStore } from '../stores/modal.js'
|
||||||
import GalleryGrid from '../components/gallery/GalleryGrid.vue'
|
import GalleryGrid from '../components/gallery/GalleryGrid.vue'
|
||||||
@@ -37,7 +37,6 @@ import { useGallerySelectionStore } from '../stores/gallerySelection.js'
|
|||||||
const store = useGalleryStore()
|
const store = useGalleryStore()
|
||||||
const modal = useModalStore()
|
const modal = useModalStore()
|
||||||
const sel = useGallerySelectionStore()
|
const sel = useGallerySelectionStore()
|
||||||
const router = useRouter()
|
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
@@ -47,9 +46,6 @@ onMounted(async () => {
|
|||||||
else if (!isNaN(tagId)) store.setTagFilter(tagId)
|
else if (!isNaN(tagId)) store.setTagFilter(tagId)
|
||||||
await store.loadInitial()
|
await store.loadInitial()
|
||||||
await store.loadTimeline()
|
await store.loadTimeline()
|
||||||
// Open modal if URL has ?image=N
|
|
||||||
const initial = parseInt(route.query.image, 10)
|
|
||||||
if (!isNaN(initial)) modal.open(initial)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(() => route.query.tag_id, (q) => {
|
watch(() => route.query.tag_id, (q) => {
|
||||||
@@ -64,19 +60,8 @@ watch(() => route.query.post_id, (q) => {
|
|||||||
store.setPostFilter(isNaN(postId) ? null : postId)
|
store.setPostFilter(isNaN(postId) ? null : postId)
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(() => route.query.image, (q) => {
|
|
||||||
const id = parseInt(q, 10)
|
|
||||||
if (!isNaN(id) && id !== modal.currentImageId) modal.open(id)
|
|
||||||
else if (isNaN(id) && modal.currentImageId !== null) modal.close()
|
|
||||||
})
|
|
||||||
|
|
||||||
function openImage(id) {
|
function openImage(id) {
|
||||||
router.push({ query: { ...route.query, image: id } })
|
modal.open(id)
|
||||||
}
|
|
||||||
function closeImage() {
|
|
||||||
const q = { ...route.query }
|
|
||||||
delete q.image
|
|
||||||
router.push({ query: q })
|
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -161,7 +161,15 @@ function setupAroundObservers() {
|
|||||||
}
|
}
|
||||||
async function loadAroundAndAnchor() {
|
async function loadAroundAndAnchor() {
|
||||||
teardownFeed()
|
teardownFeed()
|
||||||
await store.loadAround(postIdFilter.value)
|
// Pass artist_id + platform through so the anchored view stays
|
||||||
|
// scoped — the older/newer infinite scrolls then read these filters
|
||||||
|
// back via the store's _aroundParams (operator-flagged 2026-06-01:
|
||||||
|
// post-title click from the modal landed scoped but the scroll then
|
||||||
|
// pulled unfiltered global posts).
|
||||||
|
await store.loadAround(postIdFilter.value, {
|
||||||
|
artist_id: artistFilter.value,
|
||||||
|
platform: platformFilter.value,
|
||||||
|
})
|
||||||
await nextTick()
|
await nextTick()
|
||||||
const el = document.getElementById(`fc-post-${store.anchorId}`)
|
const el = document.getElementById(`fc-post-${store.anchorId}`)
|
||||||
if (el) el.scrollIntoView({ block: 'center' })
|
if (el) el.scrollIntoView({ block: 'center' })
|
||||||
|
|||||||
@@ -70,7 +70,12 @@ function onKeydown(e) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
onMounted(() => {
|
onMounted(() => {
|
||||||
if (store.images.length === 0) store.loadInitial()
|
// Operator-stated 2026-06-02: every load of the showcase view should
|
||||||
|
// show new images. Pinia persists the store across navigations, so
|
||||||
|
// returning to /showcase used to render the same set as before. Now
|
||||||
|
// we always reshuffle on mount — loadInitial() resets `seen`,
|
||||||
|
// `images`, and `exhausted` and pipelines a fresh batch.
|
||||||
|
store.loadInitial()
|
||||||
window.addEventListener('keydown', onKeydown)
|
window.addEventListener('keydown', onKeydown)
|
||||||
})
|
})
|
||||||
onUnmounted(() => window.removeEventListener('keydown', onKeydown))
|
onUnmounted(() => window.removeEventListener('keydown', onKeydown))
|
||||||
|
|||||||
@@ -27,12 +27,17 @@ describe('credentials store', () => {
|
|||||||
expect(s.byPlatform.get('patreon').credential_type).toBe('cookies')
|
expect(s.byPlatform.get('patreon').credential_type).toBe('cookies')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('upload invalidates the cache', async () => {
|
it('upload reflects the returned record into the cache', async () => {
|
||||||
|
// The previous behavior was to delete the cache entry on upload, which
|
||||||
|
// briefly showed "no credential" until a follow-up loadAll() resolved.
|
||||||
|
// Audit 2026-06-02 corrected this: upload now stores the returned
|
||||||
|
// record directly so the card updates immediately.
|
||||||
const s = useCredentialsStore()
|
const s = useCredentialsStore()
|
||||||
s.byPlatform.set('patreon', { platform: 'patreon' })
|
s.byPlatform.set('patreon', { platform: 'patreon', credential_type: 'token' })
|
||||||
stubFetch(() => ({ status: 201, body: { platform: 'patreon', credential_type: 'cookies' } }))
|
const fresh = { platform: 'patreon', credential_type: 'cookies' }
|
||||||
|
stubFetch(() => ({ status: 201, body: fresh }))
|
||||||
await s.upload('patreon', 'cookies', 'NETSCAPE_CONTENT')
|
await s.upload('patreon', 'cookies', 'NETSCAPE_CONTENT')
|
||||||
expect(s.byPlatform.has('patreon')).toBe(false)
|
expect(s.byPlatform.get('patreon')).toEqual(fresh)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('remove deletes and invalidates the cache', async () => {
|
it('remove deletes and invalidates the cache', async () => {
|
||||||
|
|||||||
+14
-7
@@ -8,18 +8,25 @@ migration code paths.
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import pytest
|
# Audit 2026-06-02: CredentialCrypto now refuses to auto-generate a
|
||||||
import pytest_asyncio
|
# Fernet key without explicit opt-in (production safety against silent
|
||||||
from sqlalchemy import create_engine
|
# key regeneration on partial restore). The test environment never has
|
||||||
from sqlalchemy.ext.asyncio import (
|
# a pre-seeded key file, so set the bootstrap flag here before any
|
||||||
|
# create_app() / CredentialCrypto() import path fires.
|
||||||
|
os.environ.setdefault("CURATOR_BOOTSTRAP_NEW_KEY", "1")
|
||||||
|
|
||||||
|
import pytest # noqa: E402
|
||||||
|
import pytest_asyncio # noqa: E402
|
||||||
|
from sqlalchemy import create_engine # noqa: E402
|
||||||
|
from sqlalchemy.ext.asyncio import ( # noqa: E402
|
||||||
AsyncSession,
|
AsyncSession,
|
||||||
async_sessionmaker,
|
async_sessionmaker,
|
||||||
create_async_engine,
|
create_async_engine,
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker # noqa: E402
|
||||||
|
|
||||||
from backend.app import create_app
|
from backend.app import create_app # noqa: E402
|
||||||
from backend.app.models import Base
|
from backend.app.models import Base # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
def _async_database_url() -> str:
|
def _async_database_url() -> str:
|
||||||
|
|||||||
@@ -37,8 +37,8 @@ async def test_artist_overview_post_count(client, db):
|
|||||||
)
|
)
|
||||||
db.add(s)
|
db.add(s)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
db.add(Post(source_id=s.id, external_post_id="p1"))
|
db.add(Post(source_id=s.id, artist_id=a.id, external_post_id="p1"))
|
||||||
db.add(Post(source_id=s.id, external_post_id="p2"))
|
db.add(Post(source_id=s.id, artist_id=a.id, external_post_id="p2"))
|
||||||
await db.flush()
|
await db.flush()
|
||||||
await db.commit()
|
await db.commit()
|
||||||
resp = await client.get("/api/artist/lyra")
|
resp = await client.get("/api/artist/lyra")
|
||||||
|
|||||||
@@ -19,7 +19,13 @@ async def test_get_and_patch_settings(client):
|
|||||||
resp = await client.get("/api/ml/settings")
|
resp = await client.get("/api/ml/settings")
|
||||||
assert resp.status_code == 200
|
assert resp.status_code == 200
|
||||||
body = await resp.get_json()
|
body = await resp.get_json()
|
||||||
assert body["suggestion_threshold_general"] == pytest.approx(0.95)
|
# Default raised 0.50 → 0.70 on 2026-06-02 (alembic 0033) — 0.50
|
||||||
|
# was too noisy in practice. The 0.70 default keeps the rail
|
||||||
|
# signal-rich without hiding everything like the original 0.95.
|
||||||
|
assert body["suggestion_threshold_general"] == pytest.approx(0.70)
|
||||||
|
# Retired threshold columns must not appear in the payload.
|
||||||
|
assert "suggestion_threshold_artist" not in body
|
||||||
|
assert "suggestion_threshold_copyright" not in body
|
||||||
|
|
||||||
resp = await client.patch(
|
resp = await client.patch(
|
||||||
"/api/ml/settings", json={"suggestion_threshold_general": 0.90}
|
"/api/ml/settings", json={"suggestion_threshold_general": 0.90}
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ async def seeded_post(db):
|
|||||||
db.add(source)
|
db.add(source)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
post = Post(
|
post = Post(
|
||||||
source_id=source.id, external_post_id="API1",
|
source_id=source.id, artist_id=artist.id, external_post_id="API1",
|
||||||
post_title="Hello", post_url="https://p/alice-api/1",
|
post_title="Hello", post_url="https://p/alice-api/1",
|
||||||
post_date=datetime.now(UTC),
|
post_date=datetime.now(UTC),
|
||||||
description="<p>hi</p>",
|
description="<p>hi</p>",
|
||||||
@@ -123,7 +123,8 @@ async def post_timeline(db):
|
|||||||
posts = []
|
posts = []
|
||||||
for i in range(5):
|
for i in range(5):
|
||||||
p = Post(
|
p = Post(
|
||||||
source_id=source.id, external_post_id=f"TL{i}",
|
source_id=source.id, artist_id=artist.id,
|
||||||
|
external_post_id=f"TL{i}",
|
||||||
post_title=f"post {i}", post_date=base + timedelta(days=i),
|
post_title=f"post {i}", post_date=base + timedelta(days=i),
|
||||||
)
|
)
|
||||||
db.add(p)
|
db.add(p)
|
||||||
@@ -189,7 +190,7 @@ async def test_detail_returns_uncapped_thumbnails(client, db):
|
|||||||
db.add(s)
|
db.add(s)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
p = Post(
|
p = Post(
|
||||||
source_id=s.id, external_post_id="DETAIL10",
|
source_id=s.id, artist_id=a.id, external_post_id="DETAIL10",
|
||||||
post_title="big post", description="<p>body</p>",
|
post_title="big post", description="<p>body</p>",
|
||||||
)
|
)
|
||||||
db.add(p)
|
db.add(p)
|
||||||
|
|||||||
@@ -26,7 +26,8 @@ async def _seed_full(db):
|
|||||||
url="https://patreon.test/alice")
|
url="https://patreon.test/alice")
|
||||||
db.add(source)
|
db.add(source)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
post = Post(source_id=source.id, external_post_id="555",
|
post = Post(source_id=source.id, artist_id=artist.id,
|
||||||
|
external_post_id="555",
|
||||||
post_url="https://patreon.test/p/555", post_title="Set 1",
|
post_url="https://patreon.test/p/555", post_title="Set 1",
|
||||||
post_date=datetime(2023, 8, 1, tzinfo=UTC),
|
post_date=datetime(2023, 8, 1, tzinfo=UTC),
|
||||||
description="<p>hi</p>", attachment_count=2)
|
description="<p>hi</p>", attachment_count=2)
|
||||||
|
|||||||
@@ -193,3 +193,61 @@ async def test_list_derives_next_check_at_when_last_checked_set(
|
|||||||
)
|
)
|
||||||
assert target["next_check_at"] is not None
|
assert target["next_check_at"] is not None
|
||||||
assert "T" in target["next_check_at"] # ISO 8601
|
assert "T" in target["next_check_at"] # ISO 8601
|
||||||
|
|
||||||
|
|
||||||
|
# --- Plan #544: POST /api/sources/{id}/backfill ----------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_endpoint_arms_source(client, artist, db):
|
||||||
|
src = Source(
|
||||||
|
artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://patreon.com/alice-backfill", enabled=True,
|
||||||
|
)
|
||||||
|
db.add(src)
|
||||||
|
await db.commit()
|
||||||
|
sid = src.id
|
||||||
|
|
||||||
|
resp = await client.post(f"/api/sources/{sid}/backfill", json={"runs": 5})
|
||||||
|
assert resp.status_code == 200
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["backfill_runs_remaining"] == 5
|
||||||
|
|
||||||
|
# GET reflects the new state.
|
||||||
|
one = await client.get(f"/api/sources/{sid}")
|
||||||
|
assert (await one.get_json())["backfill_runs_remaining"] == 5
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_endpoint_defaults_to_three(client, artist, db):
|
||||||
|
src = Source(
|
||||||
|
artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://patreon.com/alice-backfill-default", enabled=True,
|
||||||
|
)
|
||||||
|
db.add(src)
|
||||||
|
await db.commit()
|
||||||
|
resp = await client.post(f"/api/sources/{src.id}/backfill", json={})
|
||||||
|
body = await resp.get_json()
|
||||||
|
assert body["backfill_runs_remaining"] == 3
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_endpoint_rejects_out_of_range(client, artist, db):
|
||||||
|
src = Source(
|
||||||
|
artist_id=artist.id, platform="patreon",
|
||||||
|
url="https://patreon.com/alice-backfill-bad", enabled=True,
|
||||||
|
)
|
||||||
|
db.add(src)
|
||||||
|
await db.commit()
|
||||||
|
bad = await client.post(f"/api/sources/{src.id}/backfill", json={"runs": 0})
|
||||||
|
assert bad.status_code == 400
|
||||||
|
too_big = await client.post(
|
||||||
|
f"/api/sources/{src.id}/backfill", json={"runs": 99}
|
||||||
|
)
|
||||||
|
assert too_big.status_code == 400
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_endpoint_404_when_source_missing(client):
|
||||||
|
resp = await client.post("/api/sources/999999/backfill", json={"runs": 3})
|
||||||
|
assert resp.status_code == 404
|
||||||
|
|||||||
@@ -13,8 +13,16 @@ def eager():
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_trigger_thumbnail_backfill(client):
|
async def test_trigger_thumbnail_backfill_returns_counts(client):
|
||||||
|
"""The API runs the scan synchronously now and returns
|
||||||
|
{scanned, enqueued, ok, regenerated} — operator-flagged 2026-06-01:
|
||||||
|
the previous fire-and-forget shape returned only a celery_task_id,
|
||||||
|
so the admin UI couldn't show whether backfill found 0 or 5000
|
||||||
|
candidates. \"Found nothing\" was indistinguishable from \"the
|
||||||
|
worker isn't picking up the task.\""""
|
||||||
r = await client.post("/api/thumbnails/backfill")
|
r = await client.post("/api/thumbnails/backfill")
|
||||||
assert r.status_code == 202
|
assert r.status_code == 200
|
||||||
body = await r.get_json()
|
body = await r.get_json()
|
||||||
assert "celery_task_id" in body
|
assert set(body) == {"scanned", "enqueued", "ok", "regenerated"}
|
||||||
|
for key in ("scanned", "enqueued", "ok", "regenerated"):
|
||||||
|
assert isinstance(body[key], int)
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ async def _fixture(db):
|
|||||||
db.add(src)
|
db.add(src)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
post = Post(
|
post = Post(
|
||||||
source_id=src.id, external_post_id="p1",
|
source_id=src.id, artist_id=artist.id, external_post_id="p1",
|
||||||
post_date=datetime(2026, 3, 1, tzinfo=UTC),
|
post_date=datetime(2026, 3, 1, tzinfo=UTC),
|
||||||
)
|
)
|
||||||
db.add(post)
|
db.add(post)
|
||||||
|
|||||||
@@ -12,13 +12,16 @@ pytestmark = pytest.mark.integration
|
|||||||
|
|
||||||
def test_thumb_is_valid_jpeg(tmp_path):
|
def test_thumb_is_valid_jpeg(tmp_path):
|
||||||
p = tmp_path / "good.jpg"
|
p = tmp_path / "good.jpg"
|
||||||
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100)
|
# Real thumbnails are at least ~2KB; size check (MIN_THUMB_BYTES=256)
|
||||||
|
# requires the file body be plausible. 300 bytes here clears the
|
||||||
|
# floor with margin.
|
||||||
|
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 300)
|
||||||
assert _thumb_is_valid(p) is True
|
assert _thumb_is_valid(p) is True
|
||||||
|
|
||||||
|
|
||||||
def test_thumb_is_valid_png(tmp_path):
|
def test_thumb_is_valid_png(tmp_path):
|
||||||
p = tmp_path / "good.png"
|
p = tmp_path / "good.png"
|
||||||
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100)
|
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 300)
|
||||||
assert _thumb_is_valid(p) is True
|
assert _thumb_is_valid(p) is True
|
||||||
|
|
||||||
|
|
||||||
@@ -38,6 +41,16 @@ def test_thumb_is_valid_missing_file(tmp_path):
|
|||||||
assert _thumb_is_valid(tmp_path / "nope") is False
|
assert _thumb_is_valid(tmp_path / "nope") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_thumb_is_valid_header_only_below_min_size(tmp_path):
|
||||||
|
"""Operator-flagged 2026-06-01: header-only corrupt files were
|
||||||
|
silently passing the magic-byte check and backfill counted them as
|
||||||
|
`ok`, so the UI's broken-image tiles never got regenerated. Files
|
||||||
|
smaller than MIN_THUMB_BYTES are now invalid even with valid magic."""
|
||||||
|
p = tmp_path / "header_only.jpg"
|
||||||
|
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 50) # 54 bytes total
|
||||||
|
assert _thumb_is_valid(p) is False
|
||||||
|
|
||||||
|
|
||||||
# --- backfill_thumbnails planner tests ------------------------------------
|
# --- backfill_thumbnails planner tests ------------------------------------
|
||||||
|
|
||||||
|
|
||||||
@@ -80,13 +93,14 @@ def _rec(db_sync, path, *, sha, thumb_path=None, mime="image/jpeg"):
|
|||||||
|
|
||||||
def _write_jpeg(p: Path) -> Path:
|
def _write_jpeg(p: Path) -> Path:
|
||||||
p.parent.mkdir(parents=True, exist_ok=True)
|
p.parent.mkdir(parents=True, exist_ok=True)
|
||||||
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 100)
|
# ≥ MIN_THUMB_BYTES (256) so the size floor doesn't reject it.
|
||||||
|
p.write_bytes(b"\xff\xd8\xff\xe0" + b"\x00" * 300)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
def _write_png(p: Path) -> Path:
|
def _write_png(p: Path) -> Path:
|
||||||
p.parent.mkdir(parents=True, exist_ok=True)
|
p.parent.mkdir(parents=True, exist_ok=True)
|
||||||
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 100)
|
p.write_bytes(b"\x89PNG\r\n\x1a\n" + b"\x00" * 300)
|
||||||
return p
|
return p
|
||||||
|
|
||||||
|
|
||||||
@@ -111,7 +125,7 @@ def test_backfill_null_path_enqueued(db_sync, tmp_path, monkeypatch):
|
|||||||
)
|
)
|
||||||
|
|
||||||
result = m.backfill_thumbnails()
|
result = m.backfill_thumbnails()
|
||||||
assert result == {"enqueued": 1, "ok": 0, "regenerated": 0}
|
assert result == {"scanned": 1, "enqueued": 1, "ok": 0, "regenerated": 0}
|
||||||
assert delayed == [rec.id]
|
assert delayed == [rec.id]
|
||||||
|
|
||||||
|
|
||||||
@@ -134,7 +148,7 @@ def test_backfill_missing_file_clears_and_enqueues(db_sync, tmp_path, monkeypatc
|
|||||||
|
|
||||||
result = m.backfill_thumbnails()
|
result = m.backfill_thumbnails()
|
||||||
db_sync.expire_all()
|
db_sync.expire_all()
|
||||||
assert result == {"enqueued": 1, "ok": 0, "regenerated": 1}
|
assert result == {"scanned": 1, "enqueued": 1, "ok": 0, "regenerated": 1}
|
||||||
assert delayed == [rec.id]
|
assert delayed == [rec.id]
|
||||||
assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None
|
assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None
|
||||||
|
|
||||||
@@ -156,7 +170,7 @@ def test_backfill_valid_jpeg_skipped(db_sync, tmp_path, monkeypatch):
|
|||||||
|
|
||||||
result = m.backfill_thumbnails()
|
result = m.backfill_thumbnails()
|
||||||
db_sync.expire_all()
|
db_sync.expire_all()
|
||||||
assert result == {"enqueued": 0, "ok": 1, "regenerated": 0}
|
assert result == {"scanned": 1, "enqueued": 0, "ok": 1, "regenerated": 0}
|
||||||
assert delayed == []
|
assert delayed == []
|
||||||
assert db_sync.get(ImageRecord, rec.id).thumbnail_path == str(thumb)
|
assert db_sync.get(ImageRecord, rec.id).thumbnail_path == str(thumb)
|
||||||
|
|
||||||
@@ -177,7 +191,7 @@ def test_backfill_valid_png_skipped(db_sync, tmp_path, monkeypatch):
|
|||||||
)
|
)
|
||||||
|
|
||||||
result = m.backfill_thumbnails()
|
result = m.backfill_thumbnails()
|
||||||
assert result == {"enqueued": 0, "ok": 1, "regenerated": 0}
|
assert result == {"scanned": 1, "enqueued": 0, "ok": 1, "regenerated": 0}
|
||||||
assert delayed == []
|
assert delayed == []
|
||||||
|
|
||||||
|
|
||||||
@@ -198,7 +212,7 @@ def test_backfill_corrupt_magic_clears_and_enqueues(db_sync, tmp_path, monkeypat
|
|||||||
|
|
||||||
result = m.backfill_thumbnails()
|
result = m.backfill_thumbnails()
|
||||||
db_sync.expire_all()
|
db_sync.expire_all()
|
||||||
assert result == {"enqueued": 1, "ok": 0, "regenerated": 1}
|
assert result == {"scanned": 1, "enqueued": 1, "ok": 0, "regenerated": 1}
|
||||||
assert delayed == [rec.id]
|
assert delayed == [rec.id]
|
||||||
assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None
|
assert db_sync.get(ImageRecord, rec.id).thumbnail_path is None
|
||||||
|
|
||||||
@@ -238,5 +252,5 @@ def test_backfill_mixed_aggregate(db_sync, tmp_path, monkeypatch):
|
|||||||
)
|
)
|
||||||
|
|
||||||
result = m.backfill_thumbnails()
|
result = m.backfill_thumbnails()
|
||||||
assert result == {"enqueued": 3, "ok": 2, "regenerated": 2}
|
assert result == {"scanned": 5, "enqueued": 3, "ok": 2, "regenerated": 2}
|
||||||
assert sorted(delayed) == sorted([r_null.id, r_missing.id, r_bad.id])
|
assert sorted(delayed) == sorted([r_null.id, r_missing.id, r_bad.id])
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ pytestmark = pytest.mark.integration
|
|||||||
|
|
||||||
def test_load_or_create_writes_key_with_mode_0600(tmp_path):
|
def test_load_or_create_writes_key_with_mode_0600(tmp_path):
|
||||||
key_path = tmp_path / "secrets" / "credential_key.b64"
|
key_path = tmp_path / "secrets" / "credential_key.b64"
|
||||||
CredentialCrypto(key_path)
|
CredentialCrypto(key_path, bootstrap_ok=True)
|
||||||
assert key_path.exists()
|
assert key_path.exists()
|
||||||
mode = stat.S_IMODE(os.stat(key_path).st_mode)
|
mode = stat.S_IMODE(os.stat(key_path).st_mode)
|
||||||
assert mode == 0o600
|
assert mode == 0o600
|
||||||
@@ -25,9 +25,9 @@ def test_load_or_create_writes_key_with_mode_0600(tmp_path):
|
|||||||
|
|
||||||
def test_load_existing_key_is_idempotent(tmp_path):
|
def test_load_existing_key_is_idempotent(tmp_path):
|
||||||
key_path = tmp_path / "credential_key.b64"
|
key_path = tmp_path / "credential_key.b64"
|
||||||
crypto1 = CredentialCrypto(key_path)
|
crypto1 = CredentialCrypto(key_path, bootstrap_ok=True)
|
||||||
contents_after_first = key_path.read_bytes()
|
contents_after_first = key_path.read_bytes()
|
||||||
crypto2 = CredentialCrypto(key_path)
|
crypto2 = CredentialCrypto(key_path, bootstrap_ok=True)
|
||||||
contents_after_second = key_path.read_bytes()
|
contents_after_second = key_path.read_bytes()
|
||||||
assert contents_after_first == contents_after_second
|
assert contents_after_first == contents_after_second
|
||||||
# And both crypto instances decrypt each other's ciphertext
|
# And both crypto instances decrypt each other's ciphertext
|
||||||
@@ -36,7 +36,7 @@ def test_load_existing_key_is_idempotent(tmp_path):
|
|||||||
|
|
||||||
|
|
||||||
def test_encrypt_decrypt_round_trip(tmp_path):
|
def test_encrypt_decrypt_round_trip(tmp_path):
|
||||||
crypto = CredentialCrypto(tmp_path / "k")
|
crypto = CredentialCrypto(tmp_path / "k", bootstrap_ok=True)
|
||||||
plaintext = "domain.com\tTRUE\t/\tTRUE\t1700000000\tname\tvalue"
|
plaintext = "domain.com\tTRUE\t/\tTRUE\t1700000000\tname\tvalue"
|
||||||
ct = crypto.encrypt(plaintext)
|
ct = crypto.encrypt(plaintext)
|
||||||
assert isinstance(ct, bytes)
|
assert isinstance(ct, bytes)
|
||||||
@@ -45,8 +45,27 @@ def test_encrypt_decrypt_round_trip(tmp_path):
|
|||||||
|
|
||||||
|
|
||||||
def test_decrypt_with_wrong_key_raises(tmp_path):
|
def test_decrypt_with_wrong_key_raises(tmp_path):
|
||||||
crypto_a = CredentialCrypto(tmp_path / "a")
|
crypto_a = CredentialCrypto(tmp_path / "a", bootstrap_ok=True)
|
||||||
crypto_b = CredentialCrypto(tmp_path / "b")
|
crypto_b = CredentialCrypto(tmp_path / "b", bootstrap_ok=True)
|
||||||
ct = crypto_a.encrypt("secret")
|
ct = crypto_a.encrypt("secret")
|
||||||
with pytest.raises(InvalidCredentialBlob):
|
with pytest.raises(InvalidCredentialBlob):
|
||||||
crypto_b.decrypt(ct)
|
crypto_b.decrypt(ct)
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_key_without_bootstrap_raises(tmp_path, monkeypatch):
|
||||||
|
"""Audit 2026-06-02: without explicit opt-in, a missing key file
|
||||||
|
is a fatal startup error — silent regeneration on partial restore
|
||||||
|
would make every existing Credential row undecryptable."""
|
||||||
|
from backend.app.services.credential_crypto import MissingCredentialKey
|
||||||
|
monkeypatch.delenv("CURATOR_BOOTSTRAP_NEW_KEY", raising=False)
|
||||||
|
with pytest.raises(MissingCredentialKey):
|
||||||
|
CredentialCrypto(tmp_path / "absent.b64")
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_key_with_env_var_bootstraps(tmp_path, monkeypatch):
|
||||||
|
"""The env var CURATOR_BOOTSTRAP_NEW_KEY=1 is the operator's
|
||||||
|
first-time-setup opt-in for auto-creating the key file."""
|
||||||
|
monkeypatch.setenv("CURATOR_BOOTSTRAP_NEW_KEY", "1")
|
||||||
|
key_path = tmp_path / "bootstrap.b64"
|
||||||
|
CredentialCrypto(key_path) # no bootstrap_ok kwarg — relies on env
|
||||||
|
assert key_path.exists()
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ _NETSCAPE = (
|
|||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def crypto(tmp_path):
|
def crypto(tmp_path):
|
||||||
return CredentialCrypto(tmp_path / "k")
|
return CredentialCrypto(tmp_path / "k", bootstrap_ok=True)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
|
|||||||
@@ -130,7 +130,7 @@ async def test_download_source_attaches_written_files(
|
|||||||
thumbnailer=Thumbnailer(images_root=images_root),
|
thumbnailer=Thumbnailer(images_root=images_root),
|
||||||
settings=sync_settings,
|
settings=sync_settings,
|
||||||
)
|
)
|
||||||
crypto = CredentialCrypto(tmp_path / "key.b64")
|
crypto = CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True)
|
||||||
cred_service = CredentialService(db, crypto)
|
cred_service = CredentialService(db, crypto)
|
||||||
|
|
||||||
svc = DownloadService(
|
svc = DownloadService(
|
||||||
@@ -370,3 +370,246 @@ async def test_finalize_skipped_preserves_failures_clears_error(db):
|
|||||||
assert row.consecutive_failures == 4
|
assert row.consecutive_failures == 4
|
||||||
assert row.last_error is None
|
assert row.last_error is None
|
||||||
assert row.last_checked_at is not None
|
assert row.last_checked_at is not None
|
||||||
|
|
||||||
|
|
||||||
|
# --- Plan #544: backfill lifecycle + PARTIAL → status=ok -------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_decrements_after_run(
|
||||||
|
db, db_sync, tmp_path, seed_artist_and_source,
|
||||||
|
):
|
||||||
|
"""When backfill_runs_remaining > 0 going in, a non-clean / non-empty
|
||||||
|
run decrements by 1 — operator gets N runs to complete the deep scan
|
||||||
|
before tick mode resumes."""
|
||||||
|
from backend.app.services.download_service import DownloadService
|
||||||
|
from backend.app.services.importer import Importer
|
||||||
|
|
||||||
|
_artist, source = seed_artist_and_source
|
||||||
|
source.backfill_runs_remaining = 3
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
images_root = tmp_path / "images"
|
||||||
|
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||||
|
_make_jpg(f1, split="h")
|
||||||
|
|
||||||
|
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||||
|
success=True, written_paths=[str(f1)], files_downloaded=1,
|
||||||
|
stdout=f"{f1}\n",
|
||||||
|
))
|
||||||
|
|
||||||
|
sync_settings = db_sync.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
importer = Importer(
|
||||||
|
session=db_sync, images_root=images_root, import_root=images_root,
|
||||||
|
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||||
|
)
|
||||||
|
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||||
|
svc = DownloadService(
|
||||||
|
async_session=db, sync_session=db_sync,
|
||||||
|
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||||
|
)
|
||||||
|
await svc.download_source(source.id)
|
||||||
|
|
||||||
|
remaining = (await db.execute(
|
||||||
|
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert remaining == 2
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_backfill_auto_resets_on_clean_zero_files(
|
||||||
|
db, db_sync, tmp_path, seed_artist_and_source,
|
||||||
|
):
|
||||||
|
"""A clean run (rc=0) that downloaded zero files means the backfill
|
||||||
|
queue drained — reset to 0 immediately instead of wasting the rest of
|
||||||
|
the N-run budget on no-op walks."""
|
||||||
|
from backend.app.services.download_service import DownloadService
|
||||||
|
from backend.app.services.importer import Importer
|
||||||
|
|
||||||
|
_artist, source = seed_artist_and_source
|
||||||
|
source.backfill_runs_remaining = 3
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
fake_result = _make_fake_dl_result(
|
||||||
|
success=True, written_paths=[], files_downloaded=0,
|
||||||
|
)
|
||||||
|
fake_gdl = _fake_gdl_with_result(fake_result)
|
||||||
|
|
||||||
|
sync_settings = db_sync.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
importer = Importer(
|
||||||
|
session=db_sync, images_root=tmp_path / "images",
|
||||||
|
import_root=tmp_path / "images",
|
||||||
|
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||||
|
settings=sync_settings,
|
||||||
|
)
|
||||||
|
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||||
|
svc = DownloadService(
|
||||||
|
async_session=db, sync_session=db_sync,
|
||||||
|
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||||
|
)
|
||||||
|
await svc.download_source(source.id)
|
||||||
|
|
||||||
|
remaining = (await db.execute(
|
||||||
|
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert remaining == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_tick_mode_does_not_touch_backfill_counter(
|
||||||
|
db, db_sync, tmp_path, seed_artist_and_source,
|
||||||
|
):
|
||||||
|
"""When backfill_runs_remaining is already 0, downloads don't go
|
||||||
|
negative or otherwise mutate the counter."""
|
||||||
|
from backend.app.services.download_service import DownloadService
|
||||||
|
from backend.app.services.importer import Importer
|
||||||
|
|
||||||
|
_artist, source = seed_artist_and_source
|
||||||
|
assert source.backfill_runs_remaining == 0
|
||||||
|
|
||||||
|
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||||
|
success=True, written_paths=[], files_downloaded=0,
|
||||||
|
))
|
||||||
|
|
||||||
|
sync_settings = db_sync.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
importer = Importer(
|
||||||
|
session=db_sync, images_root=tmp_path / "images",
|
||||||
|
import_root=tmp_path / "images",
|
||||||
|
thumbnailer=Thumbnailer(images_root=tmp_path / "images"),
|
||||||
|
settings=sync_settings,
|
||||||
|
)
|
||||||
|
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||||
|
svc = DownloadService(
|
||||||
|
async_session=db, sync_session=db_sync,
|
||||||
|
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||||
|
)
|
||||||
|
await svc.download_source(source.id)
|
||||||
|
|
||||||
|
remaining = (await db.execute(
|
||||||
|
select(Source.backfill_runs_remaining).where(Source.id == source.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert remaining == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_partial_error_type_maps_to_ok_status(
|
||||||
|
db, db_sync, tmp_path, seed_artist_and_source,
|
||||||
|
):
|
||||||
|
"""A non-zero gallery-dl exit accompanied by PARTIAL error_type
|
||||||
|
(real files were downloaded before the run was cut short) is treated
|
||||||
|
as status=ok by the orchestrator — next tick continues."""
|
||||||
|
from backend.app.services.download_service import DownloadService
|
||||||
|
from backend.app.services.gallery_dl import ErrorType
|
||||||
|
from backend.app.services.importer import Importer
|
||||||
|
|
||||||
|
_artist, source = seed_artist_and_source
|
||||||
|
|
||||||
|
images_root = tmp_path / "images"
|
||||||
|
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||||
|
_make_jpg(f1, split="h")
|
||||||
|
|
||||||
|
fake_result = _make_fake_dl_result(
|
||||||
|
success=False, written_paths=[str(f1)], files_downloaded=1,
|
||||||
|
error_type=ErrorType.PARTIAL,
|
||||||
|
error_message="Downloaded 1 file; run did not complete in budget",
|
||||||
|
stdout=f"{f1}\n",
|
||||||
|
)
|
||||||
|
fake_gdl = _fake_gdl_with_result(fake_result)
|
||||||
|
|
||||||
|
sync_settings = db_sync.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
importer = Importer(
|
||||||
|
session=db_sync, images_root=images_root, import_root=images_root,
|
||||||
|
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||||
|
)
|
||||||
|
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||||
|
svc = DownloadService(
|
||||||
|
async_session=db, sync_session=db_sync,
|
||||||
|
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||||
|
)
|
||||||
|
event_id = await svc.download_source(source.id)
|
||||||
|
|
||||||
|
ev = (await db.execute(
|
||||||
|
select(DownloadEvent).where(DownloadEvent.id == event_id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert ev.status == "ok"
|
||||||
|
assert ev.error is None
|
||||||
|
# And the source's failure counter isn't bumped — PARTIAL isn't a failure.
|
||||||
|
src_after = (await db.execute(
|
||||||
|
select(Source.consecutive_failures).where(Source.id == source.id)
|
||||||
|
)).scalar_one()
|
||||||
|
assert src_after == 0
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_download_enqueues_thumbnail_and_ml_per_attached_image(
|
||||||
|
db, db_sync, tmp_path, seed_artist_and_source, monkeypatch,
|
||||||
|
):
|
||||||
|
"""Operator-flagged 2026-06-01: downloaded images stayed at
|
||||||
|
thumbnail_path=NULL until periodic backfill swept them up, surfacing
|
||||||
|
as broken-thumbnail tiles in the gallery. Importer.attach_in_place
|
||||||
|
deliberately skips inline generation; the calling code MUST enqueue
|
||||||
|
the thumbnail + ML tasks per attached image (matching the pattern in
|
||||||
|
tasks/import_file.py)."""
|
||||||
|
from backend.app.services.download_service import DownloadService
|
||||||
|
from backend.app.services.importer import Importer
|
||||||
|
|
||||||
|
_artist, source = seed_artist_and_source
|
||||||
|
|
||||||
|
images_root = tmp_path / "images"
|
||||||
|
f1 = images_root / "alice" / "patreon" / "post" / "a.jpg"
|
||||||
|
f2 = images_root / "alice" / "patreon" / "post" / "b.jpg"
|
||||||
|
_make_jpg(f1, split="h")
|
||||||
|
_make_jpg(f2, split="v")
|
||||||
|
|
||||||
|
fake_gdl = _fake_gdl_with_result(_make_fake_dl_result(
|
||||||
|
success=True, written_paths=[str(f1), str(f2)],
|
||||||
|
files_downloaded=2, stdout=f"{f1}\n{f2}\n",
|
||||||
|
))
|
||||||
|
|
||||||
|
sync_settings = db_sync.execute(
|
||||||
|
select(ImportSettings).where(ImportSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
sync_settings.phash_threshold = 0
|
||||||
|
importer = Importer(
|
||||||
|
session=db_sync, images_root=images_root, import_root=images_root,
|
||||||
|
thumbnailer=Thumbnailer(images_root=images_root), settings=sync_settings,
|
||||||
|
)
|
||||||
|
cred_service = CredentialService(db, CredentialCrypto(tmp_path / "key.b64", bootstrap_ok=True))
|
||||||
|
|
||||||
|
# Capture the IDs that the orchestrator hands off to each Celery task.
|
||||||
|
# The .delay() shim runs inside DownloadService._phase3_persist (lazy
|
||||||
|
# imports under ..tasks.thumbnail / ..tasks.ml), so monkeypatch the
|
||||||
|
# symbols on those modules.
|
||||||
|
thumb_calls: list[int] = []
|
||||||
|
ml_calls: list[int] = []
|
||||||
|
from backend.app.tasks import ml as ml_mod
|
||||||
|
from backend.app.tasks import thumbnail as thumb_mod
|
||||||
|
monkeypatch.setattr(
|
||||||
|
thumb_mod.generate_thumbnail, "delay",
|
||||||
|
lambda image_id: thumb_calls.append(image_id),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
ml_mod.tag_and_embed, "delay",
|
||||||
|
lambda image_id: ml_calls.append(image_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
svc = DownloadService(
|
||||||
|
async_session=db, sync_session=db_sync,
|
||||||
|
gdl=fake_gdl, importer=importer, cred_service=cred_service,
|
||||||
|
)
|
||||||
|
await svc.download_source(source.id)
|
||||||
|
|
||||||
|
# Two files attached → two thumbnail enqueues + two ML enqueues, IDs
|
||||||
|
# match the actually-imported records (not ad-hoc — drawn from the
|
||||||
|
# importer's returned image_id so future supersede paths stay covered).
|
||||||
|
assert len(thumb_calls) == 2
|
||||||
|
assert len(ml_calls) == 2
|
||||||
|
assert sorted(thumb_calls) == sorted(ml_calls)
|
||||||
|
|||||||
@@ -249,3 +249,80 @@ def test_truncate_log_caps_large_text(gdl):
|
|||||||
def test_truncate_log_passes_short_text(gdl):
|
def test_truncate_log_passes_short_text(gdl):
|
||||||
text = "small\n"
|
text = "small\n"
|
||||||
assert gdl._truncate_log(text) == text
|
assert gdl._truncate_log(text) == text
|
||||||
|
|
||||||
|
|
||||||
|
# --- Plan #544: tick/backfill skip emission + PARTIAL classifier ------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_emits_tick_skip_value(gdl):
|
||||||
|
"""Tick mode emits gallery-dl's exit:20 to short-circuit catch-up
|
||||||
|
scans once 20 contiguous archived items are seen."""
|
||||||
|
from backend.app.services.gallery_dl import TICK_SKIP_VALUE
|
||||||
|
cfg = gdl._build_config_for_source(
|
||||||
|
platform="patreon",
|
||||||
|
source_config=SourceConfig(),
|
||||||
|
artist_slug="alice",
|
||||||
|
skip_value=TICK_SKIP_VALUE,
|
||||||
|
)
|
||||||
|
assert cfg["extractor"]["skip"] == "exit:20"
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_config_emits_backfill_skip_value(gdl):
|
||||||
|
"""Backfill mode keeps gallery-dl's default skip=True so it walks the
|
||||||
|
full post history."""
|
||||||
|
from backend.app.services.gallery_dl import BACKFILL_SKIP_VALUE
|
||||||
|
cfg = gdl._build_config_for_source(
|
||||||
|
platform="patreon",
|
||||||
|
source_config=SourceConfig(),
|
||||||
|
artist_slug="alice",
|
||||||
|
skip_value=BACKFILL_SKIP_VALUE,
|
||||||
|
)
|
||||||
|
assert cfg["extractor"]["skip"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_categorize_partial_when_files_downloaded_then_nonzero_exit(gdl):
|
||||||
|
"""Non-zero exit + ≥1 file downloaded + no source-level error →
|
||||||
|
PARTIAL. Downstream maps to status=ok because the next tick continues."""
|
||||||
|
# stdout has a single downloaded file (line starting with `/`); no
|
||||||
|
# error-class indicators in stderr; non-zero return code.
|
||||||
|
etype, msg = gdl._categorize_error(
|
||||||
|
return_code=1,
|
||||||
|
stdout="/images/alice/patreon/post1/img.jpg\n",
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
assert etype == ErrorType.PARTIAL
|
||||||
|
assert "Downloaded 1 file" in msg
|
||||||
|
|
||||||
|
|
||||||
|
def test_categorize_unknown_when_no_files_and_no_pattern(gdl):
|
||||||
|
"""PARTIAL only kicks in when files were actually written. A non-zero
|
||||||
|
exit with no downloads and no recognized pattern still falls through
|
||||||
|
to UNKNOWN_ERROR — distinct from the partial-success case."""
|
||||||
|
etype, _ = gdl._categorize_error(
|
||||||
|
return_code=2,
|
||||||
|
stdout="",
|
||||||
|
stderr="some unrecognized noise\n",
|
||||||
|
)
|
||||||
|
assert etype == ErrorType.UNKNOWN_ERROR
|
||||||
|
|
||||||
|
|
||||||
|
def test_categorize_tier_limited_wins_over_partial(gdl):
|
||||||
|
"""A run that hit tier-limited warnings AND downloaded some files
|
||||||
|
classifies as TIER_LIMITED (the more specific category), not PARTIAL."""
|
||||||
|
stdout = "/images/alice/patreon/post1/img.jpg\n"
|
||||||
|
stderr = "[patreon][warning] Not allowed to view post 123\n"
|
||||||
|
etype, _ = gdl._categorize_error(
|
||||||
|
return_code=1, stdout=stdout, stderr=stderr,
|
||||||
|
)
|
||||||
|
assert etype == ErrorType.TIER_LIMITED
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_config_forwards_patreon_referer_to_ytdl(gdl):
|
||||||
|
"""Operator-flagged 2026-06-01: Mux video playback restrictions reject
|
||||||
|
yt-dlp's manifest fetch when Referer/Origin don't match Patreon. The
|
||||||
|
fix is a static `downloader.ytdl.raw-options.http_headers` block, so
|
||||||
|
it's enough to assert the keys land in the default config."""
|
||||||
|
cfg = gdl._get_default_config()
|
||||||
|
headers = cfg["downloader"]["ytdl"]["raw-options"]["http_headers"]
|
||||||
|
assert headers["Referer"] == "https://www.patreon.com/"
|
||||||
|
assert headers["Origin"] == "https://www.patreon.com"
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ async def _post(db, artist_name, slug, ext):
|
|||||||
url=f"https://patreon.test/{slug}")
|
url=f"https://patreon.test/{slug}")
|
||||||
db.add(s)
|
db.add(s)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
p = Post(source_id=s.id, external_post_id=ext)
|
p = Post(source_id=s.id, artist_id=a.id, external_post_id=ext)
|
||||||
db.add(p)
|
db.add(p)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
return a, s, p
|
return a, s, p
|
||||||
|
|||||||
@@ -152,7 +152,8 @@ async def _seed_image_with_post(
|
|||||||
db.add(source)
|
db.add(source)
|
||||||
await db.flush()
|
await db.flush()
|
||||||
post = Post(
|
post = Post(
|
||||||
source_id=source.id, external_post_id=external_post_id,
|
source_id=source.id, artist_id=artist.id,
|
||||||
|
external_post_id=external_post_id,
|
||||||
post_title="A Post", post_date=post_date,
|
post_title="A Post", post_date=post_date,
|
||||||
)
|
)
|
||||||
db.add(post)
|
db.add(post)
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user