Compare commits
161 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9b5ec86222 | |||
| d80a5255ed | |||
| 89c83ee5de | |||
| 69b5637bd6 | |||
| d3192f1843 | |||
| 51749e05db | |||
| 5a5694f200 | |||
| 50d6c42207 | |||
| bb1a938cc0 | |||
| 67c7ca8603 | |||
| fc0293029d | |||
| eed42a260a | |||
| 61b14e8f65 | |||
| 7d1c701b67 | |||
| 447bf73519 | |||
| b638382cd5 | |||
| 6104452d2e | |||
| b59828635e | |||
| 17903068b4 | |||
| fac5ae6ce5 | |||
| af0d39ed52 | |||
| d9a14e890d | |||
| ad2a5fc5fe | |||
| 31d400ab0a | |||
| 0da0e47784 | |||
| 503c8854bc | |||
| 571938781a | |||
| d45ce5e426 | |||
| 0cf3a02797 | |||
| 6ab495292a | |||
| c98db303d0 | |||
| 2d0fca8729 | |||
| a2858892e9 | |||
| 7f5e0603de | |||
| 49f6765326 | |||
| b23b19bf58 | |||
| c17a6e6c40 | |||
| 64053a5f57 | |||
| b6c5638eab | |||
| 466ee898ab | |||
| e00deee451 | |||
| 87d3198f89 | |||
| 5010de6178 | |||
| 3063af6a74 | |||
| ba2de60439 | |||
| ff9846ce64 | |||
| 279b0560d9 | |||
| af09bf58af | |||
| 77bfc32b02 | |||
| aea2701c28 | |||
| d13efe1b69 | |||
| a444cf82d1 | |||
| ca6d26d236 | |||
| 6ab7fd5c7f | |||
| 7ddf94220d | |||
| 5366047d55 | |||
| ffdbdbaf07 | |||
| 2923257529 | |||
| a017771621 | |||
| 25e555cab6 | |||
| 934731e9ef | |||
| 3c7ab44e74 | |||
| 4c67c116b2 | |||
| 06f98acf3e | |||
| f8b667604f | |||
| 4371ddb7e7 | |||
| 40cc11be5b | |||
| 0b78264d62 | |||
| 9eae636047 | |||
| f8105046dc | |||
| d631ed023c | |||
| c64261593d | |||
| 11572f469a | |||
| 1f6d94f51d | |||
| ba3fd2a118 | |||
| 6a255482ea | |||
| af5aa21e45 | |||
| 83c1745fd0 | |||
| ead60978e3 | |||
| 7a4de7278d | |||
| 7f8073c4c8 | |||
| a3bc98a53c | |||
| 9bf095179a | |||
| e7c3f4e9c9 | |||
| cefc064e0f | |||
| 29f3a485b0 | |||
| 06757daf80 | |||
| 2bcaa20b22 | |||
| 9726d6f4b5 | |||
| 6c34f86477 | |||
| 1f548d8a7b | |||
| eedf8d109a | |||
| ab63d94249 | |||
| eadaa716af | |||
| 0efb187eb1 | |||
| e86b91dfe2 | |||
| 4f4ddecf75 | |||
| 04d5d62cfe | |||
| a723ef436b | |||
| 18bb25f140 | |||
| 17433c69d4 | |||
| 3bf41ecac3 | |||
| bae077e323 | |||
| 775941609d | |||
| d3984ccb0d | |||
| 7d3a3b4a83 | |||
| 6684907577 | |||
| 2d44a26bdf | |||
| 0de726ed48 | |||
| 3006e84cc0 | |||
| cbc3e11a53 | |||
| d68f39ca50 | |||
| 2cfbb284d5 | |||
| a94f6a2789 | |||
| a1ed53136e | |||
| 9504870c9a | |||
| f24dc81764 | |||
| 48be921b8d | |||
| 7939dba9ed | |||
| 8f6547f8d7 | |||
| a66a695977 | |||
| 2638cf1a35 | |||
| 9bb4211722 | |||
| 524a26c618 | |||
| dfe2fda564 | |||
| 409724b981 | |||
| 5289fa3879 | |||
| ab362bc79c | |||
| a4df279343 | |||
| 62ec70b9e4 | |||
| ebac34e17b | |||
| 0963bf0db3 | |||
| 0c4b8aef8c | |||
| a69bd1baa8 | |||
| 87d53db0cb | |||
| 4c3ba20198 | |||
| 8838b325fb | |||
| 437bf4d37a | |||
| f33808b977 | |||
| d2acec61ae | |||
| 6c6e8bdb6d | |||
| c9696a2faf | |||
| 544e30bfb9 | |||
| a7f715ec43 | |||
| 0bbcdee3bd | |||
| 4068a97392 | |||
| 0563b2d750 | |||
| 7ef2ecd82f | |||
| 86ae396914 | |||
| 15ae5ef4fa | |||
| 65bd1c22c3 | |||
| f77e75147d | |||
| 723f023e6a | |||
| 19744fa41d | |||
| e6f128c894 | |||
| e9891ee9f3 | |||
| 581b778528 | |||
| 6d314d662f | |||
| b54243a1ff | |||
| e3ceefc820 | |||
| aa12a57f97 |
@@ -9,10 +9,15 @@ name: CI
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
# pull_request trigger intentionally absent — with branches: [dev, main]
|
||||
# above, every PR commit already fires CI via the push event on dev. Adding
|
||||
# pull_request would duplicate runs on dev→main PRs. FC has no fork PRs
|
||||
# (single-operator Forgejo repo) so push coverage is complete.
|
||||
# Renovate opens PRs from `renovate/*` branches into `dev`. Those branches
|
||||
# never push to dev/main, so the push trigger above gives them NO pre-merge
|
||||
# CI — a bump could only be validated after it was already merged. This
|
||||
# pull_request trigger (base `dev` only) validates Renovate PRs before merge.
|
||||
# It deliberately does NOT fire on dev→main PRs (base `main`), which still
|
||||
# rely on the dev push run — so no duplicate runs. FC has no fork PRs
|
||||
# (single-operator Forgejo repo), so secrets-on-PR is not a concern.
|
||||
pull_request:
|
||||
branches: [dev]
|
||||
|
||||
jobs:
|
||||
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
|
||||
@@ -92,10 +97,10 @@ jobs:
|
||||
# If we want strict lockfile-based reproducibility later, commit a
|
||||
# package-lock.json and flip this back to `npm ci`.
|
||||
- run: npm install --no-audit --no-fund
|
||||
# `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS
|
||||
# with no .ts files and no JSDoc annotations, so vue-tsc has nothing
|
||||
# to type-check. Re-enable once we add a tsconfig.json and either
|
||||
# convert to TS or add JSDoc.
|
||||
# No type-check step: the frontend is pure JS (no .ts files, no JSDoc),
|
||||
# so a type-checker has nothing to do. The vue-tsc devDep + its `check`
|
||||
# script were dropped 2026-07-11 rather than bumped to v3. If we add
|
||||
# TS/JSDoc later, re-add a tsconfig.json + vue-tsc + a type-check step.
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
lint:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:22-bookworm-slim
|
||||
image: node:24-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install web-ext
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# syntax=docker/dockerfile:1.25
|
||||
|
||||
FROM node:22-alpine AS frontend-builder
|
||||
FROM node:24-alpine AS frontend-builder
|
||||
WORKDIR /build
|
||||
COPY frontend/package.json frontend/package-lock.json* ./
|
||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# syntax=docker/dockerfile:1.25
|
||||
|
||||
FROM python:3.14-slim
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
|
||||
@@ -265,6 +265,8 @@ class Worker:
|
||||
self._embedder = None
|
||||
self._embedder_lock = threading.Lock()
|
||||
self._proposers = None
|
||||
self._proposers_sig = None # detector-config signature the current
|
||||
# proposers were built for (#134)
|
||||
self._proposers_lock = threading.Lock()
|
||||
|
||||
# --- held-lease bookkeeping --------------------------------------------
|
||||
@@ -796,14 +798,52 @@ class Worker:
|
||||
self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype)
|
||||
return self._embedder
|
||||
|
||||
def _ensure_proposers(self):
|
||||
if self._proposers is not None:
|
||||
return self._proposers
|
||||
def _effective_cfg(self, det: dict | None):
|
||||
"""The detector config for a job: the LEASE-ANNOUNCED block (the DB/UI
|
||||
source of truth, #134) overlaid on the env cfg, or the env cfg unchanged
|
||||
when the server announced nothing (bootstrap / older backend). An enabled
|
||||
toggle off maps to empty weights — the 'weights == "" → proposer skipped'
|
||||
contract in detectors.py. detector_level + ccip_model stay env (separate
|
||||
settings, not part of the crop-proposer config)."""
|
||||
if not det:
|
||||
return self.cfg
|
||||
|
||||
def _w(p: str) -> str:
|
||||
slot = det.get(p) or {}
|
||||
return (slot.get("weights") or "") if slot.get("enabled", True) else ""
|
||||
|
||||
from dataclasses import replace
|
||||
return replace(
|
||||
self.cfg,
|
||||
person_weights=_w("person"),
|
||||
person_conf=float(det["person"]["conf"]),
|
||||
anatomy_weights=_w("anatomy"),
|
||||
anatomy_conf=float(det["anatomy"]["conf"]),
|
||||
panel_weights=_w("panel"),
|
||||
panel_conf=float(det["panel"]["conf"]),
|
||||
max_figures=int(det["max_figures"]),
|
||||
max_components=int(det["max_components"]),
|
||||
max_panels=int(det["max_panels"]),
|
||||
max_regions=int(det["max_regions"]),
|
||||
dedupe_iou=float(det["dedupe_iou"]),
|
||||
)
|
||||
|
||||
def _proposers_for(self, eff):
|
||||
"""Proposers built from the effective detector config, rebuilt ONLY when
|
||||
that config changes — so an operator's UI edit takes effect on the next
|
||||
lease with no restart, and steady-state reuses the loaded YOLO models."""
|
||||
sig = (
|
||||
eff.person_weights, eff.person_conf,
|
||||
eff.anatomy_weights, eff.anatomy_conf,
|
||||
eff.panel_weights, eff.panel_conf,
|
||||
eff.max_figures, eff.max_components, eff.max_panels,
|
||||
)
|
||||
with self._proposers_lock:
|
||||
if self._proposers is None:
|
||||
if self._proposers is None or self._proposers_sig != sig:
|
||||
from .detectors import Proposers
|
||||
self._proposers = Proposers(self.cfg)
|
||||
return self._proposers
|
||||
self._proposers = Proposers(eff)
|
||||
self._proposers_sig = sig
|
||||
return self._proposers
|
||||
|
||||
def _consume(self, job: dict, frames: list, stop_evt: threading.Event) -> bool:
|
||||
"""Detect + embed the decoded frames and submit the result. Returns True
|
||||
@@ -856,7 +896,8 @@ class Worker:
|
||||
else ["figure", "face", "concept", "panel"]
|
||||
)
|
||||
embedder = self._ensure_embedder(model_name) if want_siglip else None
|
||||
proposers = self._ensure_proposers()
|
||||
eff = self._effective_cfg(job.get("detectors"))
|
||||
proposers = self._proposers_for(eff)
|
||||
|
||||
regions = []
|
||||
ccip_ev = self.cfg.ccip_model or "ccip-default"
|
||||
@@ -922,7 +963,7 @@ class Worker:
|
||||
# we never SigLIP the same region twice — saves GPU and a
|
||||
# slot against max_regions. High-IoU + kind-aware, so
|
||||
# intentional nested crops (figure ⊃ head) survive.
|
||||
pending = dedupe_crops(pending, self.cfg.dedupe_iou)
|
||||
pending = dedupe_crops(pending, eff.dedupe_iou)
|
||||
vecs = embedder.embed_batch([c for c, _ in pending])
|
||||
for (_c, tmpl), vec in zip(pending, vecs, strict=True):
|
||||
tmpl["siglip_embedding"] = vec
|
||||
@@ -932,7 +973,7 @@ class Worker:
|
||||
# mp4, 64 sampled frames × ~32 regions) would otherwise burn
|
||||
# ~38s of GPU across every frame before the submit is even
|
||||
# truncated. Bounds the WORK, not just the POST body.
|
||||
if len(regions) >= self.cfg.max_regions:
|
||||
if len(regions) >= eff.max_regions:
|
||||
break
|
||||
|
||||
# A Stop mid-frame-loop leaves partial regions — don't submit those;
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
"""tag.is_system + seed the three hygiene system tags
|
||||
|
||||
Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a
|
||||
character poison that character's head and CCIP references; banners/editor
|
||||
screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the
|
||||
product ships — not operator configuration — so the seed lives here.
|
||||
|
||||
Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive,
|
||||
matching TagService.rename's collision stance) instead of inserting a
|
||||
duplicate, so an operator who already tagged `wip` keeps their applications.
|
||||
|
||||
Revision ID: 0075
|
||||
Revises: 0074
|
||||
Create Date: 2026-07-03
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0075"
|
||||
down_revision: Union[str, None] = "0074"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"tag",
|
||||
sa.Column(
|
||||
"is_system", sa.Boolean(), nullable=False,
|
||||
server_default=sa.false(),
|
||||
),
|
||||
)
|
||||
conn = op.get_bind()
|
||||
for name in SYSTEM_TAG_NAMES:
|
||||
adopted = conn.execute(
|
||||
sa.text(
|
||||
"UPDATE tag SET is_system = true "
|
||||
"WHERE lower(name) = lower(:name) AND kind = 'general'"
|
||||
),
|
||||
{"name": name},
|
||||
)
|
||||
if adopted.rowcount == 0:
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"INSERT INTO tag (name, kind, is_system) "
|
||||
"VALUES (:name, 'general', true)"
|
||||
),
|
||||
{"name": name},
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The seeded rows survive as ordinary general tags — dropping the flag is
|
||||
# enough to disarm the mechanism, and deleting rows would orphan any
|
||||
# operator applications made while the flag existed.
|
||||
op.drop_column("tag", "is_system")
|
||||
@@ -0,0 +1,82 @@
|
||||
"""pixiv_seen_media + pixiv_failed_media: per-source ledgers
|
||||
|
||||
Revision ID: 0076
|
||||
Revises: 0075
|
||||
Create Date: 2026-07-03
|
||||
|
||||
Pixiv native ingester (milestone #129, gallery-dl → native-core migration).
|
||||
Mirrors the Patreon (0037/0038) and SubscribeStar (0054) ledger tables: a
|
||||
seen-ledger so routine walks skip already-ingested media (recovery bypasses
|
||||
it) and a dead-letter ledger so persistently-failing media stops re-burning
|
||||
backfill chunks. Pixiv URLs carry no content hash, so `filehash` is always the
|
||||
synthesized ``<illust_id>:p<num>`` / ``<illust_id>:ugoira`` key — String(128)
|
||||
matches the siblings. UNIQUE (source_id, filehash) is the upsert key on each.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0076"
|
||||
down_revision: Union[str, None] = "0075"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"pixiv_seen_media",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"source_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("source.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("filehash", sa.String(128), nullable=False),
|
||||
sa.Column("post_id", sa.String(64), nullable=True),
|
||||
sa.Column(
|
||||
"seen_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("NOW()"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_seen_media_source_id"
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"pixiv_failed_media",
|
||||
sa.Column("id", sa.Integer, primary_key=True),
|
||||
sa.Column(
|
||||
"source_id",
|
||||
sa.Integer,
|
||||
sa.ForeignKey("source.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
),
|
||||
sa.Column("filehash", sa.String(128), nullable=False),
|
||||
sa.Column("attempts", sa.Integer, nullable=False, server_default="1"),
|
||||
sa.Column("last_error", sa.Text, nullable=True),
|
||||
sa.Column(
|
||||
"first_failed_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("NOW()"),
|
||||
),
|
||||
sa.Column(
|
||||
"last_failed_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.text("NOW()"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_failed_media_source_id"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("pixiv_failed_media")
|
||||
op.drop_table("pixiv_seen_media")
|
||||
@@ -0,0 +1,32 @@
|
||||
"""drop uq_artist_name — decouple display name from identity/storage
|
||||
|
||||
Revision ID: 0077
|
||||
Revises: 0076
|
||||
Create Date: 2026-07-04
|
||||
|
||||
Artist model fragility fix (milestone #130). One `slug` column was doing
|
||||
identity + storage-path + display, and BOTH `name` and `slug` were UNIQUE, so
|
||||
the display name couldn't be edited freely and two genuinely different creators
|
||||
collided. Decouple: `slug` stays the immutable, unique storage/identity key (the
|
||||
on-disk path component — untouched here); `name` becomes freely editable, NON-
|
||||
unique display text. This migration only drops the `uq_artist_name` constraint;
|
||||
no data moves and no path changes.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0077"
|
||||
down_revision: Union[str, None] = "0076"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_constraint("uq_artist_name", "artist", type_="unique")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Re-adding the UNIQUE would fail if duplicate names now exist; callers that
|
||||
# need to reverse this must dedupe names first.
|
||||
op.create_unique_constraint("uq_artist_name", "artist", ["name"])
|
||||
@@ -0,0 +1,83 @@
|
||||
"""ml_settings crop-proposer / detector config (#134)
|
||||
|
||||
Move the WHERE-to-crop detector config (per-proposer enable + weights + conf,
|
||||
plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU
|
||||
agent in the lease (like the embedder model) — no restart, agent env is now
|
||||
bootstrap-only. All server_defaults are the working values so existing rows +
|
||||
fresh installs crop out-of-the-box with all three proposers ON.
|
||||
|
||||
Revision ID: 0078
|
||||
Revises: 0077
|
||||
Create Date: 2026-07-05
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0078"
|
||||
down_revision: Union[str, None] = "0077"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
_ANATOMY_DEFAULT = (
|
||||
"https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt"
|
||||
)
|
||||
_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt"
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_person_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.true()))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_person_weights", sa.String(512), nullable=False,
|
||||
server_default="yolo11n.pt"))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_person_conf", sa.Float(), nullable=False,
|
||||
server_default=sa.text("0.35")))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_anatomy_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.true()))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_anatomy_weights", sa.String(512), nullable=False,
|
||||
server_default=_ANATOMY_DEFAULT))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_anatomy_conf", sa.Float(), nullable=False,
|
||||
server_default=sa.text("0.30")))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_panel_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.true()))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_panel_weights", sa.String(512), nullable=False,
|
||||
server_default=_PANEL_DEFAULT))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_panel_conf", sa.Float(), nullable=False,
|
||||
server_default=sa.text("0.30")))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_max_figures", sa.Integer(), nullable=False,
|
||||
server_default=sa.text("8")))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_max_components", sa.Integer(), nullable=False,
|
||||
server_default=sa.text("8")))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_max_panels", sa.Integer(), nullable=False,
|
||||
server_default=sa.text("8")))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_max_regions", sa.Integer(), nullable=False,
|
||||
server_default=sa.text("128")))
|
||||
op.add_column("ml_settings", sa.Column(
|
||||
"detector_dedupe_iou", sa.Float(), nullable=False,
|
||||
server_default=sa.text("0.85")))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for col in (
|
||||
"detector_person_enabled", "detector_person_weights", "detector_person_conf",
|
||||
"detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf",
|
||||
"detector_panel_enabled", "detector_panel_weights", "detector_panel_conf",
|
||||
"detector_max_figures", "detector_max_components", "detector_max_panels",
|
||||
"detector_max_regions", "detector_dedupe_iou",
|
||||
):
|
||||
op.drop_column("ml_settings", col)
|
||||
@@ -0,0 +1,77 @@
|
||||
"""character prototype store (#1317) — precomputed, incremental CCIP references
|
||||
|
||||
New tables character_prototype + ccip_prototype_state, plus MLSettings columns
|
||||
ccip_ref_signature (cheap global change gate) + ccip_prototype_cap (per-character
|
||||
reference cap). The reference set the CCIP matcher uses becomes a precomputed
|
||||
artifact refreshed incrementally off the request path. See milestone 138 /
|
||||
backend.app.services.ml.character_prototypes.
|
||||
|
||||
Revision ID: 0079
|
||||
Revises: 0078
|
||||
Create Date: 2026-07-06
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
revision: str = "0079"
|
||||
down_revision: Union[str, None] = "0078"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# Matches models.image_region.CCIP_DIM (the CCIP figure-embedding width).
|
||||
_CCIP_DIM = 768
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"character_prototype",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"tag_id", sa.Integer(),
|
||||
sa.ForeignKey("tag.id", ondelete="CASCADE"), nullable=False,
|
||||
),
|
||||
sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=False),
|
||||
sa.Column(
|
||||
"region_id", sa.Integer(),
|
||||
sa.ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_character_prototype_tag_id", "character_prototype", ["tag_id"]
|
||||
)
|
||||
op.create_table(
|
||||
"ccip_prototype_state",
|
||||
sa.Column(
|
||||
"tag_id", sa.Integer(),
|
||||
sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
|
||||
),
|
||||
sa.Column("fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column("ccip_ref_signature", sa.String(128), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"ccip_prototype_cap", sa.Integer(), nullable=False,
|
||||
server_default=sa.text("64"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("ml_settings", "ccip_prototype_cap")
|
||||
op.drop_column("ml_settings", "ccip_ref_signature")
|
||||
op.drop_table("ccip_prototype_state")
|
||||
op.drop_index(
|
||||
"ix_character_prototype_tag_id", table_name="character_prototype"
|
||||
)
|
||||
op.drop_table("character_prototype")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""tag_head.train_fingerprint (#1317 phase 2) — incremental head retraining
|
||||
|
||||
A per-head training-data fingerprint (positive + rejection count/latest-timestamp)
|
||||
so a manual Retrain refits only the tags whose data changed; the nightly run
|
||||
ignores it (full reconcile). Nullable — a NULL fingerprint (existing heads) forces
|
||||
a refit on the first incremental run, then it's stamped.
|
||||
|
||||
Revision ID: 0080
|
||||
Revises: 0079
|
||||
Create Date: 2026-07-06
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0080"
|
||||
down_revision: Union[str, None] = "0079"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"tag_head",
|
||||
sa.Column("train_fingerprint", sa.String(128), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("tag_head", "train_fingerprint")
|
||||
@@ -0,0 +1,43 @@
|
||||
"""stricter auto-apply defaults (milestone 139) — cut auto-apply misfires
|
||||
|
||||
head_auto_apply_min_positives 30→50 and ccip_auto_apply_threshold 0.92→0.95
|
||||
(operator-asked 2026-07-06). The head graduation precision bar stays 0.97 — the
|
||||
operator confirmed the general-tag confidence was already well tuned; only the
|
||||
support floor + the CCIP match confidence are raised. The model defaults change
|
||||
for fresh installs; here we bump the existing singleton row IFF it is still at
|
||||
the previous default, so a deliberate operator change is NOT clobbered.
|
||||
|
||||
Revision ID: 0081
|
||||
Revises: 0080
|
||||
Create Date: 2026-07-06
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0081"
|
||||
down_revision: Union[str, None] = "0080"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE ml_settings SET head_auto_apply_min_positives = 50 "
|
||||
"WHERE head_auto_apply_min_positives = 30"
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE ml_settings SET ccip_auto_apply_threshold = 0.95 "
|
||||
"WHERE ccip_auto_apply_threshold = 0.92"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE ml_settings SET head_auto_apply_min_positives = 30 "
|
||||
"WHERE head_auto_apply_min_positives = 50"
|
||||
)
|
||||
op.execute(
|
||||
"UPDATE ml_settings SET ccip_auto_apply_threshold = 0.92 "
|
||||
"WHERE ccip_auto_apply_threshold = 0.95"
|
||||
)
|
||||
@@ -0,0 +1,85 @@
|
||||
"""presentation-chrome auto-hide (#141) — settings knobs + review table
|
||||
|
||||
MLSettings gains presentation_auto_apply_enabled / _threshold and
|
||||
presentation_conflict_threshold: banner + editor-screenshot auto-hide on the
|
||||
sweep with a FLAT threshold (decoupled from content-head graduation), and a
|
||||
conflict threshold that flags an auto-hide that "also looks like content".
|
||||
|
||||
New table presentation_review records an auto-hidden chrome image that also
|
||||
scored high on a content head, surfaced in the Hidden view for a keep-hidden /
|
||||
un-hide decision. Resolved rows are pruned by retention.
|
||||
|
||||
Revision ID: 0082
|
||||
Revises: 0081
|
||||
Create Date: 2026-07-07
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0082"
|
||||
down_revision: Union[str, None] = "0081"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"presentation_auto_apply_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.text("true"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"presentation_auto_apply_threshold", sa.Float(), nullable=False,
|
||||
server_default=sa.text("0.90"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"presentation_conflict_threshold", sa.Float(), nullable=False,
|
||||
server_default=sa.text("0.50"),
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"presentation_review",
|
||||
sa.Column(
|
||||
"image_record_id", sa.Integer(),
|
||||
sa.ForeignKey("image_record.id", ondelete="CASCADE"),
|
||||
primary_key=True,
|
||||
),
|
||||
sa.Column(
|
||||
"tag_id", sa.Integer(),
|
||||
sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
|
||||
),
|
||||
sa.Column(
|
||||
"conflict_tag_id", sa.Integer(),
|
||||
sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True,
|
||||
),
|
||||
sa.Column("conflict_score", sa.Float(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
# The review list queries the unresolved flags (resolved_at IS NULL).
|
||||
op.create_index(
|
||||
"ix_presentation_review_resolved_at", "presentation_review",
|
||||
["resolved_at"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_presentation_review_resolved_at", table_name="presentation_review"
|
||||
)
|
||||
op.drop_table("presentation_review")
|
||||
op.drop_column("ml_settings", "presentation_conflict_threshold")
|
||||
op.drop_column("ml_settings", "presentation_auto_apply_threshold")
|
||||
op.drop_column("ml_settings", "presentation_auto_apply_enabled")
|
||||
@@ -0,0 +1,73 @@
|
||||
"""post-text translation via Interpreter (milestone 143) — Post columns + settings
|
||||
|
||||
Post gains the translated title/description + the detected source language,
|
||||
Interpreter engine_version (cache key), and translated_at — filled by the
|
||||
translate sweep. ImportSettings gains translation_enabled (OFF by default),
|
||||
interpreter_base_url (EMPTY — the operator sets their own, behind a reverse
|
||||
proxy), and translation_target_lang (en).
|
||||
|
||||
Revision ID: 0083
|
||||
Revises: 0082
|
||||
Create Date: 2026-07-07
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0083"
|
||||
down_revision: Union[str, None] = "0082"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"post", sa.Column("post_title_translated", sa.Text(), nullable=True)
|
||||
)
|
||||
op.add_column(
|
||||
"post", sa.Column("description_translated", sa.Text(), nullable=True)
|
||||
)
|
||||
op.add_column(
|
||||
"post",
|
||||
sa.Column("translated_source_lang", sa.String(8), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"post",
|
||||
sa.Column("translation_engine_version", sa.String(128), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"post",
|
||||
sa.Column("translated_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"translation_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.text("false"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"interpreter_base_url", sa.Text(), nullable=False, server_default="",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"translation_target_lang", sa.Text(), nullable=False,
|
||||
server_default="en",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "translation_target_lang")
|
||||
op.drop_column("import_settings", "interpreter_base_url")
|
||||
op.drop_column("import_settings", "translation_enabled")
|
||||
op.drop_column("post", "translated_at")
|
||||
op.drop_column("post", "translation_engine_version")
|
||||
op.drop_column("post", "translated_source_lang")
|
||||
op.drop_column("post", "description_translated")
|
||||
op.drop_column("post", "post_title_translated")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""translation strictness setting + per-post translation override (milestone 155)
|
||||
|
||||
ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance
|
||||
floor, now operator-tunable in the UI; default 0.9 — stricter than the old
|
||||
hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at
|
||||
~0.86). Post gains ``translation_override`` — a sticky per-post choice of
|
||||
auto / force / original so the operator can force a skipped translation on, or
|
||||
knock a wrongly-translated one back to the original, and have it survive a
|
||||
Re-translate-all.
|
||||
|
||||
Revision ID: 0084
|
||||
Revises: 0083
|
||||
Create Date: 2026-07-10
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0084"
|
||||
down_revision: Union[str, None] = "0083"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"translation_min_confidence", sa.Float(), nullable=False,
|
||||
server_default=sa.text("0.9"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"post",
|
||||
sa.Column(
|
||||
"translation_override", sa.String(16), nullable=False,
|
||||
server_default="auto",
|
||||
),
|
||||
)
|
||||
op.create_check_constraint(
|
||||
"ck_post_translation_override",
|
||||
"post",
|
||||
"translation_override IN ('auto', 'force', 'original')",
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint("ck_post_translation_override", "post", type_="check")
|
||||
op.drop_column("post", "translation_override")
|
||||
op.drop_column("import_settings", "translation_min_confidence")
|
||||
@@ -0,0 +1,35 @@
|
||||
"""title-based WIP auto-tagging (task #1458) — ImportSettings toggle
|
||||
|
||||
ImportSettings gains wip_title_tagging_enabled (ON by default): when a freshly
|
||||
imported post's title explicitly declares work-in-progress ("WIP" / "work in
|
||||
progress"), the importer applies the `wip` system tag to its images. No new
|
||||
table — the tag itself is the seeded `wip` system tag (migration 0075) and the
|
||||
application reuses image_tag with source='wip_title'.
|
||||
|
||||
Revision ID: 0085
|
||||
Revises: 0084
|
||||
Create Date: 2026-07-12
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0085"
|
||||
down_revision: Union[str, None] = "0084"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"wip_title_tagging_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.text("true"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "wip_title_tagging_enabled")
|
||||
@@ -0,0 +1,61 @@
|
||||
"""process auto-apply settings + review mode (#1464) — system-tag refactor
|
||||
|
||||
The system-tag behavior refactor gives `wip` / `editor screenshot` (the PROCESS
|
||||
group) their own provisional auto-apply, parallel to the presentation (chrome)
|
||||
sweep. MLSettings gains three knobs: enabled (OFF by default — a new whole-library
|
||||
auto-tagger is opt-in), the flat apply threshold, and the ring-loud conflict
|
||||
threshold. presentation_review gains a `mode` column so one review surface serves
|
||||
both chrome and process flags (existing rows backfill 'chrome'). server_defaults
|
||||
so the existing rows fill cleanly.
|
||||
|
||||
Revision ID: 0086
|
||||
Revises: 0085
|
||||
Create Date: 2026-07-13
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0086"
|
||||
down_revision: Union[str, None] = "0085"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"process_auto_apply_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.text("false"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"process_auto_apply_threshold", sa.Float(), nullable=False,
|
||||
server_default="0.90",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"process_conflict_threshold", sa.Float(), nullable=False,
|
||||
server_default="0.50",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"presentation_review",
|
||||
sa.Column(
|
||||
"mode", sa.String(16), nullable=False,
|
||||
server_default="chrome",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("presentation_review", "mode")
|
||||
op.drop_column("ml_settings", "process_conflict_threshold")
|
||||
op.drop_column("ml_settings", "process_auto_apply_threshold")
|
||||
op.drop_column("ml_settings", "process_auto_apply_enabled")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""soft WIP title tier toggle (#1474) — ImportSettings.wip_soft_title_tagging_enabled
|
||||
|
||||
The soft tier also tags sketch/doodle/scribble titles, but with a provisional source
|
||||
that never trains the head. OFF by default (a lower-precision tier is opt-in).
|
||||
server_default so the existing singleton row (id=1) fills cleanly.
|
||||
|
||||
Revision ID: 0087
|
||||
Revises: 0086
|
||||
Create Date: 2026-07-13
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0087"
|
||||
down_revision: Union[str, None] = "0086"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"wip_soft_title_tagging_enabled", sa.Boolean(), nullable=False,
|
||||
server_default=sa.text("false"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "wip_soft_title_tagging_enabled")
|
||||
@@ -7,6 +7,7 @@ Action surfaces:
|
||||
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
||||
POST /api/admin/tags/prune-unused (Tier A)
|
||||
POST /api/admin/posts/prune-bare (Tier A)
|
||||
POST /api/admin/posts/refetch-external (Tier A)
|
||||
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
||||
|
||||
Tier-C ops take a dry_run body flag (returns projection inline,
|
||||
@@ -22,7 +23,7 @@ from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Artist
|
||||
from ..models import Artist, Post
|
||||
from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
@@ -155,6 +156,10 @@ async def tag_delete(tag_id: int):
|
||||
)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except ValueError as exc:
|
||||
# System tags (#128) — the training-hygiene machinery keys on
|
||||
# these rows.
|
||||
return _bad("system_tag", detail=str(exc))
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@@ -276,6 +281,44 @@ async def posts_reconcile_duplicates():
|
||||
return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
|
||||
|
||||
|
||||
@admin_bp.route("/posts/refetch-external", methods=["POST"])
|
||||
async def posts_refetch_external():
|
||||
"""Surgical re-fetch of a post's external file-host links (operator
|
||||
2026-07-03): the normal cadence never re-walks deep back-catalogue posts,
|
||||
so a deleted external file only comes back by resetting its ExternalLink
|
||||
row(s) — this endpoint does that per post and dispatches the fetches.
|
||||
Sha-dedupe discards payload files that still exist, so only what's
|
||||
missing lands. Body: {external_post_id: str, source_id?: int (to
|
||||
disambiguate the same external id across sources)}."""
|
||||
from ..services.external_links import refetch_links_for_post
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
ext_id = str(body.get("external_post_id") or "").strip()
|
||||
if not ext_id:
|
||||
return _bad("missing_external_post_id",
|
||||
detail="external_post_id is required")
|
||||
raw_source = body.get("source_id")
|
||||
try:
|
||||
source_id = int(raw_source) if raw_source is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_source_id", detail="source_id must be an integer")
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = select(Post.id).where(Post.external_post_id == ext_id)
|
||||
if source_id is not None:
|
||||
stmt = stmt.where(Post.source_id == source_id)
|
||||
post_ids = (await session.execute(stmt)).scalars().all()
|
||||
if not post_ids:
|
||||
return _bad("post_not_found", status=404,
|
||||
detail=f"no post with external_post_id {ext_id!r}")
|
||||
results = {}
|
||||
for pid in post_ids:
|
||||
results[str(pid)] = await session.run_sync(
|
||||
lambda s, p=pid: refetch_links_for_post(s, p)
|
||||
)
|
||||
return jsonify({"posts": results})
|
||||
|
||||
|
||||
def _reset_content_confirm_token(projection: dict) -> str:
|
||||
"""Stable 8-hex token derived from the live counts (mirrors the Tier-C
|
||||
bulk-delete token): it changes whenever the data changes, so the apply can
|
||||
|
||||
@@ -31,6 +31,24 @@ async def create_or_get():
|
||||
}), 201
|
||||
|
||||
|
||||
@artists_bp.route("/<int:artist_id>", methods=["PATCH"])
|
||||
async def rename(artist_id: int):
|
||||
"""Rename an artist's DISPLAY NAME (#130). Name only — the slug and every
|
||||
on-disk path stay put, so this is instant and safe. Name is non-unique."""
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict) or not isinstance(body.get("name"), str):
|
||||
return jsonify({"error": "invalid_body"}), 400
|
||||
async with get_session() as session:
|
||||
svc = ArtistService(session)
|
||||
try:
|
||||
artist = await svc.rename(artist_id, body["name"])
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": "empty_name", "detail": str(exc)}), 400
|
||||
if artist is None:
|
||||
return jsonify({"error": "not_found"}), 404
|
||||
return jsonify({"id": artist.id, "name": artist.name, "slug": artist.slug})
|
||||
|
||||
|
||||
@artists_bp.route("/autocomplete", methods=["GET"])
|
||||
async def autocomplete():
|
||||
q = request.args.get("q") or ""
|
||||
|
||||
@@ -84,11 +84,15 @@ async def quick_add_source():
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session).quick_add_source(url)
|
||||
# crypto lets a pixiv add resolve the artist's display name via the
|
||||
# stored OAuth token (else it falls back to the numeric id). #130.
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad(
|
||||
"unknown_platform",
|
||||
|
||||
+131
-3
@@ -3,9 +3,18 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.gallery_service import GalleryService
|
||||
from ..models import (
|
||||
ImageRecord,
|
||||
PresentationReview,
|
||||
Tag,
|
||||
TagSuggestionRejection,
|
||||
)
|
||||
from ..models.tag import image_tag
|
||||
from ..services.gallery_service import GalleryService, image_url, thumbnail_url
|
||||
|
||||
gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
|
||||
|
||||
@@ -77,6 +86,9 @@ def _parse_filters():
|
||||
platform = request.args.get("platform") or None
|
||||
untagged = request.args.get("untagged") in ("1", "true", "yes")
|
||||
no_artist = request.args.get("no_artist") in ("1", "true", "yes")
|
||||
# Show the presentation chrome (banner / editor screenshot) that the default
|
||||
# gallery hides — the Hidden view sets this (milestone 141).
|
||||
include_hidden = request.args.get("include_hidden") in ("1", "true", "yes")
|
||||
date_from = _parse_date(request.args.get("date_from"))
|
||||
date_to = _parse_date(request.args.get("date_to"))
|
||||
if date_to is not None:
|
||||
@@ -88,6 +100,7 @@ def _parse_filters():
|
||||
"platform": platform,
|
||||
"untagged": untagged, "no_artist": no_artist,
|
||||
"date_from": date_from, "date_to": date_to,
|
||||
"include_hidden": include_hidden,
|
||||
}
|
||||
return filters, sort
|
||||
|
||||
@@ -132,12 +145,32 @@ async def similar():
|
||||
filters, _sort = _parse_filters()
|
||||
except (KeyError, ValueError):
|
||||
return jsonify({"error": "similar_to query param required"}), 400
|
||||
# Explore passes exclude_wip=1 to also drop work-in-progress from the
|
||||
# rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274).
|
||||
exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True")
|
||||
# Explore reach (#1476): 0 = nearest (gallery default), →1 reaches into farther
|
||||
# distance bands so the walk can escape a dense cluster. exclude_ids = the
|
||||
# breadcrumb, so already-walked images aren't re-served as neighbours.
|
||||
try:
|
||||
reach = max(0.0, min(1.0, float(request.args.get("reach", "0"))))
|
||||
except ValueError:
|
||||
reach = 0.0
|
||||
exclude_ids = [
|
||||
int(x) for x in request.args.get("exclude_ids", "").split(",")
|
||||
if x.strip().isdigit()
|
||||
] or None
|
||||
# post_id is the exclusive post-detail view — not a similarity scope.
|
||||
scope = {k: v for k, v in filters.items() if k != "post_id"}
|
||||
# include_hidden is a gallery-browse flag; similar() has its OWN presentation
|
||||
# exclusion (a similarity-quality concern, #1274), so drop it here (#141).
|
||||
scope = {
|
||||
k: v for k, v in filters.items() if k not in ("post_id", "include_hidden")
|
||||
}
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
images = await svc.similar(image_id=similar_to, limit=limit, **scope)
|
||||
images = await svc.similar(
|
||||
image_id=similar_to, limit=limit, exclude_wip=exclude_wip,
|
||||
reach=reach, exclude_ids=exclude_ids, **scope)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
if images is None:
|
||||
@@ -211,6 +244,101 @@ async def jump():
|
||||
return jsonify({"cursor": cursor})
|
||||
|
||||
|
||||
# -- Hidden-view review (#141): auto-hidden chrome flagged "also looks like
|
||||
# content", surfaced in the gallery's Show-hidden review strip. -----------
|
||||
@gallery_bp.route("/hidden-review", methods=["GET"])
|
||||
async def hidden_review():
|
||||
"""Unresolved system-tag auto-apply review flags (chrome + process, #1464),
|
||||
most-concerning first (highest content score) — for the review strip. `mode`
|
||||
tells the client whether the flagged tag hid the image ('chrome') or left it
|
||||
visible ('process'), which decides the resolve labels (un-hide vs remove-tag)."""
|
||||
ptag = aliased(Tag)
|
||||
ctag = aliased(Tag)
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(
|
||||
PresentationReview.image_record_id,
|
||||
PresentationReview.tag_id,
|
||||
PresentationReview.conflict_tag_id,
|
||||
PresentationReview.conflict_score,
|
||||
PresentationReview.mode,
|
||||
ImageRecord.path, ImageRecord.thumbnail_path,
|
||||
ImageRecord.sha256, ImageRecord.mime,
|
||||
ptag.name.label("tag_name"),
|
||||
ctag.name.label("conflict_name"),
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.id == PresentationReview.image_record_id)
|
||||
.join(ptag, ptag.id == PresentationReview.tag_id)
|
||||
.outerjoin(ctag, ctag.id == PresentationReview.conflict_tag_id)
|
||||
.where(PresentationReview.resolved_at.is_(None))
|
||||
.order_by(PresentationReview.conflict_score.desc())
|
||||
)).all()
|
||||
return jsonify({"items": [
|
||||
{
|
||||
"image_id": r.image_record_id,
|
||||
"tag_id": r.tag_id,
|
||||
"tag_name": r.tag_name,
|
||||
"conflict_tag_id": r.conflict_tag_id,
|
||||
"conflict_name": r.conflict_name,
|
||||
"conflict_score": r.conflict_score,
|
||||
"mode": r.mode,
|
||||
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
|
||||
"image_url": image_url(r.path),
|
||||
}
|
||||
for r in rows
|
||||
]})
|
||||
|
||||
|
||||
@gallery_bp.route(
|
||||
"/hidden-review/<int:image_id>/<int:tag_id>/keep", methods=["POST"]
|
||||
)
|
||||
async def hidden_review_keep(image_id, tag_id):
|
||||
"""Keep the auto-hide: resolve the flag; the tag stays applied (#141)."""
|
||||
async with get_session() as session:
|
||||
await session.execute(
|
||||
update(PresentationReview)
|
||||
.where(
|
||||
PresentationReview.image_record_id == image_id,
|
||||
PresentationReview.tag_id == tag_id,
|
||||
)
|
||||
.values(resolved_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@gallery_bp.route(
|
||||
"/hidden-review/<int:image_id>/<int:tag_id>/unhide", methods=["POST"]
|
||||
)
|
||||
async def hidden_review_unhide(image_id, tag_id):
|
||||
"""Un-hide: remove the presentation tag (image returns to the gallery), record
|
||||
a rejection so the head LEARNS it misfired, and resolve the flag (#141)."""
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
async with get_session() as session:
|
||||
await session.execute(
|
||||
delete(image_tag).where(
|
||||
image_tag.c.image_record_id == image_id,
|
||||
image_tag.c.tag_id == tag_id,
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
pg_insert(TagSuggestionRejection)
|
||||
.values(image_record_id=image_id, tag_id=tag_id)
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
await session.execute(
|
||||
update(PresentationReview)
|
||||
.where(
|
||||
PresentationReview.image_record_id == image_id,
|
||||
PresentationReview.tag_id == tag_id,
|
||||
)
|
||||
.values(resolved_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@gallery_bp.route("/image/<int:image_id>", methods=["GET"])
|
||||
async def image_detail(image_id: int):
|
||||
async with get_session() as session:
|
||||
|
||||
@@ -269,6 +269,33 @@ async def lease():
|
||||
).scalars()
|
||||
} if ids else {}
|
||||
await session.commit()
|
||||
# Crop-proposer config, announced FROM THE SETTING like embed_model_name
|
||||
# (#134): the agent builds its detectors from this, rebuilding live when
|
||||
# it changes — so tuning is a DB/UI edit, never an agent restart. Same
|
||||
# block for every job in the batch (it's global), built once. An enabled
|
||||
# toggle off is carried through so the agent skips that proposer.
|
||||
detectors = {
|
||||
"person": {
|
||||
"enabled": ml.detector_person_enabled,
|
||||
"weights": ml.detector_person_weights,
|
||||
"conf": ml.detector_person_conf,
|
||||
},
|
||||
"anatomy": {
|
||||
"enabled": ml.detector_anatomy_enabled,
|
||||
"weights": ml.detector_anatomy_weights,
|
||||
"conf": ml.detector_anatomy_conf,
|
||||
},
|
||||
"panel": {
|
||||
"enabled": ml.detector_panel_enabled,
|
||||
"weights": ml.detector_panel_weights,
|
||||
"conf": ml.detector_panel_conf,
|
||||
},
|
||||
"max_figures": ml.detector_max_figures,
|
||||
"max_components": ml.detector_max_components,
|
||||
"max_panels": ml.detector_max_panels,
|
||||
"max_regions": ml.detector_max_regions,
|
||||
"dedupe_iou": ml.detector_dedupe_iou,
|
||||
}
|
||||
out = []
|
||||
for j in jobs:
|
||||
img = imgs.get(j.image_record_id)
|
||||
@@ -290,6 +317,7 @@ async def lease():
|
||||
# re-embed, never an agent change.
|
||||
"embed_model_name": ml.embedder_model_name,
|
||||
"embed_version": ml.embedder_model_version,
|
||||
"detectors": detectors,
|
||||
})
|
||||
return jsonify({"jobs": out})
|
||||
|
||||
|
||||
@@ -8,6 +8,26 @@ from ..models import MLSettings
|
||||
ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
||||
|
||||
|
||||
# Crop-proposer / detector config (#134). Announced to the GPU agent in the lease
|
||||
# → tunable here with no restart. weights = ultralytics name | URL | hf_repo::file
|
||||
# (empty, or enabled off, skips that proposer).
|
||||
_DETECTOR_FIELDS = (
|
||||
"detector_person_enabled",
|
||||
"detector_person_weights",
|
||||
"detector_person_conf",
|
||||
"detector_anatomy_enabled",
|
||||
"detector_anatomy_weights",
|
||||
"detector_anatomy_conf",
|
||||
"detector_panel_enabled",
|
||||
"detector_panel_weights",
|
||||
"detector_panel_conf",
|
||||
"detector_max_figures",
|
||||
"detector_max_components",
|
||||
"detector_max_panels",
|
||||
"detector_max_regions",
|
||||
"detector_dedupe_iou",
|
||||
)
|
||||
|
||||
_EDITABLE = (
|
||||
"cpu_embed_enabled",
|
||||
"video_frame_interval_seconds",
|
||||
@@ -19,8 +39,15 @@ _EDITABLE = (
|
||||
"ccip_match_threshold",
|
||||
"ccip_auto_apply_enabled",
|
||||
"ccip_auto_apply_threshold",
|
||||
"presentation_auto_apply_enabled",
|
||||
"presentation_auto_apply_threshold",
|
||||
"presentation_conflict_threshold",
|
||||
"process_auto_apply_enabled",
|
||||
"process_auto_apply_threshold",
|
||||
"process_conflict_threshold",
|
||||
"embedder_model_name",
|
||||
"embedder_model_version",
|
||||
*_DETECTOR_FIELDS,
|
||||
)
|
||||
|
||||
|
||||
@@ -75,7 +102,14 @@ async def get_settings():
|
||||
"ccip_match_threshold": s.ccip_match_threshold,
|
||||
"ccip_auto_apply_enabled": s.ccip_auto_apply_enabled,
|
||||
"ccip_auto_apply_threshold": s.ccip_auto_apply_threshold,
|
||||
"presentation_auto_apply_enabled": s.presentation_auto_apply_enabled,
|
||||
"presentation_auto_apply_threshold": s.presentation_auto_apply_threshold,
|
||||
"presentation_conflict_threshold": s.presentation_conflict_threshold,
|
||||
"process_auto_apply_enabled": s.process_auto_apply_enabled,
|
||||
"process_auto_apply_threshold": s.process_auto_apply_threshold,
|
||||
"process_conflict_threshold": s.process_conflict_threshold,
|
||||
"embedder_model_name": s.embedder_model_name,
|
||||
**{f: getattr(s, f) for f in _DETECTOR_FIELDS},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -128,11 +162,36 @@ def _validate(p: dict) -> str | None:
|
||||
return "ccip_match_threshold must be between 0.5 and 0.999"
|
||||
if not (0.5 <= float(p["ccip_auto_apply_threshold"]) <= 0.999):
|
||||
return "ccip_auto_apply_threshold must be between 0.5 and 0.999"
|
||||
# Presentation chrome auto-hide (#141). Auto-apply runs high (hiding is
|
||||
# consequential); the conflict cut is a plain probability [0,1].
|
||||
if not (0.5 <= float(p["presentation_auto_apply_threshold"]) <= 0.999):
|
||||
return "presentation_auto_apply_threshold must be between 0.5 and 0.999"
|
||||
if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0):
|
||||
return "presentation_conflict_threshold must be between 0 and 1"
|
||||
# Process auto-apply (#1464). wip/editor stay VISIBLE so a false apply is
|
||||
# low-harm (excludes-from-training + a review flag), but keep the same bar.
|
||||
if not (0.5 <= float(p["process_auto_apply_threshold"]) <= 0.999):
|
||||
return "process_auto_apply_threshold must be between 0.5 and 0.999"
|
||||
if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
|
||||
return "process_conflict_threshold must be between 0 and 1"
|
||||
# Embedder model swap (#1190): both must be non-empty. Changing them means a
|
||||
# different embedding space — the operator must re-embed + retrain after.
|
||||
for key in ("embedder_model_name", "embedder_model_version"):
|
||||
if not str(p[key]).strip():
|
||||
return f"{key} must not be empty"
|
||||
# Crop proposers (#134). Weights may be empty (that proposer is just off);
|
||||
# confidences are probabilities; caps are positive counts; IoU is [0,1].
|
||||
for key in ("detector_person_conf", "detector_anatomy_conf", "detector_panel_conf"):
|
||||
if not (0.0 <= float(p[key]) <= 1.0):
|
||||
return f"{key} must be between 0 and 1"
|
||||
for key in (
|
||||
"detector_max_figures", "detector_max_components",
|
||||
"detector_max_panels", "detector_max_regions",
|
||||
):
|
||||
if int(p[key]) < 1:
|
||||
return f"{key} must be >= 1"
|
||||
if not (0.0 <= float(p["detector_dedupe_iou"]) <= 1.0):
|
||||
return "detector_dedupe_iou must be between 0 and 1"
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -3,12 +3,28 @@
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import ImportSettings, Post
|
||||
from ..services import interpreter_client as ic
|
||||
from ..services.post_feed_service import PostFeedService
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
from ..utils.text import html_to_plain
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
||||
|
||||
_TRANSLATION_OVERRIDES = ("auto", "force", "original")
|
||||
|
||||
|
||||
def _queue_for_sweep(post: Post) -> None:
|
||||
"""Mark a post untranslated (all translation columns NULL) so the periodic
|
||||
sweep re-runs it under its new override — used when Interpreter is down and we
|
||||
can't translate inline."""
|
||||
post.post_title_translated = None
|
||||
post.description_translated = None
|
||||
post.translated_source_lang = None
|
||||
post.translation_engine_version = None
|
||||
post.translated_at = None
|
||||
|
||||
|
||||
@posts_bp.route("", methods=["GET"])
|
||||
async def list_posts():
|
||||
@@ -82,3 +98,70 @@ async def get_post(post_id: int):
|
||||
if item is None:
|
||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||
return jsonify(item)
|
||||
|
||||
|
||||
@posts_bp.route("/<int:post_id>/translation-override", methods=["POST"])
|
||||
async def set_translation_override(post_id: int):
|
||||
"""Sticky per-post translation override (milestone 155). Body:
|
||||
``{"override": "auto" | "force" | "original"}``.
|
||||
|
||||
'original' keeps the original (clears any stored translation now — no
|
||||
Interpreter needed). 'force'/'auto' translate the post immediately if the
|
||||
service is up (force bypasses the acceptance floor; auto re-runs the gate);
|
||||
if it's down we save the flag and mark the post untranslated so the next sweep
|
||||
applies it. The override persists, so the sweep + Re-translate-all keep
|
||||
honoring it. Returns the updated translation fields + an ``applied`` status."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
override = body.get("override")
|
||||
if override not in _TRANSLATION_OVERRIDES:
|
||||
return _bad(
|
||||
"invalid_override",
|
||||
detail=f"override must be one of {list(_TRANSLATION_OVERRIDES)}",
|
||||
)
|
||||
|
||||
# Lazy import (mirrors settings.py) so the API module doesn't pull the celery
|
||||
# task graph at import time.
|
||||
from ..tasks.translation import _store_translation, _translate_field
|
||||
|
||||
async with get_session() as session:
|
||||
post = await session.get(Post, post_id)
|
||||
if post is None:
|
||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||
post.translation_override = override
|
||||
cfg = await ImportSettings.load(session)
|
||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
||||
|
||||
if override == "original":
|
||||
_store_translation(post, (None, None, None), (None, None, None), target)
|
||||
applied = "cleared"
|
||||
else:
|
||||
base_url = (cfg.interpreter_base_url or "").strip()
|
||||
if cfg.translation_enabled and base_url and ic.health(base_url):
|
||||
force = override == "force"
|
||||
title = (post.post_title or "").strip()
|
||||
desc = (html_to_plain(post.description) if post.description else "") or ""
|
||||
desc = desc.strip()
|
||||
mc = cfg.translation_min_confidence
|
||||
try:
|
||||
title_res = _translate_field(title, base_url, target, mc, force=force)
|
||||
desc_res = _translate_field(desc, base_url, target, mc, force=force)
|
||||
except ic.InterpreterUnavailable:
|
||||
_queue_for_sweep(post)
|
||||
applied = "queued"
|
||||
else:
|
||||
_store_translation(post, title_res, desc_res, target)
|
||||
applied = "translated"
|
||||
else:
|
||||
# Disabled / no URL / unhealthy → let the sweep apply it later.
|
||||
_queue_for_sweep(post)
|
||||
applied = "queued"
|
||||
|
||||
await session.commit()
|
||||
return jsonify({
|
||||
"id": post.id,
|
||||
"translation_override": post.translation_override,
|
||||
"post_title_translated": post.post_title_translated,
|
||||
"description_translated": post.description_translated,
|
||||
"translated_source_lang": post.translated_source_lang,
|
||||
"applied": applied,
|
||||
})
|
||||
|
||||
+223
-2
@@ -1,12 +1,24 @@
|
||||
"""Settings API: import filters, system stats."""
|
||||
|
||||
import asyncio
|
||||
import secrets
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, or_, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, Artist, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
|
||||
from ..models import (
|
||||
AppSetting,
|
||||
Artist,
|
||||
ImageRecord,
|
||||
ImportBatch,
|
||||
ImportSettings,
|
||||
ImportTask,
|
||||
Post,
|
||||
Tag,
|
||||
TaskRun,
|
||||
)
|
||||
from ..services import interpreter_client as ic
|
||||
|
||||
settings_bp = Blueprint("settings", __name__, url_prefix="/api")
|
||||
|
||||
@@ -32,6 +44,12 @@ _EDITABLE_FIELDS = (
|
||||
"extdl_mediafire_enabled",
|
||||
"extdl_dropbox_enabled",
|
||||
"extdl_pixeldrain_enabled",
|
||||
"translation_enabled",
|
||||
"interpreter_base_url",
|
||||
"translation_target_lang",
|
||||
"translation_min_confidence",
|
||||
"wip_title_tagging_enabled",
|
||||
"wip_soft_title_tagging_enabled",
|
||||
)
|
||||
|
||||
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
||||
@@ -69,6 +87,12 @@ async def get_import_settings():
|
||||
"extdl_mediafire_enabled": row.extdl_mediafire_enabled,
|
||||
"extdl_dropbox_enabled": row.extdl_dropbox_enabled,
|
||||
"extdl_pixeldrain_enabled": row.extdl_pixeldrain_enabled,
|
||||
"translation_enabled": row.translation_enabled,
|
||||
"interpreter_base_url": row.interpreter_base_url,
|
||||
"translation_target_lang": row.translation_target_lang,
|
||||
"translation_min_confidence": row.translation_min_confidence,
|
||||
"wip_title_tagging_enabled": row.wip_title_tagging_enabled,
|
||||
"wip_soft_title_tagging_enabled": row.wip_soft_title_tagging_enabled,
|
||||
})
|
||||
|
||||
|
||||
@@ -128,12 +152,41 @@ async def update_import_settings():
|
||||
for tog in _EXTDL_TOGGLE_FIELDS:
|
||||
if tog in body and not isinstance(body[tog], bool):
|
||||
return jsonify({"error": f"{tog} must be a boolean"}), 400
|
||||
# Translation (#143): base URL may be empty (feature off until set — no
|
||||
# default host; the operator points it at their own Interpreter proxy).
|
||||
if "translation_enabled" in body and not isinstance(
|
||||
body["translation_enabled"], bool
|
||||
):
|
||||
return jsonify({"error": "translation_enabled must be a boolean"}), 400
|
||||
for key in ("interpreter_base_url", "translation_target_lang"):
|
||||
if key in body and not isinstance(body[key], str):
|
||||
return jsonify({"error": f"{key} must be a string"}), 400
|
||||
# Acceptance floor (milestone 155): latin-script translations below this
|
||||
# Interpreter confidence are kept as the original.
|
||||
if "translation_min_confidence" in body:
|
||||
v = body["translation_min_confidence"]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||
return jsonify(
|
||||
{"error": "translation_min_confidence must be a number in [0, 1]"}
|
||||
), 400
|
||||
if "series_suggest_threshold" in body:
|
||||
v = body["series_suggest_threshold"]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||
return jsonify(
|
||||
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
||||
), 400
|
||||
if "wip_title_tagging_enabled" in body and not isinstance(
|
||||
body["wip_title_tagging_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "wip_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
if "wip_soft_title_tagging_enabled" in body and not isinstance(
|
||||
body["wip_soft_title_tagging_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "wip_soft_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
@@ -145,6 +198,18 @@ async def update_import_settings():
|
||||
return await get_import_settings()
|
||||
|
||||
|
||||
@settings_bp.route("/settings/wip-title/scan", methods=["POST"])
|
||||
async def wip_title_scan():
|
||||
"""Enqueue the back-catalogue WIP-title scan (task #1458 Settings button):
|
||||
apply the `wip` system tag to EXISTING posts whose title declares
|
||||
work-in-progress. New imports are tagged live by the importer; this catches
|
||||
the existing library. Returns the Celery task id (202)."""
|
||||
from ..tasks.maintenance import backfill_wip_title_tags
|
||||
|
||||
r = backfill_wip_title_tags.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@settings_bp.route("/system/stats", methods=["GET"])
|
||||
async def system_stats():
|
||||
async with get_session() as session:
|
||||
@@ -270,3 +335,159 @@ async def rotate_extension_api_key():
|
||||
row.value = new_value
|
||||
await session.commit()
|
||||
return jsonify({"key": new_value})
|
||||
|
||||
|
||||
# --- Translation (#143): live status + manual "Translate now" --------------
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/status", methods=["GET"])
|
||||
async def translation_status():
|
||||
"""For the Settings card: is it on, is a URL set, is the service reachable,
|
||||
and how many posts still await translation. Health runs the sync client in a
|
||||
thread so the event loop isn't blocked."""
|
||||
translation_tasks = (
|
||||
"backend.app.tasks.translation.translate_posts",
|
||||
"backend.app.tasks.translation.retranslate_posts",
|
||||
)
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
untranslated = (await session.execute(
|
||||
select(func.count(Post.id))
|
||||
.where(Post.translated_source_lang.is_(None))
|
||||
.where(or_(
|
||||
Post.post_title.is_not(None), Post.description.is_not(None),
|
||||
))
|
||||
)).scalar_one()
|
||||
# Live progress: is a sweep running now, and what did the last one do?
|
||||
# (run-until-done re-enqueues itself, so `active` stays true across a
|
||||
# bulk re-translate; `last_run` surfaces a completed run's outcome.)
|
||||
active = (await session.execute(
|
||||
select(func.count(TaskRun.id))
|
||||
.where(TaskRun.task_name.in_(translation_tasks))
|
||||
.where(TaskRun.status == "running")
|
||||
)).scalar_one()
|
||||
last = (await session.execute(
|
||||
select(TaskRun.task_name, TaskRun.status, TaskRun.finished_at)
|
||||
.where(TaskRun.task_name.in_(translation_tasks))
|
||||
.where(TaskRun.finished_at.is_not(None))
|
||||
.order_by(TaskRun.finished_at.desc())
|
||||
.limit(1)
|
||||
)).first()
|
||||
base_url = (cfg.interpreter_base_url or "").strip()
|
||||
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
||||
return jsonify({
|
||||
"enabled": cfg.translation_enabled,
|
||||
"base_url_set": bool(base_url),
|
||||
"healthy": healthy,
|
||||
"untranslated_count": int(untranslated),
|
||||
"active": int(active) > 0,
|
||||
"last_run": {
|
||||
"task": last[0].rsplit(".", 1)[-1],
|
||||
"status": last[1],
|
||||
"finished_at": last[2].isoformat() if last[2] else None,
|
||||
} if last else None,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/test", methods=["POST"])
|
||||
async def translation_test():
|
||||
"""On-demand reachability check for a GIVEN Interpreter base URL (the Settings
|
||||
'Test connection' button) — pings /v1/health without saving, so the operator
|
||||
can verify a URL before enabling. Health runs in a thread (sync client)."""
|
||||
body = await request.get_json()
|
||||
base_url = ""
|
||||
if isinstance(body, dict):
|
||||
base_url = (body.get("base_url") or "").strip()
|
||||
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
||||
return jsonify({"healthy": healthy})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/probe", methods=["POST"])
|
||||
async def translation_probe():
|
||||
"""Diagnostic for the Settings 'Test translation' box: translate a pasted
|
||||
snippet WITHOUT saving anything, returning what Interpreter *detected*
|
||||
(language + confidence) alongside the result. Lets the operator see why a
|
||||
given string was (mis-)detected — e.g. a short English title flagged as
|
||||
another language — so a detection guard can be tuned from real numbers.
|
||||
Read-only: no post is touched. Uses the currently-saved base URL + target."""
|
||||
body = await request.get_json(silent=True)
|
||||
body = body if isinstance(body, dict) else {}
|
||||
text = (body.get("text") or "").strip()
|
||||
if not text:
|
||||
return jsonify({"error": "provide text to translate"}), 400
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
base_url = (cfg.interpreter_base_url or "").strip()
|
||||
if not base_url:
|
||||
return jsonify({"error": "no Interpreter base URL is set"}), 400
|
||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
||||
try:
|
||||
res = await asyncio.to_thread(
|
||||
ic.translate, [text], base_url=base_url, target=target,
|
||||
)
|
||||
except ic.InterpreterUnavailable as e:
|
||||
return jsonify({"error": f"Interpreter unavailable: {e}"}), 503
|
||||
except ic.InterpreterBadRequest as e:
|
||||
return jsonify({"error": f"Interpreter rejected the request: {e}"}), 400
|
||||
translations = res.get("translations") or []
|
||||
return jsonify({
|
||||
"target": target,
|
||||
"detected_lang": res.get("detected_lang"),
|
||||
"detected_confidence": res.get("detected_confidence"),
|
||||
"engine": res.get("engine"),
|
||||
"engine_version": res.get("engine_version"),
|
||||
"translated": translations[0] if translations else None,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/run", methods=["POST"])
|
||||
async def translation_run():
|
||||
"""Enqueue the translate sweep now (the Settings 'Translate now' button).
|
||||
Runs in drain mode — run-until-done — so one press chases the whole
|
||||
untranslated backlog to zero rather than a single 300-post chunk."""
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
|
||||
return jsonify(
|
||||
{"error": "translation is disabled or no base URL is set"}
|
||||
), 400
|
||||
from ..tasks.translation import translate_posts
|
||||
|
||||
r = translate_posts.delay(drain=True)
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/retranslate", methods=["POST"])
|
||||
async def translation_retranslate():
|
||||
"""Re-translate stored translations after a model change (m146). Body:
|
||||
``{"artist_id": <int>}`` aims at one artist; ``{"all": true}`` re-runs every
|
||||
artist. ``all`` must be explicit so an empty/typo body can't wipe everything.
|
||||
Clears the scoped translations and enqueues the run-until-done retranslate
|
||||
sweep (the Interpreter cache re-translates on a changed model, is cache-fast
|
||||
otherwise). Same enabled + base-URL guard as 'Translate now'."""
|
||||
body = await request.get_json(silent=True)
|
||||
body = body if isinstance(body, dict) else {}
|
||||
artist_id = body.get("artist_id")
|
||||
do_all = bool(body.get("all"))
|
||||
if artist_id is None and not do_all:
|
||||
return jsonify(
|
||||
{"error": "provide artist_id, or all=true to re-translate everything"}
|
||||
), 400
|
||||
if artist_id is not None:
|
||||
try:
|
||||
artist_id = int(artist_id)
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({"error": "artist_id must be an integer"}), 400
|
||||
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
|
||||
return jsonify(
|
||||
{"error": "translation is disabled or no base URL is set"}
|
||||
), 400
|
||||
from ..tasks.translation import retranslate_posts
|
||||
|
||||
# artist_id wins when both are sent; otherwise all=true → None (every artist).
|
||||
artist_ids = [artist_id] if artist_id is not None else None
|
||||
r = retranslate_posts.delay(artist_ids=artist_ids)
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
+19
-44
@@ -136,6 +136,25 @@ async def delete_source(source_id: int):
|
||||
return "", 204
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/reassign", methods=["POST"])
|
||||
async def reassign_source(source_id: int):
|
||||
"""Move this source (and the content it brought in) to another artist
|
||||
(#130). Files don't move — the slug is immutable — so this just re-attributes
|
||||
the source, its posts, and its images. Body: {target_artist_id}."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
target = body.get("target_artist_id")
|
||||
if not isinstance(target, int):
|
||||
return _bad("invalid_body", detail="target_artist_id (int) required")
|
||||
async with get_session() as session:
|
||||
try:
|
||||
record = await SourceService(session).reassign(source_id, target)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except ArtistNotFoundError:
|
||||
return _bad("artist_not_found", detail="target artist not found", status=404)
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
||||
async def set_backfill(source_id: int):
|
||||
"""Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
|
||||
@@ -211,50 +230,6 @@ async def set_backfill(source_id: int):
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/preview", methods=["POST"])
|
||||
async def preview_source_endpoint(source_id: int):
|
||||
"""Plan #708 B4: dry-run — count what a backfill WOULD download for a native
|
||||
platform (Patreon today), without downloading. Walks the first few feed pages
|
||||
and counts media not already in the seen/dead ledgers. Returns
|
||||
{total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason
|
||||
(unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no
|
||||
cheap dry-run — their verify is a slow --simulate)."""
|
||||
from pathlib import Path
|
||||
|
||||
from ..services.credential_service import CredentialService
|
||||
from ..services.download_backends import preview_source, uses_native_ingester
|
||||
from ..tasks._sync_engine import sync_session_factory
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
rec = await SourceService(session).get(source_id)
|
||||
if rec is None:
|
||||
return _bad("not_found", status=404)
|
||||
if not uses_native_ingester(rec.platform):
|
||||
return _bad(
|
||||
"unsupported",
|
||||
detail="Preview is only available for native-ingester platforms.",
|
||||
status=400,
|
||||
)
|
||||
cred = CredentialService(session, _get_crypto())
|
||||
cookies_path = await cred.get_cookies_path(rec.platform)
|
||||
|
||||
# The walk + ledger reads are sync (run off the request loop); the process
|
||||
# sync engine is the same one the download task uses.
|
||||
result = await preview_source(
|
||||
platform=rec.platform,
|
||||
url=rec.url,
|
||||
source_id=source_id,
|
||||
config_overrides=rec.config_overrides or {},
|
||||
cookies_path=str(cookies_path) if cookies_path else None,
|
||||
images_root=Path("/images"),
|
||||
sync_session_factory=sync_session_factory(),
|
||||
)
|
||||
if "error" in result:
|
||||
return _bad("preview_failed", detail=result["error"], status=409)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
||||
async def check_source(source_id: int):
|
||||
"""FC-3c: enqueue a download for this source.
|
||||
|
||||
@@ -11,10 +11,11 @@ suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
|
||||
|
||||
@suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
|
||||
async def get_suggestions(image_id: int):
|
||||
# ?min=<float> overrides the configured per-category thresholds so the typed
|
||||
# tag-input dropdown can surface EVERY stored prediction (min=0), including
|
||||
# low-confidence actions/features, in canonical formatting. Omitted → the
|
||||
# curated above-threshold list the Suggestions panel uses.
|
||||
# ?min=<float> overrides the per-head suggest thresholds for INCLUSION. The
|
||||
# rail sends min=0 in its single per-image fetch to get EVERY head (each row
|
||||
# still carries above_threshold vs its natural cut), then derives the panel
|
||||
# (above_threshold) and the typed dropdown (all, filtered by text) client-side
|
||||
# — no second request. Omitted → only above-threshold rows.
|
||||
override = None
|
||||
raw_min = request.args.get("min")
|
||||
if raw_min is not None:
|
||||
@@ -36,16 +37,19 @@ async def get_suggestions(image_id: int):
|
||||
"category": s.category,
|
||||
"score": round(s.score, 4),
|
||||
"source": s.source,
|
||||
"creates_new_tag": s.creates_new_tag,
|
||||
# raw model key (alias is stored under this) + whether an
|
||||
# operator alias produced this suggestion — drive the
|
||||
# modal's "Treat as alias"/"Remove alias" affordances.
|
||||
"raw_name": s.raw_name,
|
||||
"via_alias": s.via_alias,
|
||||
# whether the score cleared the head's own suggest cut.
|
||||
# The single min=0 fetch returns every head; the panel
|
||||
# shows above_threshold, the typed dropdown shows all and
|
||||
# annotates each match with its score.
|
||||
"above_threshold": s.above_threshold,
|
||||
# operator dismissed this tag for this image — surfaced
|
||||
# (not dropped) so the rail can show it rejected + offer
|
||||
# one-click un-reject.
|
||||
"rejected": s.rejected,
|
||||
# the crop region that produced this tag (#1206) —
|
||||
# {bbox,kind,detector} or null (whole-image won). Drives
|
||||
# the hover→overlay highlight.
|
||||
"grounding": s.grounding,
|
||||
}
|
||||
for s in items
|
||||
]
|
||||
@@ -69,26 +73,6 @@ async def accept_suggestion(image_id: int):
|
||||
return jsonify({"accepted": True, "tag_id": tag_id})
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
"/images/<int:image_id>/suggestions/alias", methods=["POST"]
|
||||
)
|
||||
async def alias_suggestion(image_id: int):
|
||||
body = await request.get_json()
|
||||
required = {"alias_string", "alias_category", "canonical_tag_id"}
|
||||
if not body or not required.issubset(body):
|
||||
return jsonify({"error": f"required: {sorted(required)}"}), 400
|
||||
canonical_tag_id = body["canonical_tag_id"]
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).add_alias_and_accept(
|
||||
image_id,
|
||||
body["alias_string"],
|
||||
body["alias_category"],
|
||||
canonical_tag_id,
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"accepted": True, "tag_id": canonical_tag_id})
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
"/images/<int:image_id>/suggestions/dismiss", methods=["POST"]
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from ..models.tag import image_tag
|
||||
from ..models.tag_suggestion_rejection import TagSuggestionRejection
|
||||
from ..services.bulk_tag_service import BulkTagService
|
||||
from ..services.ml.aliases import AliasService
|
||||
from ..services.ml.heads import ground_applied_tag
|
||||
from ..services.series_match_service import SeriesMatchService
|
||||
from ..services.series_service import SeriesError, SeriesService
|
||||
from ..services.tag_directory_service import TagDirectoryService
|
||||
@@ -310,6 +311,21 @@ async def confirm_tag_on_image(image_id: int, tag_id: int):
|
||||
return "", 204
|
||||
|
||||
|
||||
@tags_bp.route(
|
||||
"/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
|
||||
)
|
||||
async def tag_grounding(image_id: int, tag_id: int):
|
||||
"""Which crop region best explains an ALREADY-APPLIED tag on this image
|
||||
(#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
|
||||
mirroring the suggestion rail's live grounding. Computed on demand (applied
|
||||
tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
|
||||
has_head: bool}; has_head False means the tag has no head to localize with,
|
||||
so the chip shows no overlay."""
|
||||
async with get_session() as session:
|
||||
grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
|
||||
return jsonify({"grounding": grounding, "has_head": has_head})
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
|
||||
async def get_tag(tag_id: int):
|
||||
"""Resolve a single tag (used by the gallery to label its active
|
||||
@@ -324,6 +340,7 @@ async def get_tag(tag_id: int):
|
||||
"name": tag.name,
|
||||
"kind": tag.kind.value,
|
||||
"fandom_id": tag.fandom_id,
|
||||
"is_system": tag.is_system,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -390,6 +407,7 @@ async def update_tag(tag_id: int):
|
||||
"name": tag.name,
|
||||
"kind": tag.kind.value,
|
||||
"fandom_id": tag.fandom_id,
|
||||
"is_system": tag.is_system,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -35,6 +35,7 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.backup",
|
||||
"backend.app.tasks.admin",
|
||||
"backend.app.tasks.library_audit",
|
||||
"backend.app.tasks.translation",
|
||||
],
|
||||
)
|
||||
app.conf.update(
|
||||
@@ -63,9 +64,20 @@ def make_celery() -> Celery:
|
||||
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||
# Translation backfill hits the LLM (~1–6s/item) → the long lane so it
|
||||
# never starves the quick self-healing sweeps (#143).
|
||||
"backend.app.tasks.translation.*": {"queue": "maintenance_long"},
|
||||
},
|
||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||
task_acks_late=True,
|
||||
# Deploy graceful-shutdown safety: with acks_late, a task killed because
|
||||
# it outran the container's stop-grace window (SIGKILL) is re-queued
|
||||
# rather than silently lost. Safe because our long tasks are idempotent +
|
||||
# chunked (translation per-post commit, downloads terminal-status, audits
|
||||
# chunk) and the 5-min recovery sweeps re-drive anything left non-terminal
|
||||
# — a re-run resumes cleanly and never corrupts. No redeliver-loop risk:
|
||||
# heavy GPU work is tombstoned via gpu_queue, not run inline in a worker.
|
||||
task_reject_on_worker_lost=True,
|
||||
worker_prefetch_multiplier=1,
|
||||
# Broker resilience (2026-06-24): a swarm overlay-network blip after a
|
||||
# redeploy left Redis healthy but transiently unreachable, and a worker
|
||||
@@ -103,10 +115,24 @@ def make_celery() -> Celery:
|
||||
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
||||
"schedule": 86400.0, # daily
|
||||
},
|
||||
"cleanup-orphaned-temp-files": {
|
||||
"task": "backend.app.tasks.maintenance.cleanup_orphaned_temp_files",
|
||||
"schedule": 86400.0, # daily — sweep .part/.partial left by a
|
||||
# download/import killed mid-write (graceful-shutdown fallout)
|
||||
},
|
||||
"train-heads-nightly": {
|
||||
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
||||
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
||||
},
|
||||
"refresh-character-prototypes": {
|
||||
"task": "backend.app.tasks.ml.refresh_character_prototypes",
|
||||
"schedule": 900.0, # ~15 min; cheap global-gate no-op when idle (#1317)
|
||||
},
|
||||
"reconcile-character-prototypes-nightly": {
|
||||
"task": "backend.app.tasks.ml.refresh_character_prototypes",
|
||||
"schedule": 86400.0, # nightly FULL reconcile (belt-and-suspenders)
|
||||
"args": (True,), # full=True
|
||||
},
|
||||
"apply-head-tags-daily": {
|
||||
"task": "backend.app.tasks.ml.scheduled_apply_head_tags",
|
||||
"schedule": 86400.0, # no-op unless head_auto_apply_enabled
|
||||
@@ -138,6 +164,38 @@ def make_celery() -> Celery:
|
||||
"task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
|
||||
"schedule": 86400.0, # no-op unless ccip_auto_apply_enabled
|
||||
},
|
||||
"retract-auto-tags-daily": {
|
||||
"task": "backend.app.tasks.ml.scheduled_retract_auto_tags",
|
||||
"schedule": 86400.0, # soft auto-apply: drop auto-tags now below
|
||||
# their threshold (m139); no-op unless the auto-apply switch is on
|
||||
},
|
||||
"presentation-auto-apply-daily": {
|
||||
"task": "backend.app.tasks.ml.scheduled_presentation_auto_apply",
|
||||
"schedule": 86400.0, # auto-hide banner chrome (#141);
|
||||
# no-op unless presentation_auto_apply_enabled
|
||||
},
|
||||
"process-auto-apply-daily": {
|
||||
"task": "backend.app.tasks.ml.scheduled_process_auto_apply",
|
||||
"schedule": 86400.0, # auto-tag wip/editor process art (#1464);
|
||||
# no-op unless process_auto_apply_enabled (opt-in)
|
||||
},
|
||||
"soft-wip-conflict-audit-daily": {
|
||||
"task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
||||
"schedule": 86400.0, # flag ring-loud soft-WIP (sketch/doodle) tags
|
||||
# for review (#1474); no-op with no content heads
|
||||
},
|
||||
"prune-presentation-reviews-daily": {
|
||||
"task": "backend.app.tasks.ml.prune_presentation_reviews",
|
||||
"schedule": 86400.0, # retention: drop resolved review flags >30d
|
||||
},
|
||||
"translate-posts-8h": {
|
||||
"task": "backend.app.tasks.translation.translate_posts",
|
||||
"schedule": 28800.0, # every 8h: steady-state cadence for the
|
||||
# trickle of newly-imported posts (no-op unless translation
|
||||
# configured + healthy). One bounded 300-chunk per fire — the
|
||||
# one-time backlog drains via the "Translate now" button
|
||||
# (drain=True, run-until-done), not this sweep.
|
||||
},
|
||||
"snapshot-head-metrics-daily": {
|
||||
"task": "backend.app.tasks.maintenance.snapshot_head_metrics",
|
||||
"schedule": 86400.0,
|
||||
|
||||
@@ -5,6 +5,7 @@ from .artist import Artist
|
||||
from .artist_visit import ArtistVisit
|
||||
from .backup_run import BackupRun
|
||||
from .base import Base
|
||||
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
||||
from .credential import Credential
|
||||
from .download_event import DownloadEvent
|
||||
from .external_link import ExternalLink
|
||||
@@ -23,8 +24,11 @@ from .library_audit_run import LibraryAuditRun
|
||||
from .ml_settings import MLSettings
|
||||
from .patreon_failed_media import PatreonFailedMedia
|
||||
from .patreon_seen_media import PatreonSeenMedia
|
||||
from .pixiv_failed_media import PixivFailedMedia
|
||||
from .pixiv_seen_media import PixivSeenMedia
|
||||
from .post import Post
|
||||
from .post_attachment import PostAttachment
|
||||
from .presentation_review import PresentationReview
|
||||
from .series_chapter import SeriesChapter
|
||||
from .series_page import SeriesPage
|
||||
from .series_suggestion import SeriesSuggestion
|
||||
@@ -48,10 +52,13 @@ __all__ = [
|
||||
"Credential",
|
||||
"PatreonFailedMedia",
|
||||
"PatreonSeenMedia",
|
||||
"PixivFailedMedia",
|
||||
"PixivSeenMedia",
|
||||
"SubscribeStarFailedMedia",
|
||||
"SubscribeStarSeenMedia",
|
||||
"Post",
|
||||
"PostAttachment",
|
||||
"PresentationReview",
|
||||
"SeriesChapter",
|
||||
"SeriesPage",
|
||||
"SeriesSuggestion",
|
||||
@@ -75,6 +82,8 @@ __all__ = [
|
||||
"HeadTrainingRun",
|
||||
"TagAlias",
|
||||
"TagHead",
|
||||
"CharacterPrototype",
|
||||
"CcipPrototypeState",
|
||||
"TagPositiveConfirmation",
|
||||
"TagSuggestionRejection",
|
||||
"TaskRun",
|
||||
|
||||
@@ -15,7 +15,14 @@ class Artist(Base):
|
||||
__tablename__ = "artist"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
# Display name: freely editable, NON-unique (two real creators can share a
|
||||
# name). Decoupled from identity/storage in migration 0077 (#130) — renaming
|
||||
# touches ONLY this. Was unique until then.
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
# Storage/identity key: IMMUTABLE + unique. This is the on-disk path
|
||||
# component (download_service artist_slug = artist.slug → images_root/<slug>/
|
||||
# <platform>/…), so it is set once at creation (collision-suffixed) and NEVER
|
||||
# changes — a rename must not move files. Existing artists keep their slug.
|
||||
slug: Mapped[str] = mapped_column(String(255), nullable=False, unique=True)
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Precomputed CCIP character prototypes (#1317, milestone 138).
|
||||
|
||||
The live matcher (ccip.match_image) needs each character's reference figure
|
||||
vectors. Building that on the request path reloaded EVERY figure CCIP vector in
|
||||
the library on any change (~4s, invalidated by every character accept). These
|
||||
tables make the references a PRECOMPUTED, INCREMENTAL artifact refreshed off the
|
||||
request path (services.ml.character_prototypes):
|
||||
|
||||
- CharacterPrototype: a character's reference vectors, capped to
|
||||
MLSettings.ccip_prototype_cap so MATCH cost doesn't grow with a character's
|
||||
popularity. The async matcher only READS these.
|
||||
- CcipPrototypeState: a per-character fingerprint (reference count + max region
|
||||
id) so a refresh rebuilds ONLY the characters whose references changed, and
|
||||
its updated_at lets the matcher's cache reload just the advanced characters.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
from .image_region import CCIP_DIM
|
||||
|
||||
|
||||
class CharacterPrototype(Base):
|
||||
__tablename__ = "character_prototype"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
# The character tag these vectors identify. CASCADE: deleting the tag drops
|
||||
# its prototypes.
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
# A reference figure/face CCIP vector (same space as
|
||||
# ImageRegion.ccip_embedding).
|
||||
ccip_embedding: Mapped[list[float]] = mapped_column(
|
||||
Vector(CCIP_DIM), nullable=False
|
||||
)
|
||||
# Provenance: the region this vector was copied from. SET NULL so pruning a
|
||||
# region doesn't delete the prototype mid-cycle (the next refresh reconciles).
|
||||
region_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
|
||||
|
||||
class CcipPrototypeState(Base):
|
||||
__tablename__ = "ccip_prototype_state"
|
||||
|
||||
# One row per character that currently has prototypes.
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
# count(reference regions) + max(region id) at last build — the cheap
|
||||
# per-character change detector that drives incremental rebuilds.
|
||||
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
# Bumped when this character's prototypes are rebuilt; the matcher cache
|
||||
# reloads only characters whose updated_at advanced.
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -92,6 +92,48 @@ class ImportSettings(Base):
|
||||
Boolean, nullable=False, default=True, server_default="true",
|
||||
)
|
||||
|
||||
# -- Post-text translation via the Interpreter LAN service (milestone 143).
|
||||
# Off by default with NO default host — it needs a reachable Interpreter
|
||||
# service (the operator's, behind a reverse proxy), which not every install
|
||||
# has; the operator sets the URL and flips it on. Empty base_url OR disabled
|
||||
# → the translate sweep no-ops.
|
||||
translation_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="false",
|
||||
)
|
||||
interpreter_base_url: Mapped[str] = mapped_column(
|
||||
Text, nullable=False, default="", server_default="",
|
||||
)
|
||||
translation_target_lang: Mapped[str] = mapped_column(
|
||||
Text, nullable=False, default="en", server_default="en",
|
||||
)
|
||||
# The latin-script acceptance floor for the translation gate: a translation
|
||||
# whose Interpreter-reported confidence is below this is kept as the original
|
||||
# (operator-tunable, milestone 155). Default 0.9 — stricter than the old
|
||||
# hardcoded 0.8, because Interpreter confidently mis-detects short ASCII
|
||||
# English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
|
||||
# trusted regardless (script-detected). Per-post overrides handle the misses.
|
||||
translation_min_confidence: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.9, server_default="0.9",
|
||||
)
|
||||
|
||||
# Title-based WIP auto-tagging (task #1458). When a freshly-imported post's
|
||||
# TITLE explicitly declares work-in-progress ("WIP" / "work in progress"),
|
||||
# the importer applies the `wip` system tag to its images — the artist's own
|
||||
# label, used to keep unfinished pieces out of the Explore/gallery browse. ON
|
||||
# by default (rule 26 — the feature works out of the box). Gates only the
|
||||
# LIVE import hook; the existing catalogue is caught by the operator-triggered
|
||||
# "Scan existing posts" backfill (which runs regardless of this flag).
|
||||
wip_title_tagging_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True, server_default="true",
|
||||
)
|
||||
# Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
|
||||
# a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
|
||||
# are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
|
||||
# precision tier is opt-in (the ring-loud audit surfaces false positives).
|
||||
wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="false",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def load(cls, session) -> ImportSettings:
|
||||
"""The singleton settings row (id=1), via an async session."""
|
||||
|
||||
@@ -63,7 +63,9 @@ class MLSettings(Base):
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=30
|
||||
# Support floor raised 30→50 (operator-asked 2026-07-06): a head needs
|
||||
# more human labels before it may fire without a human.
|
||||
Integer, nullable=False, default=50
|
||||
)
|
||||
# CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
|
||||
# over-fired (high-reference characters matched a scatter of images); 0.85
|
||||
@@ -78,7 +80,47 @@ class MLSettings(Base):
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
ccip_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.92
|
||||
# Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident
|
||||
# character matches auto-tag.
|
||||
Float, nullable=False, default=0.95
|
||||
)
|
||||
# -- Presentation chrome auto-hide (#141) -------------------------------
|
||||
# `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
|
||||
# with its OWN flat threshold (decoupled from content-head graduation) and is
|
||||
# HIDDEN from the gallery. Hiding is consequential so it runs HIGH. When an
|
||||
# image would be auto-hidden but ALSO scores >= presentation_conflict_threshold
|
||||
# on a content head, it's still hidden but flagged for review
|
||||
# (PresentationReview, mode='chrome') instead of buried silently. ON by default
|
||||
# (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
|
||||
# screenshot` are no longer chrome — they went to the PROCESS path below.
|
||||
presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
presentation_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.90
|
||||
)
|
||||
presentation_conflict_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
)
|
||||
# -- Process auto-apply (#1464) ----------------------------------------
|
||||
# `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
|
||||
# screenshots that must stay OUT of head/CCIP training but, unlike chrome,
|
||||
# remain VISIBLE in the gallery (operator 2026-07-12). They auto-apply on the
|
||||
# sweep with their OWN flat threshold and a PROVISIONAL source (`process_auto`,
|
||||
# in training_data._AUTO_SOURCES) so the head NEVER trains on its own output —
|
||||
# it learns only from title (`wip_title`) + manual labels, which breaks the
|
||||
# runaway loop. When a process tag would be applied but the image ALSO scores
|
||||
# >= process_conflict_threshold on a content head, it's flagged for review
|
||||
# (PresentationReview, mode='process') rather than silently marked. OFF by
|
||||
# default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
|
||||
process_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False
|
||||
)
|
||||
process_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.90
|
||||
)
|
||||
process_conflict_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
)
|
||||
# Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
|
||||
# existing libraries keep their stored value until the operator re-embeds.
|
||||
@@ -91,6 +133,82 @@ class MLSettings(Base):
|
||||
embedder_model_name: Mapped[str] = mapped_column(
|
||||
String(128), nullable=False, default="google/siglip2-so400m-patch16-512"
|
||||
)
|
||||
# -- Crop proposers / detectors (#1202, #134) --------------------------
|
||||
# WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config
|
||||
# lives HERE (DB) and is announced to the GPU agent in the lease — same as
|
||||
# the embedder model — so it is UI-tunable with NO restart, and the agent's
|
||||
# env is bootstrap-only. Each weights spec is an ultralytics builtin name,
|
||||
# an http(s) URL, or "hf_repo::file" (agent's _resolve). enabled off (or an
|
||||
# empty weights) skips that proposer. All ON by default (operator 2026-07-05)
|
||||
# so a fresh install crops out-of-the-box.
|
||||
# person: general COCO figure detector for Western/realistic art the anime
|
||||
# person-detector misses → NMS-merged with imgutils → CCIP + concept.
|
||||
detector_person_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
detector_person_weights: Mapped[str] = mapped_column(
|
||||
String(512), nullable=False, default="yolo11n.pt"
|
||||
)
|
||||
detector_person_conf: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.35
|
||||
)
|
||||
# anatomy: booru_yolo anime/furry/NSFW torso components → concept crops.
|
||||
# Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the
|
||||
# upstream repo so the URL resolves. License UNSTATED — fine for a private
|
||||
# homelab (operator accepted #1202).
|
||||
detector_anatomy_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
detector_anatomy_weights: Mapped[str] = mapped_column(
|
||||
String(512), nullable=False,
|
||||
default=(
|
||||
"https://github.com/aperveyev/booru_yolo/raw/main/models/"
|
||||
"yolov11m_aa22.pt"
|
||||
),
|
||||
)
|
||||
detector_anatomy_conf: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.30
|
||||
)
|
||||
# panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x).
|
||||
detector_panel_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
)
|
||||
detector_panel_weights: Mapped[str] = mapped_column(
|
||||
String(512), nullable=False,
|
||||
default="mosesb/best-comic-panel-detection::best.pt",
|
||||
)
|
||||
detector_panel_conf: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.30
|
||||
)
|
||||
# Per-frame caps bound the crop→embed explosion; max_regions is the hard
|
||||
# per-job backstop; dedupe_iou drops near-duplicate crops before the embed.
|
||||
detector_max_figures: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=8
|
||||
)
|
||||
detector_max_components: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=8
|
||||
)
|
||||
detector_max_panels: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=8
|
||||
)
|
||||
detector_max_regions: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=128
|
||||
)
|
||||
detector_dedupe_iou: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.85
|
||||
)
|
||||
# -- CCIP character prototypes (#1317) ---------------------------------
|
||||
# The per-character reference set is precomputed + refreshed INCREMENTALLY
|
||||
# (services.ml.character_prototypes) instead of rebuilt on the request path.
|
||||
# ccip_ref_signature is the cheap GLOBAL gate — when it's unchanged the
|
||||
# refresh no-ops; ccip_prototype_cap bounds the reference vectors kept per
|
||||
# character so MATCH cost doesn't grow with a character's popularity.
|
||||
ccip_ref_signature: Mapped[str | None] = mapped_column(
|
||||
String(128), nullable=True
|
||||
)
|
||||
ccip_prototype_cap: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=64
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""PixivFailedMedia — per-source dead-letter ledger of Pixiv media that keeps
|
||||
failing to download/validate.
|
||||
|
||||
Mirror of PatreonFailedMedia/SubscribeStarFailedMedia. Media that fails every
|
||||
walk (404'd pximg URL, deleted work, persistently-corrupt bytes) would
|
||||
otherwise re-error forever and re-burn backfill chunks. After ``attempts``
|
||||
reaches the dead-letter threshold the ingester skips it on routine
|
||||
tick/backfill walks (recovery still re-attempts). A later clean download
|
||||
clears the row.
|
||||
|
||||
`filehash` is the same synthesized ``<illust_id>:p<num>`` /
|
||||
``<illust_id>:ugoira`` key the seen-ledger uses. UNIQUE (source_id, filehash)
|
||||
is the upsert key.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PixivFailedMedia(Base):
|
||||
__tablename__ = "pixiv_failed_media"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_failed_media_source_id"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
last_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,42 @@
|
||||
"""PixivSeenMedia — per-source ledger of Pixiv media already
|
||||
downloaded+processed.
|
||||
|
||||
Mirror of PatreonSeenMedia/SubscribeStarSeenMedia for the Pixiv native
|
||||
ingester (replacing gallery-dl). One queryable row per (source, media) so
|
||||
routine walks skip media we've already ingested; recovery mode bypasses the
|
||||
ledger to re-walk.
|
||||
|
||||
Pixiv original URLs carry no content hash, so `filehash` is always the
|
||||
synthesized ``<illust_id>:p<num>`` (page) / ``<illust_id>:ugoira`` (frame
|
||||
zip) key — stable across any URL-shape drift. String(128) matches the sibling
|
||||
ledgers.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PixivSeenMedia(Base):
|
||||
__tablename__ = "pixiv_seen_media"
|
||||
__table_args__ = (
|
||||
# Dedup key the downloader upserts against: one ledger row per
|
||||
# (source, media). A second sighting of the same media is a no-op.
|
||||
UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_seen_media_source_id"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -8,7 +8,17 @@ artist-filter queries don't depend on the Source detour).
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -23,6 +33,10 @@ class Post(Base):
|
||||
# (created in alembic 0030) covers that case via
|
||||
# (artist_id, external_post_id).
|
||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||
CheckConstraint(
|
||||
"translation_override IN ('auto', 'force', 'original')",
|
||||
name="ck_post_translation_override",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
@@ -47,6 +61,34 @@ class Post(Base):
|
||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
attachment_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
# -- Post-text translation (milestone 143). Filled by the translate_posts
|
||||
# sweep via the Interpreter LAN service so viewing is instant.
|
||||
# translated_source_lang is the DETECTED original language; "en" (or a
|
||||
# passthrough) means nothing to translate and the *_translated columns stay
|
||||
# NULL. engine_version keys the Interpreter cache — re-runs are ~1ms and a
|
||||
# model upgrade re-translates instead of serving stale.
|
||||
post_title_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
description_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
translated_source_lang: Mapped[str | None] = mapped_column(
|
||||
String(8), nullable=True
|
||||
)
|
||||
translation_engine_version: Mapped[str | None] = mapped_column(
|
||||
String(128), nullable=True
|
||||
)
|
||||
translated_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
# Sticky per-post override of the translation decision (milestone 155):
|
||||
# 'auto' = the acceptance gate decides; 'force' = always store Interpreter's
|
||||
# translation even below the confidence floor (rescue a skipped legit-foreign
|
||||
# title); 'original' = never translate, keep the original (kill a confidently
|
||||
# mis-flagged one the floor can't catch). The sweep reads this on every run,
|
||||
# and re-translate leaves 'original' posts alone, so the choice survives a
|
||||
# Re-translate-all.
|
||||
translation_override: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="auto", server_default="auto",
|
||||
)
|
||||
|
||||
downloaded_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""PresentationReview — a system-tag the auto-apply sweep applied that ALSO looked
|
||||
like real content, flagged for operator review (milestone 141 + #1464).
|
||||
|
||||
When a sweep applies a system tag but the image ALSO scores highly on a content
|
||||
head, it still applies the tag but records this row so a review strip can surface
|
||||
it ("⚠ also looks like <conflict tag>"). Two modes (#1464): 'chrome' (banner —
|
||||
image is HIDDEN, review is keep-hidden / un-hide) and 'process' (wip / editor
|
||||
screenshot — image stays VISIBLE, review is confirm / remove-tag). Resolved rows
|
||||
are pruned by retention.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, ForeignKey, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class PresentationReview(Base):
|
||||
__tablename__ = "presentation_review"
|
||||
|
||||
image_record_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
# The presentation tag that was auto-applied (banner / editor screenshot).
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
# The content tag the image ALSO scored high on — the "concerning" signal.
|
||||
# SET NULL (not CASCADE): losing the conflict tag shouldn't erase the flag.
|
||||
conflict_tag_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
conflict_score: Mapped[float] = mapped_column(Float, nullable=False)
|
||||
# Which sweep flagged this (#1464): 'chrome' (banner, hidden) or 'process'
|
||||
# (wip / editor screenshot, shown). Drives which review strip surfaces it and
|
||||
# what "resolve" means (un-hide vs remove-tag). Existing rows backfill 'chrome'.
|
||||
mode: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="chrome", server_default="chrome"
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
# Set when the operator keeps-hidden or un-hides; retention prunes resolved.
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
@@ -10,6 +10,7 @@ from datetime import datetime
|
||||
from enum import StrEnum
|
||||
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Column,
|
||||
DateTime,
|
||||
@@ -17,6 +18,7 @@ from sqlalchemy import (
|
||||
Integer,
|
||||
String,
|
||||
Table,
|
||||
false,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy import (
|
||||
@@ -41,6 +43,21 @@ class TagKind(StrEnum):
|
||||
# to keep historic tag rows queryable.
|
||||
|
||||
|
||||
# The seeded system tags (migration 0075). Two behavior groups (#1464):
|
||||
# CHROME (banner): clusters on UI chrome, not content → HIDDEN from the default
|
||||
# gallery + from similarity; auto-applied via the sweep's chrome mode.
|
||||
# PROCESS (wip, editor screenshot): real-but-unfinished art / program screenshots
|
||||
# → SHOWN in the gallery (operator 2026-07-12), but excluded from the Explore
|
||||
# rabbit-hole; auto-applied via the sweep's process mode (provisional source,
|
||||
# ring-loud review guard).
|
||||
# ALL three are excluded from OTHER concepts' head/CCIP training (training-hygiene,
|
||||
# keyed on is_system); a system tag's OWN head trains on them — that's what makes
|
||||
# auto-flagging work.
|
||||
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
|
||||
CHROME_SYSTEM_TAGS = ("banner",)
|
||||
PROCESS_SYSTEM_TAGS = ("wip", "editor screenshot")
|
||||
WIP_SYSTEM_TAG = "wip"
|
||||
|
||||
image_tag = Table(
|
||||
"image_tag",
|
||||
Base.metadata,
|
||||
@@ -74,6 +91,14 @@ class Tag(Base):
|
||||
fandom_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
)
|
||||
# System tags ship with FC (wip / banner / editor screenshot, seeded in
|
||||
# migration 0075) and drive the training-hygiene exclusions: images
|
||||
# carrying one are excluded from OTHER concepts' head training and from
|
||||
# CCIP identity references. The mechanism keys on these exact rows, so
|
||||
# they're protected from rename/merge-away/re-fandom in TagService.
|
||||
is_system: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default=false()
|
||||
)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
|
||||
@@ -73,5 +73,12 @@ class TagHead(Base):
|
||||
trained_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
# Training-data fingerprint (positives + rejections) at last fit — the
|
||||
# incremental-retrain change detector (#1317 p2). A manual Retrain refits only
|
||||
# heads whose fingerprint moved; the nightly run ignores it (full reconcile).
|
||||
# NULL forces a refit (pre-fingerprint heads).
|
||||
train_fingerprint: Mapped[str | None] = mapped_column(
|
||||
String(128), nullable=True
|
||||
)
|
||||
# Extra detail (auto-apply operating point, F1, etc.) — non-load-bearing.
|
||||
metrics: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)
|
||||
|
||||
@@ -283,6 +283,23 @@ class ArtistService:
|
||||
await self.session.commit()
|
||||
return artist, created
|
||||
|
||||
async def rename(self, artist_id: int, name: str) -> Artist | None:
|
||||
"""Change the display NAME only (#130). The slug — and every on-disk path
|
||||
keyed off it — is IMMUTABLE, so a rename never moves files or risks a path
|
||||
collision. Name is free text and NON-unique (migration 0077). Returns the
|
||||
updated Artist, None if not found; raises ValueError on empty name."""
|
||||
cleaned = (name or "").strip()
|
||||
if not cleaned:
|
||||
raise ValueError("artist name must not be empty")
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == artist_id)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return None
|
||||
artist.name = cleaned
|
||||
await self.session.commit()
|
||||
return artist
|
||||
|
||||
async def autocomplete(self, prefix: str, limit: int = 20) -> list[Artist]:
|
||||
cleaned = (prefix or "").strip()
|
||||
if not cleaned:
|
||||
|
||||
@@ -17,7 +17,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
from sqlalchemy import and_, delete, func, or_, select, update
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from ..models import (
|
||||
@@ -203,6 +203,9 @@ def _unused_tag_conditions() -> list:
|
||||
Tag.id.not_in(used_via_series),
|
||||
Tag.id.not_in(used_via_chapter),
|
||||
Tag.id.not_in(used_via_fandom),
|
||||
# System tags (#128) ship with zero applications and must survive a
|
||||
# prune — the training-hygiene machinery keys on the rows.
|
||||
Tag.is_system.is_(False),
|
||||
]
|
||||
|
||||
|
||||
@@ -402,6 +405,8 @@ def delete_tag(session: Session, *, tag_id: int) -> dict:
|
||||
tag = session.get(Tag, tag_id)
|
||||
if tag is None:
|
||||
raise LookupError(f"tag id not found: {tag_id}")
|
||||
if tag.is_system:
|
||||
raise ValueError(f"'{tag.name}' is a system tag and cannot be deleted")
|
||||
associations_count = count_tag_associations(session, tag_id=tag_id)
|
||||
info = {"id": tag.id, "name": tag.name, "kind": tag.kind.value}
|
||||
session.delete(tag)
|
||||
@@ -731,7 +736,10 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
|
||||
heads retrain from whatever the operator re-tags. (The API route gates the
|
||||
live run behind a preview-derived confirm token for exactly this reason.)
|
||||
|
||||
PRESERVED: fandom + series tags and their series_page ordering. CASCADE on
|
||||
PRESERVED: fandom + series tags and their series_page ordering, AND the
|
||||
system hygiene tags (#128) WITH their applications — the reset re-tags
|
||||
CONTENT concepts, while wip/banner flags describe the file itself and
|
||||
re-flagging hundreds of banners by hand would be pure loss. CASCADE on
|
||||
image_tag / tag_alias / tag_suggestion_rejection clears each deleted tag's
|
||||
applications + metadata. Tag.fandom_id is SET NULL, so deleting character
|
||||
tags never touches the fandom rows. Irreversible except via DB backup
|
||||
@@ -744,7 +752,9 @@ def reset_content_tagging(session: Session, *, dry_run: bool = False) -> dict:
|
||||
"sample_names": [first 50],
|
||||
and on live runs "deleted": total}
|
||||
"""
|
||||
predicate = Tag.kind.in_(RESETTABLE_TAG_KINDS)
|
||||
predicate = and_(
|
||||
Tag.kind.in_(RESETTABLE_TAG_KINDS), Tag.is_system.is_(False)
|
||||
)
|
||||
rows = session.execute(
|
||||
select(Tag.id, Tag.name, Tag.kind).where(predicate)
|
||||
).all()
|
||||
|
||||
@@ -24,15 +24,17 @@ import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .native_ingest_common import NativeIngestError
|
||||
from .patreon_ingester import PatreonIngester
|
||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||
from .pixiv_client import user_id_from_url
|
||||
from .pixiv_ingester import PixivIngester
|
||||
from .subscribestar_ingester import SubscribeStarIngester
|
||||
|
||||
# Platforms whose download + verify go through the native ingester rather than
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord, pixiv,
|
||||
# deviantart) until they migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord,
|
||||
# deviantart — the latter slated for retirement, not migration) until they
|
||||
# migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "pixiv"})
|
||||
|
||||
# Mirrors patreon_resolver._CAMPAIGNS_URL — surfaced in resolution-failure
|
||||
# messages so the operator sees the exact lookup endpoint that was hit.
|
||||
@@ -46,6 +48,7 @@ def _native_ingester_cls(platform: str):
|
||||
dispatch pick up the replacement."""
|
||||
return {
|
||||
"patreon": PatreonIngester,
|
||||
"pixiv": PixivIngester,
|
||||
"subscribestar": SubscribeStarIngester,
|
||||
}[platform]
|
||||
|
||||
@@ -98,14 +101,34 @@ async def _resolve_native_campaign_id(
|
||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
||||
feed id IS the creator URL; Pixiv's is the numeric user id parsed straight
|
||||
from it (no lookup → resolved None either way). Patreon resolves the
|
||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
||||
so phase 3 caches it)."""
|
||||
if platform == "subscribestar":
|
||||
return url, None
|
||||
if platform == "pixiv":
|
||||
return user_id_from_url(url), None
|
||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||
|
||||
|
||||
def _campaign_resolution_error(platform: str, url: str) -> str:
|
||||
"""Operator-facing message for a native source whose campaign id could not
|
||||
be resolved — names the platform's own lookup mechanism."""
|
||||
if platform == "pixiv":
|
||||
return (
|
||||
f"Could not extract a pixiv user id. source_url={url!r} — expected "
|
||||
"a URL like https://www.pixiv.net/users/<id>."
|
||||
)
|
||||
vanity = extract_vanity(url)
|
||||
return (
|
||||
f"Could not resolve Patreon campaign id. source_url={url!r}; "
|
||||
f"vanity={vanity!r}; "
|
||||
f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
||||
"(vanity lookup failed — cookies expired or creator moved?)"
|
||||
)
|
||||
|
||||
|
||||
async def _run_native_ingester(
|
||||
ctx: dict, source_config, mode: str | None, gdl, sync_session_factory,
|
||||
) -> tuple[DownloadResult, str | None]:
|
||||
@@ -123,9 +146,9 @@ async def _run_native_ingester(
|
||||
platform, ctx["url"], ctx["cookies_path"], overrides
|
||||
)
|
||||
if not campaign_id:
|
||||
# Only reachable for Patreon (SubscribeStar's campaign id is the URL).
|
||||
# Patreon: vanity lookup failed. Pixiv: no numeric user id in the URL.
|
||||
# (SubscribeStar's campaign id is the URL itself — never lands here.)
|
||||
url = ctx["url"]
|
||||
vanity = extract_vanity(url)
|
||||
return (
|
||||
DownloadResult(
|
||||
success=False,
|
||||
@@ -133,12 +156,7 @@ async def _run_native_ingester(
|
||||
artist_slug=ctx["artist_slug"],
|
||||
platform=platform,
|
||||
error_type=ErrorType.NOT_FOUND,
|
||||
error_message=(
|
||||
f"Could not resolve Patreon campaign id. source_url={url!r}; "
|
||||
f"vanity={vanity!r}; "
|
||||
f"lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
||||
"(vanity lookup failed — cookies expired or creator moved?)"
|
||||
),
|
||||
error_message=_campaign_resolution_error(platform, url),
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -161,6 +179,10 @@ async def _run_native_ingester(
|
||||
validate=gdl._validate_files,
|
||||
rate_limit=rate_limit,
|
||||
request_sleep=request_sleep,
|
||||
# Uniform across adapters: token platforms (pixiv) authenticate with
|
||||
# it, cookie platforms accept-and-ignore — so this construction stays
|
||||
# platform-agnostic.
|
||||
auth_token=ctx["auth_token"],
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
dl_result = await loop.run_in_executor(
|
||||
@@ -181,54 +203,6 @@ async def _run_native_ingester(
|
||||
return dl_result, resolved_campaign_id
|
||||
|
||||
|
||||
async def preview_source(
|
||||
*,
|
||||
platform: str,
|
||||
url: str,
|
||||
source_id: int,
|
||||
config_overrides: dict | None,
|
||||
cookies_path: str | None,
|
||||
images_root: Path,
|
||||
sync_session_factory,
|
||||
page_limit: int = 3,
|
||||
) -> dict:
|
||||
"""Dry-run preview for a native platform (plan #708 B4): resolve the campaign
|
||||
id, then walk a few pages counting media not already seen/dead — no download.
|
||||
|
||||
Returns the preview dict (total_new / posts_scanned / pages_scanned /
|
||||
has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
|
||||
Native-only — the caller gates on `uses_native_ingester`.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
campaign_id, _ = await _resolve_native_campaign_id(
|
||||
platform, url, cookies_path, config_overrides or {}
|
||||
)
|
||||
if not campaign_id:
|
||||
vanity = extract_vanity(url)
|
||||
return {
|
||||
"error": (
|
||||
f"Couldn't resolve the campaign id. source_url={url!r}; "
|
||||
f"vanity={vanity!r}; lookup=GET {_CAMPAIGNS_API}?filter[vanity]={vanity or ''} "
|
||||
"(cookies expired, or the creator moved/renamed?)."
|
||||
)
|
||||
}
|
||||
ingester = _native_ingester_cls(platform)(
|
||||
images_root=images_root,
|
||||
cookies_path=cookies_path,
|
||||
session_factory=sync_session_factory,
|
||||
)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
result = await loop.run_in_executor(
|
||||
None,
|
||||
lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
|
||||
)
|
||||
except NativeIngestError as exc:
|
||||
return {"error": f"Couldn't preview: {exc}"}
|
||||
return result
|
||||
|
||||
|
||||
async def verify_source_credential(
|
||||
*,
|
||||
platform: str,
|
||||
@@ -245,13 +219,18 @@ async def verify_source_credential(
|
||||
this and render the result.
|
||||
"""
|
||||
if uses_native_ingester(platform):
|
||||
# Native ingester platforms verify via their own lightweight auth probe
|
||||
# (one authenticated feed fetch). SubscribeStar's probe takes the creator
|
||||
# URL directly; Patreon's resolves the campaign id first.
|
||||
# Native ingester platforms verify via their own lightweight auth probe.
|
||||
# SubscribeStar's probe takes the creator URL directly; Patreon's
|
||||
# resolves the campaign id first; Pixiv's is one OAuth refresh (the
|
||||
# exact call that fails when the token is bad — no feed walk).
|
||||
if platform == "subscribestar":
|
||||
from .subscribestar_ingester import verify_subscribestar_credential
|
||||
|
||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||
if platform == "pixiv":
|
||||
from .pixiv_ingester import verify_pixiv_credential
|
||||
|
||||
return await verify_pixiv_credential(auth_token)
|
||||
from .patreon_ingester import verify_patreon_credential
|
||||
|
||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||
|
||||
@@ -418,7 +418,8 @@ class DownloadService:
|
||||
# the duplicate file). Empty outside recapture mode.
|
||||
relink_pairs = getattr(dl_result, "relink_source_paths", None) or []
|
||||
relinked = 0
|
||||
for rel_str, rel_url in relink_pairs:
|
||||
post_linked = 0
|
||||
for rel_str, rel_url, rel_post_id in relink_pairs:
|
||||
rel_path = Path(rel_str)
|
||||
if not rel_path.exists(): # noqa: ASYNC240
|
||||
continue
|
||||
@@ -428,13 +429,27 @@ class DownloadService:
|
||||
|
||||
if await loop.run_in_executor(None, _relink):
|
||||
relinked += 1
|
||||
|
||||
# #1288: link the on-disk image to its Post. Recapture disk-skips the
|
||||
# media (never re-imported), so a pre-existing image (e.g. one pulled
|
||||
# under the old gallery-dl path) otherwise stays orphaned even after
|
||||
# the post record is written. Idempotent for already-linked images.
|
||||
def _link(p=rel_path, pid=rel_post_id):
|
||||
return self.importer.link_existing_image_to_post(
|
||||
p, pid, source=source_row, artist=artist,
|
||||
)
|
||||
|
||||
if await loop.run_in_executor(None, _link):
|
||||
post_linked += 1
|
||||
if relink_pairs:
|
||||
# recapture diagnostic: how many on-disk images got their
|
||||
# source_filehash backfilled (inline-image localization). < total is
|
||||
# normal — files already carrying a filehash are skipped (NULL-only).
|
||||
# source_filehash backfilled (inline-image localization) and how many
|
||||
# got (re)linked to their Post. < total for source_filehash is normal
|
||||
# (files already carrying a filehash are skipped, NULL-only).
|
||||
log.info(
|
||||
"recap: relinked source_filehash on %d/%d on-disk image(s)",
|
||||
relinked, len(relink_pairs),
|
||||
"recap: relinked source_filehash on %d and linked %d/%d on-disk "
|
||||
"image(s) to their post",
|
||||
relinked, post_linked, len(relink_pairs),
|
||||
)
|
||||
|
||||
# Kick the off-platform file-host downloader for any links this run
|
||||
|
||||
@@ -8,6 +8,7 @@ and returns a JSON-shaped dict for the API layer.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
@@ -18,6 +19,8 @@ from ..utils.slug import slugify
|
||||
from .db_helpers import get_or_create
|
||||
from .source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
"""URL didn't match any platform pattern."""
|
||||
@@ -32,9 +35,14 @@ class InvalidUrlError(Exception):
|
||||
# reviewers catch drift.
|
||||
_PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("patreon", re.compile(
|
||||
# Three creator URL shapes — bare (patreon.com/Atole), `c/`, and `cw/`
|
||||
# (the "creator workspace" URL served once subscribed, see
|
||||
# patreon_resolver._VANITY_RE). A trailing sub-path is allowed so a
|
||||
# creator's inner page still derives the slug. Nav pages stay excluded.
|
||||
r"^https?://(?:www\.)?patreon\.com/"
|
||||
r"(?!home$|search\b|messages\b|notifications\b|library\b|settings\b|posts\b|c/)"
|
||||
r"(?P<slug>[^/?#]+)/?$",
|
||||
r"(?:cw/|c/)?"
|
||||
r"(?!(?:home|search|messages|notifications|library|settings|posts)(?:[/?#]|$))"
|
||||
r"(?P<slug>[^/?#]+)",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
("subscribestar", re.compile(
|
||||
@@ -61,15 +69,38 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
|
||||
|
||||
class ExtensionService:
|
||||
def __init__(self, session: AsyncSession) -> None:
|
||||
def __init__(self, session: AsyncSession, crypto=None) -> None:
|
||||
self.session = session
|
||||
# Optional decryptor for resolving a token-auth platform's display name
|
||||
# (pixiv) at add-time. None → skip resolution, fall back to the handle.
|
||||
self._crypto = crypto
|
||||
|
||||
async def quick_add_source(self, url: str) -> dict:
|
||||
platform, raw_slug = self._derive(url)
|
||||
artist, created_artist = await self._find_or_create_artist(raw_slug)
|
||||
# Identity by SOURCE handle (#130): an existing (platform, url) source
|
||||
# keeps its artist on re-add — even if that artist was since renamed (its
|
||||
# frozen slug no longer matches the current name). Only a genuinely new
|
||||
# source resolves/creates an artist.
|
||||
existing = (await self.session.execute(
|
||||
select(Source).where(Source.platform == platform, Source.url == url)
|
||||
)).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == existing.artist_id)
|
||||
)).scalar_one()
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
# New source → name the artist properly by resolving the real display
|
||||
# name from the platform (falls back to the URL handle).
|
||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||
artist, created_artist = await self._find_or_create_artist(name)
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
return self._shape(source, artist, created_source, created_artist)
|
||||
|
||||
@staticmethod
|
||||
def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
|
||||
return {
|
||||
"source": {
|
||||
"id": source.id,
|
||||
@@ -87,6 +118,50 @@ class ExtensionService:
|
||||
"created_artist": created_artist,
|
||||
}
|
||||
|
||||
async def _resolve_artist_name(
|
||||
self, platform: str, raw_slug: str, url: str
|
||||
) -> str:
|
||||
"""The real display name for a new artist, resolved from the platform at
|
||||
add-time (#130). Our native platforms each have a name source — pixiv the
|
||||
app API (token), patreon the campaigns API, subscribestar the profile
|
||||
page (both cookies). Other platforms (and any failure — no credential,
|
||||
network error) fall back to the URL handle, which is already readable.
|
||||
The resolvers are sync, so they run in an executor."""
|
||||
if self._crypto is None or platform not in ("pixiv", "patreon", "subscribestar"):
|
||||
return raw_slug
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
cred = CredentialService(self.session, self._crypto)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
if platform == "pixiv":
|
||||
token = await cred.get_token("pixiv")
|
||||
if not token:
|
||||
return raw_slug
|
||||
from .pixiv_client import PixivClient
|
||||
name = await loop.run_in_executor(
|
||||
None, PixivClient(token).resolve_display_name, raw_slug
|
||||
)
|
||||
elif platform == "patreon":
|
||||
cookies = await cred.get_cookies_path("patreon")
|
||||
from .patreon_resolver import resolve_display_name
|
||||
name = await loop.run_in_executor(
|
||||
None, resolve_display_name, raw_slug,
|
||||
str(cookies) if cookies else None,
|
||||
)
|
||||
else: # subscribestar
|
||||
cookies = await cred.get_cookies_path("subscribestar")
|
||||
from .subscribestar_client import SubscribeStarClient
|
||||
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||
name = await loop.run_in_executor(
|
||||
None, client.resolve_display_name, url
|
||||
)
|
||||
except Exception as exc: # resolution is best-effort — never block the add
|
||||
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||
return raw_slug
|
||||
return name or raw_slug
|
||||
|
||||
async def probe(self, url: str) -> dict:
|
||||
"""Read-only resolution of a creator-page URL against the FC DB.
|
||||
Returns one of:
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Surgical re-fetch of a post's external file-host links.
|
||||
|
||||
The normal download cadence never re-walks deep back-catalogue posts (the
|
||||
seen-gates exist precisely to keep old items from resurfacing), so when a
|
||||
file that CAME from an ExternalLink is deleted — e.g. the failure-triage
|
||||
recovery flow removing a corrupt original — a plain source re-check will
|
||||
never bring it back. The link ROW is the durable, per-post handle: resetting
|
||||
it to pending and dispatching the fetch re-downloads exactly that link's
|
||||
payload, and sha-dedupe at import discards anything that still exists — so
|
||||
only the missing file actually lands. (Operator 2026-07-03: recovery must
|
||||
not require artist-wide deep scans.)
|
||||
"""
|
||||
import logging
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import ExternalLink
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def refetch_links_for_post(session: Session, post_id: int) -> dict:
|
||||
"""Reset every settled ExternalLink on a post to pending (fresh attempt
|
||||
budget) and dispatch its fetch. In-flight ('downloading') links are left
|
||||
alone. Commits. Returns {links_total, links_reset}."""
|
||||
links = session.execute(
|
||||
select(ExternalLink).where(ExternalLink.post_id == post_id)
|
||||
).scalars().all()
|
||||
reset_ids = []
|
||||
for link in links:
|
||||
if link.status == "downloading":
|
||||
continue
|
||||
link.status = "pending"
|
||||
link.attempts = 0
|
||||
link.last_error = None
|
||||
link.completed_at = None
|
||||
reset_ids.append(link.id)
|
||||
session.commit()
|
||||
if reset_ids:
|
||||
# Lazy import (services -> tasks would cycle at module load). The
|
||||
# 10-min extdl sweep would pick pending rows up anyway — dispatching
|
||||
# directly just skips the wait.
|
||||
from ..tasks.external import fetch_external_link
|
||||
|
||||
for lid in reset_ids:
|
||||
fetch_external_link.delay(lid)
|
||||
log.info(
|
||||
"external refetch: post %s — %d/%d link(s) reset + dispatched",
|
||||
post_id, len(reset_ids), len(links),
|
||||
)
|
||||
return {"links_total": len(links), "links_reset": len(reset_ids)}
|
||||
@@ -157,7 +157,7 @@ class DownloadResult:
|
||||
# pairs for already-present media whose ImageRecord.source_filehash should be
|
||||
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
||||
# on the gallery-dl path and outside recapture.
|
||||
relink_source_paths: list[tuple[str, str]] = field(default_factory=list)
|
||||
relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
return_code: int = 0
|
||||
@@ -298,8 +298,9 @@ class GalleryDLService:
|
||||
# removed at the plan-#697 cutover — it now uses the native ingester
|
||||
# (services/patreon_ingester.py), not gallery-dl.
|
||||
PLATFORM_DEFAULTS = {
|
||||
# subscribestar removed — it's a native-ingester platform now (#71); the
|
||||
# remaining entries are the gallery-dl platforms not yet migrated.
|
||||
# subscribestar removed — native-ingester platform now (#71); pixiv
|
||||
# removed likewise (#129). The remaining entries are the gallery-dl
|
||||
# platforms not yet migrated.
|
||||
"hentaifoundry": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
@@ -315,12 +316,6 @@ class GalleryDLService:
|
||||
"reactions": False,
|
||||
"threads": True,
|
||||
},
|
||||
"pixiv": {
|
||||
"content_types": ["all"],
|
||||
"directory": ["{category}"],
|
||||
"filename": "{id}_{title[:50]}_{num:>02}.{extension}",
|
||||
"ugoira": True,
|
||||
},
|
||||
"deviantart": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
@@ -694,9 +689,6 @@ class GalleryDLService:
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})
|
||||
config["extractor"]["discord"]["token"] = auth_token
|
||||
if auth_token and platform == "pixiv":
|
||||
config["extractor"].setdefault("pixiv", {})
|
||||
config["extractor"]["pixiv"]["refresh-token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
@@ -882,8 +874,6 @@ class GalleryDLService:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})["token"] = auth_token
|
||||
if auth_token and platform == "pixiv":
|
||||
config["extractor"].setdefault("pixiv", {})["refresh-token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
|
||||
@@ -22,8 +22,16 @@ from sqlalchemy import Select, and_, distinct, exists, func, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
|
||||
from ..models.tag import image_tag
|
||||
from ..models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
Post,
|
||||
Source,
|
||||
Tag,
|
||||
TagPositiveConfirmation,
|
||||
)
|
||||
from ..models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
|
||||
from .pagination import decode_cursor, encode_cursor
|
||||
from .tag_query import (
|
||||
fandom_join_alias,
|
||||
@@ -179,7 +187,7 @@ def _apply_scope(
|
||||
stmt, *, tag_ids, post_id, artist_id, media_type,
|
||||
tag_or_groups=None, tag_exclude=None,
|
||||
platform=None, untagged=False, no_artist=False,
|
||||
date_from=None, date_to=None,
|
||||
date_from=None, date_to=None, hidden_tag_ids=None,
|
||||
):
|
||||
"""Apply the composable gallery filters to a statement.
|
||||
|
||||
@@ -216,6 +224,12 @@ def _apply_scope(
|
||||
stmt = stmt.where(image_in_any_tag_scope(group))
|
||||
if tag_exclude:
|
||||
stmt = stmt.where(~image_in_any_tag_scope(tag_exclude))
|
||||
# Presentation chrome (banner / editor screenshot) is hidden from the default
|
||||
# gallery — an implicit exclude the caller supplies unless the operator asked
|
||||
# to include hidden or is explicitly filtering for a presentation tag
|
||||
# (milestone 141). `wip` is NOT hidden. Resolved to ids by _hidden_tag_ids.
|
||||
if hidden_tag_ids:
|
||||
stmt = stmt.where(~image_in_any_tag_scope(hidden_tag_ids))
|
||||
prov = _provenance_clause(post_id, artist_id)
|
||||
if prov is not None:
|
||||
stmt = stmt.where(prov)
|
||||
@@ -382,6 +396,25 @@ def _diversify_similar(src, rows, limit, *, dup_threshold=8, lam=0.40):
|
||||
return [kept[i] for i in order]
|
||||
|
||||
|
||||
def _reach_sample(rows, limit, reach):
|
||||
"""From a distance-sorted candidate pool (nearest first), pick a spread of ranks
|
||||
that MIXES near (tag the current cluster) and mid-far (escape it) BEFORE dedup +
|
||||
MMR — the Explore "reach" dial (#1476).
|
||||
|
||||
reach in (0, 1]: the sampled span grows outward from the anchor (0.25→1.0 of the
|
||||
pool), evenly strided from rank 0 so the nearest are still represented. In a
|
||||
dense signature the nearest ranks are near-identical, so reaching farther is the
|
||||
only way to hand MMR genuinely different content — MMR alone can't escape a pool
|
||||
that's already all-near. reach<=0 or a small pool passes through unchanged."""
|
||||
n = len(rows)
|
||||
want = max(limit * 8, 100)
|
||||
if reach <= 0 or n <= want:
|
||||
return rows
|
||||
span = int(min(1.0, 0.25 + 0.75 * reach) * n)
|
||||
idx = sorted({min(int(i * span / want), n - 1) for i in range(want)})
|
||||
return [rows[i] for i in idx]
|
||||
|
||||
|
||||
async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
|
||||
"""Map image_id -> {"name","slug"} via the canonical
|
||||
image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
|
||||
@@ -402,6 +435,32 @@ class GalleryService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
|
||||
async def _hidden_tag_ids(
|
||||
self, include_hidden, tag_ids, tag_or_groups,
|
||||
) -> list[int] | None:
|
||||
"""Chrome (banner) tag ids to implicitly exclude from a gallery query, or
|
||||
None. None when the caller asked to include hidden, when the operator is
|
||||
explicitly filtering FOR a chrome tag (they clearly want to see it), or when
|
||||
no chrome tags exist. (milestone 141; #1464: editor screenshot is now PROCESS
|
||||
— shown — so only `banner` hides here.)"""
|
||||
if include_hidden:
|
||||
return None
|
||||
rows = await self.session.execute(
|
||||
select(Tag.id).where(
|
||||
Tag.is_system.is_(True),
|
||||
Tag.name.in_(CHROME_SYSTEM_TAGS),
|
||||
)
|
||||
)
|
||||
pres = [r[0] for r in rows]
|
||||
if not pres:
|
||||
return None
|
||||
explicit = set(tag_ids or [])
|
||||
for group in tag_or_groups or []:
|
||||
explicit.update(group)
|
||||
if explicit & set(pres):
|
||||
return None
|
||||
return pres
|
||||
|
||||
async def scroll(
|
||||
self,
|
||||
cursor: str | None,
|
||||
@@ -418,12 +477,16 @@ class GalleryService:
|
||||
no_artist: bool = False,
|
||||
date_from: datetime | None = None,
|
||||
date_to: datetime | None = None,
|
||||
include_hidden: bool = False,
|
||||
) -> GalleryPage:
|
||||
if limit < 1 or limit > 200:
|
||||
raise ValueError("limit must be between 1 and 200")
|
||||
_require_single_filter(
|
||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||
)
|
||||
hidden = await self._hidden_tag_ids(
|
||||
include_hidden, tag_ids, tag_or_groups,
|
||||
)
|
||||
|
||||
# eff is the ACTIVE sort column (effective_date or earliest_post_date);
|
||||
# the cursor, ordering and year/month grouping all key off it, so the
|
||||
@@ -436,7 +499,7 @@ class GalleryService:
|
||||
artist_id=artist_id, media_type=media_type,
|
||||
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
||||
platform=platform, untagged=untagged, no_artist=no_artist,
|
||||
date_from=date_from, date_to=date_to,
|
||||
date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
|
||||
)
|
||||
|
||||
descending = sort not in _ASCENDING_SORTS
|
||||
@@ -489,6 +552,7 @@ class GalleryService:
|
||||
no_artist: bool = False,
|
||||
date_from: datetime | None = None,
|
||||
date_to: datetime | None = None,
|
||||
include_hidden: bool = False,
|
||||
) -> list[TimelineBucket]:
|
||||
eff = _effective_date_col()
|
||||
year_col = func.date_part("year", eff).label("yr")
|
||||
@@ -500,12 +564,15 @@ class GalleryService:
|
||||
_require_single_filter(
|
||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||
)
|
||||
hidden = await self._hidden_tag_ids(
|
||||
include_hidden, tag_ids, tag_or_groups,
|
||||
)
|
||||
stmt = _apply_scope(
|
||||
stmt, tag_ids=tag_ids, post_id=post_id,
|
||||
artist_id=artist_id, media_type=media_type,
|
||||
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
||||
platform=platform, untagged=untagged, no_artist=no_artist,
|
||||
date_from=date_from, date_to=date_to,
|
||||
date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
|
||||
)
|
||||
stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
@@ -519,7 +586,7 @@ class GalleryService:
|
||||
tag_exclude: list[int] | None = None,
|
||||
platform: str | None = None, untagged: bool = False,
|
||||
no_artist: bool = False, date_from: datetime | None = None,
|
||||
date_to: datetime | None = None,
|
||||
date_to: datetime | None = None, include_hidden: bool = False,
|
||||
) -> str | None:
|
||||
"""Returns a cursor that, when passed to scroll() with the same sort,
|
||||
positions at the first image of the given year-month. None if the
|
||||
@@ -536,12 +603,15 @@ class GalleryService:
|
||||
_require_single_filter(
|
||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||
)
|
||||
hidden = await self._hidden_tag_ids(
|
||||
include_hidden, tag_ids, tag_or_groups,
|
||||
)
|
||||
stmt = _apply_scope(
|
||||
stmt, tag_ids=tag_ids, post_id=post_id,
|
||||
artist_id=artist_id, media_type=media_type,
|
||||
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
||||
platform=platform, untagged=untagged, no_artist=no_artist,
|
||||
date_from=date_from, date_to=date_to,
|
||||
date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
|
||||
)
|
||||
descending = sort != "oldest"
|
||||
if descending:
|
||||
@@ -566,6 +636,7 @@ class GalleryService:
|
||||
platform: str | None = None,
|
||||
untagged: bool = False, no_artist: bool = False,
|
||||
date_from: datetime | None = None, date_to: datetime | None = None,
|
||||
include_hidden: bool = False,
|
||||
) -> GalleryFacets:
|
||||
"""Live facet counts scoped to the current filter. Each facet GROUP is
|
||||
computed with all OTHER active filters applied but its OWN selection
|
||||
@@ -576,10 +647,14 @@ class GalleryService:
|
||||
_require_single_filter(
|
||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||
)
|
||||
hidden = await self._hidden_tag_ids(
|
||||
include_hidden, tag_ids, tag_or_groups,
|
||||
)
|
||||
common = {
|
||||
"tag_ids": tag_ids, "post_id": post_id,
|
||||
"artist_id": artist_id, "media_type": media_type,
|
||||
"tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
|
||||
"hidden_tag_ids": hidden,
|
||||
}
|
||||
|
||||
# total — the full active filter (the headline result count).
|
||||
@@ -660,6 +735,8 @@ class GalleryService:
|
||||
platform: str | None = None,
|
||||
untagged: bool = False, no_artist: bool = False,
|
||||
date_from: datetime | None = None, date_to: datetime | None = None,
|
||||
exclude_wip: bool = False,
|
||||
reach: float = 0.0, exclude_ids: list[int] | None = None,
|
||||
) -> list[GalleryImage] | None:
|
||||
"""Visual "more like this": images near `image_id`'s SigLIP embedding
|
||||
(pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
|
||||
@@ -688,15 +765,44 @@ class GalleryService:
|
||||
# wide pool there's nothing but the near-dupes to choose from. Widened
|
||||
# (5×→8×, cap 200→400) so the stronger MMR has genuinely distinct
|
||||
# neighbourhoods to reach into for more variance (operator, 2026-07-01).
|
||||
pool_n = min(400, max(limit * 8, 100))
|
||||
# Explore's reach>0 (#1476) widens it a LOT more: in a dense signature the
|
||||
# nearest few hundred are all near-identical, so far-enough candidates only
|
||||
# exist deeper in the ranked pool. _reach_sample then strides across them.
|
||||
if reach > 0:
|
||||
pool_n = min(1000, max(limit * 25, 100))
|
||||
else:
|
||||
pool_n = min(400, max(limit * 8, 100))
|
||||
distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding)
|
||||
eff = _effective_date_col()
|
||||
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
|
||||
stmt = _outer_join_primary_post(stmt)
|
||||
# Chrome (banner, #128) clusters on UI rather than content, so near any one
|
||||
# of them they'd fill the grid → excluded from CANDIDATES always (the anchor
|
||||
# itself may be a banner). PROCESS art (wip / editor screenshot) stays
|
||||
# surfaced here by default (real content; only the training pipelines exclude
|
||||
# it), but the Explore rabbit-hole passes exclude_wip to also drop the whole
|
||||
# process group so a browse doesn't keep surfacing work-in-progress
|
||||
# (operator, 2026-07-08; #1464 — editor now rides with wip here).
|
||||
excluded_system_tags = CHROME_SYSTEM_TAGS
|
||||
if exclude_wip:
|
||||
excluded_system_tags = (*CHROME_SYSTEM_TAGS, *PROCESS_SYSTEM_TAGS)
|
||||
presentation = (
|
||||
select(image_tag.c.image_record_id)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(
|
||||
Tag.is_system.is_(True),
|
||||
Tag.name.in_(excluded_system_tags),
|
||||
)
|
||||
)
|
||||
stmt = stmt.where(
|
||||
ImageRecord.siglip_embedding.is_not(None),
|
||||
ImageRecord.id != image_id,
|
||||
ImageRecord.id.not_in(presentation),
|
||||
)
|
||||
# Anti-revisit (#1476): the Explore walk passes its breadcrumb so already-
|
||||
# walked images aren't re-served as neighbours — → can't loop you back in.
|
||||
if exclude_ids:
|
||||
stmt = stmt.where(ImageRecord.id.not_in(exclude_ids))
|
||||
stmt = _apply_scope(
|
||||
stmt, tag_ids=tag_ids, post_id=None,
|
||||
artist_id=artist_id, media_type=media_type,
|
||||
@@ -706,6 +812,10 @@ class GalleryService:
|
||||
)
|
||||
stmt = stmt.order_by(distance.asc()).limit(pool_n)
|
||||
rows = (await self.session.execute(stmt)).all()
|
||||
# Explore reach: stride across an outward-growing distance span so the pool
|
||||
# handed to MMR spans near→mid-far, not just the tight cluster (#1476).
|
||||
if reach > 0:
|
||||
rows = _reach_sample(rows, limit, reach)
|
||||
rows = _diversify_similar(src, rows, limit)
|
||||
artists = await _artists_for(self.session, [r[0].id for r in rows])
|
||||
return _gallery_images(rows, artists)
|
||||
@@ -717,8 +827,14 @@ class GalleryService:
|
||||
# Self-join Tag to resolve a character's fandom NAME (not just id) so the
|
||||
# modal chip can label it without an N+1 (shared tag_query helpers).
|
||||
fandom_alias = fandom_join_alias()
|
||||
# source drives the auto-applied badge; confirmed = operator affirmed the
|
||||
# tag (positive + retraction-shielded, milestone 139).
|
||||
confirmed = exists().where(
|
||||
TagPositiveConfirmation.image_record_id == image_id,
|
||||
TagPositiveConfirmation.tag_id == Tag.id,
|
||||
).label("confirmed")
|
||||
tag_stmt = (
|
||||
select(*tag_columns(fandom_alias))
|
||||
select(*tag_columns(fandom_alias), image_tag.c.source, confirmed)
|
||||
.select_from(
|
||||
Tag.__table__
|
||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||
|
||||
@@ -47,9 +47,21 @@ from .attachment_store import AttachmentStore
|
||||
from .audits import single_color
|
||||
from .link_extract import extract_external_links
|
||||
from .thumbnailer import Thumbnailer
|
||||
from .wip_title import (
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
WIP_TITLE_SOURCE,
|
||||
apply_wip_image_tags,
|
||||
matches_soft_wip_title,
|
||||
matches_wip_title,
|
||||
resolve_wip_tag_id,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Sentinel for the lazily-resolved wip tag id (distinguishes "not resolved yet"
|
||||
# from a genuine None = tag absent, so absence is cached and not re-queried).
|
||||
_UNSET = object()
|
||||
|
||||
|
||||
class SkipReason(StrEnum):
|
||||
too_small = "too_small"
|
||||
@@ -183,6 +195,10 @@ class Importer:
|
||||
# invalidated mid-Importer (Importer instances are per-task /
|
||||
# per-archive-import so cross-instance staleness is harmless).
|
||||
self._phash_candidates: list[tuple] | None = None
|
||||
# Lazily-resolved `wip` system tag id for title-based WIP auto-tagging
|
||||
# (task #1458). Sentinel _UNSET so a genuine None (tag absent) is cached
|
||||
# and not re-queried per media. Importer is per-task, so this can't stale.
|
||||
self._wip_tag_id: int | None = _UNSET
|
||||
|
||||
def _phash_candidates_cache(self) -> list[tuple]:
|
||||
"""Cached `(phash, width, height, id)` rows from image_record.
|
||||
@@ -933,6 +949,10 @@ class Importer:
|
||||
# Thumbnail is queued separately by the calling task; the importer
|
||||
# does not generate thumbnails inline so the import queue stays moving.
|
||||
|
||||
# Title-based WIP auto-tag (task #1458): fresh import only, after the
|
||||
# sidecar has linked the post so record.primary_post_id / its title exist.
|
||||
self._maybe_apply_wip_title(record)
|
||||
|
||||
self.session.commit()
|
||||
return ImportResult(status="imported", image_id=record.id)
|
||||
|
||||
@@ -976,6 +996,47 @@ class Importer:
|
||||
self.session.commit()
|
||||
return ImportResult(status="refreshed", image_id=existing.id)
|
||||
|
||||
def _maybe_apply_wip_title(self, record: ImageRecord) -> None:
|
||||
"""Auto-apply the `wip` system tag to a FRESHLY-imported image when its
|
||||
primary post's TITLE explicitly declares work-in-progress (task #1458 —
|
||||
the artist's own "WIP" / "work in progress" label).
|
||||
|
||||
Called ONLY from the two new-record paths (never deep-scan / supersede),
|
||||
so a manually-removed WIP tag is never re-applied by a routine re-scan —
|
||||
removal sticks. The existing catalogue is covered separately by the
|
||||
operator-triggered backfill sweep. Gated by the settings toggle, and
|
||||
best-effort: any failure is logged, never allowed to fail the import."""
|
||||
hard_on = self.settings.wip_title_tagging_enabled
|
||||
soft_on = self.settings.wip_soft_title_tagging_enabled
|
||||
if not (hard_on or soft_on):
|
||||
return
|
||||
if record.primary_post_id is None:
|
||||
return
|
||||
try:
|
||||
title = self.session.execute(
|
||||
select(Post.post_title).where(Post.id == record.primary_post_id)
|
||||
).scalar_one_or_none()
|
||||
# HARD tier ("WIP"/"work in progress") wins — higher precision, and it
|
||||
# trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
|
||||
# that never trains (source wip_title_soft).
|
||||
if hard_on and matches_wip_title(title):
|
||||
source = WIP_TITLE_SOURCE
|
||||
elif soft_on and matches_soft_wip_title(title):
|
||||
source = WIP_TITLE_SOFT_SOURCE
|
||||
else:
|
||||
return
|
||||
if self._wip_tag_id is _UNSET:
|
||||
self._wip_tag_id = resolve_wip_tag_id(self.session)
|
||||
if self._wip_tag_id is None:
|
||||
return
|
||||
apply_wip_image_tags(
|
||||
self.session, [record.id], self._wip_tag_id, source=source
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — a tag must never fail an import
|
||||
log.warning(
|
||||
"wip-title auto-tag failed for image %s: %s", record.id, exc
|
||||
)
|
||||
|
||||
def _apply_post_fields(self, post: Post, sd) -> None:
|
||||
"""Write a parsed sidecar's post-level fields onto a Post — the SINGLE
|
||||
predicate shared by BOTH ingest paths: the per-media path (_apply_sidecar)
|
||||
@@ -1253,6 +1314,10 @@ class Importer:
|
||||
# per-post Source row.
|
||||
self._apply_sidecar(record, path, artist, explicit_source=source)
|
||||
|
||||
# Title-based WIP auto-tag (task #1458): fresh import only, see the
|
||||
# matching call in _import_media.
|
||||
self._maybe_apply_wip_title(record)
|
||||
|
||||
self.session.commit()
|
||||
return ImportResult(status="imported", image_id=record.id)
|
||||
|
||||
@@ -1313,6 +1378,113 @@ class Importer:
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
def link_existing_image_to_post(
|
||||
self, path: Path, external_post_id: str, *,
|
||||
source: Source | None = None, artist: Artist | None = None,
|
||||
) -> bool:
|
||||
"""Recapture back-link: associate an ALREADY-on-disk image (matched by
|
||||
its stored path) with its post — the link _apply_sidecar normally makes
|
||||
at per-media import time.
|
||||
|
||||
Recapture disk-skips downloaded media, so a pre-existing image (e.g. one
|
||||
pulled under the old gallery-dl path, imported as a bare record with no
|
||||
post) never gets its `image_provenance` row / `primary_post_id`. This
|
||||
backfills it from the walk's (on-disk path, post external id) pairing:
|
||||
find the ImageRecord by path, find/attach the Post by (source,
|
||||
external_post_id), upsert provenance. Idempotent (issue #1288). Returns
|
||||
True when a record matched and was linked; no-op when the file has no
|
||||
record or no id is given."""
|
||||
if not external_post_id:
|
||||
return False
|
||||
record = self.session.execute(
|
||||
select(ImageRecord).where(ImageRecord.path == str(path))
|
||||
).scalar_one_or_none()
|
||||
if record is None:
|
||||
return False
|
||||
artist_id = record.artist_id or (artist.id if artist else None)
|
||||
if artist_id is None:
|
||||
return False
|
||||
if record.artist_id is None:
|
||||
record.artist_id = artist_id
|
||||
post = self._find_or_create_post(
|
||||
source_id=source.id if source else None,
|
||||
external_post_id=str(external_post_id),
|
||||
artist_id=artist_id,
|
||||
)
|
||||
self._attach_provenance(
|
||||
record, post, source_id=source.id if source else None,
|
||||
)
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
def _attach_provenance(
|
||||
self, record: ImageRecord, post: Post, *,
|
||||
source_id: int | None, captured_metadata: dict | None = None,
|
||||
) -> None:
|
||||
"""Upsert the (image, post) `image_provenance` link + keep
|
||||
`primary_post_id` and the denormalized gallery sort keys aligned. Shared
|
||||
by _apply_sidecar (fresh per-media import) and link_existing_image_to_post
|
||||
(recapture back-link, #1288) so the two paths can't diverge. Idempotent.
|
||||
|
||||
Race-safe (image_record_id, post_id) upsert — mirrors the
|
||||
_find_or_create_source/post savepoint pattern. The plain
|
||||
SELECT-then-INSERT pattern lost a race when two workers ran on the same
|
||||
(image, post) pair (e.g. the 5-min recovery sweep re-enqueued a
|
||||
still-running long import), planting duplicates that then broke
|
||||
.scalar_one_or_none() on every later deep-scan rederive
|
||||
(MultipleResultsFound). Alembic 0021's uq_image_provenance_image_post
|
||||
UNIQUE makes this savepoint trip on collision."""
|
||||
exists = self.session.execute(
|
||||
select(ImageProvenance.id).where(
|
||||
ImageProvenance.image_record_id == record.id,
|
||||
ImageProvenance.post_id == post.id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists is None:
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
self.session.add(
|
||||
ImageProvenance(
|
||||
image_record_id=record.id,
|
||||
post_id=post.id,
|
||||
source_id=source_id,
|
||||
captured_metadata=captured_metadata,
|
||||
)
|
||||
)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
if record.primary_post_id is None:
|
||||
record.primary_post_id = post.id
|
||||
# Keep the denormalized gallery sort key (alembic 0035) aligned with
|
||||
# the primary post's publish date so /scroll orders off
|
||||
# ix_image_record_effective_date instead of COALESCE-ing across the
|
||||
# post join. Only override when THIS post is the primary AND carries
|
||||
# a date; otherwise the column keeps its created_at-equivalent server
|
||||
# default (matches the old COALESCE(post_date, created_at) fallback).
|
||||
if record.primary_post_id == post.id and post.post_date is not None:
|
||||
record.effective_date = post.post_date
|
||||
# earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this
|
||||
# image's provenance posts, not just the primary — so the gallery can
|
||||
# sort by original publish rather than the download/repost the primary
|
||||
# points at. Recompute from provenance whenever a dated post is linked;
|
||||
# the provenance row for THIS post was committed above, so the MIN
|
||||
# includes it. Leaves the created_at default when no linked post is dated.
|
||||
if post.post_date is not None:
|
||||
earliest = self.session.execute(
|
||||
select(func.min(Post.post_date))
|
||||
.select_from(ImageProvenance)
|
||||
.join(Post, Post.id == ImageProvenance.post_id)
|
||||
.where(
|
||||
ImageProvenance.image_record_id == record.id,
|
||||
Post.post_date.is_not(None),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if earliest is not None:
|
||||
record.earliest_post_date = earliest
|
||||
self.session.flush()
|
||||
|
||||
def _apply_sidecar(
|
||||
self,
|
||||
record: ImageRecord,
|
||||
@@ -1386,65 +1558,12 @@ class Importer:
|
||||
if not self.post_first:
|
||||
self._apply_post_fields(post, sd)
|
||||
|
||||
# Race-safe (image_record_id, post_id) upsert — mirrors the
|
||||
# _find_or_create_source/post savepoint pattern. The plain
|
||||
# SELECT-then-INSERT pattern lost a race when two workers ran
|
||||
# _apply_sidecar on the same (image, post) pair (e.g. the 5-min
|
||||
# recovery sweep re-enqueued a still-running long import), planting
|
||||
# duplicates that then broke .scalar_one_or_none() on every later
|
||||
# deep-scan rederive (MultipleResultsFound). Alembic 0021 adds the
|
||||
# uq_image_provenance_image_post UNIQUE so this savepoint actually
|
||||
# trips on collision.
|
||||
exists = self.session.execute(
|
||||
select(ImageProvenance.id).where(
|
||||
ImageProvenance.image_record_id == record.id,
|
||||
ImageProvenance.post_id == post.id,
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if exists is None:
|
||||
sp = self.session.begin_nested()
|
||||
try:
|
||||
self.session.add(
|
||||
ImageProvenance(
|
||||
image_record_id=record.id,
|
||||
post_id=post.id,
|
||||
source_id=src.id if src else None,
|
||||
captured_metadata=sd.raw,
|
||||
)
|
||||
)
|
||||
self.session.flush()
|
||||
sp.commit()
|
||||
except IntegrityError:
|
||||
sp.rollback()
|
||||
if record.primary_post_id is None:
|
||||
record.primary_post_id = post.id
|
||||
# Keep the denormalized gallery sort key (alembic 0035) aligned with
|
||||
# the primary post's publish date so /scroll orders off
|
||||
# ix_image_record_effective_date instead of COALESCE-ing across the
|
||||
# post join. Only override when THIS post is the primary AND carries
|
||||
# a date; otherwise the column keeps its created_at-equivalent server
|
||||
# default (matches the old COALESCE(post_date, created_at) fallback).
|
||||
if record.primary_post_id == post.id and post.post_date is not None:
|
||||
record.effective_date = post.post_date
|
||||
# earliest_post_date (alembic 0071) = MIN(post_date) across ALL of this
|
||||
# image's provenance posts, not just the primary — so the gallery can
|
||||
# sort by original publish rather than the download/repost the primary
|
||||
# points at. Recompute from provenance whenever a dated post is linked;
|
||||
# the provenance row for THIS post was committed above, so the MIN
|
||||
# includes it. Leaves the created_at default when no linked post is dated.
|
||||
if post.post_date is not None:
|
||||
earliest = self.session.execute(
|
||||
select(func.min(Post.post_date))
|
||||
.select_from(ImageProvenance)
|
||||
.join(Post, Post.id == ImageProvenance.post_id)
|
||||
.where(
|
||||
ImageProvenance.image_record_id == record.id,
|
||||
Post.post_date.is_not(None),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if earliest is not None:
|
||||
record.earliest_post_date = earliest
|
||||
self.session.flush()
|
||||
# Link the (image, post) provenance + keep primary_post_id / the gallery
|
||||
# sort keys aligned — shared with the recapture back-link path (#1288).
|
||||
self._attach_provenance(
|
||||
record, post, source_id=src.id if src else None,
|
||||
captured_metadata=sd.raw,
|
||||
)
|
||||
|
||||
def _copy_to_library(
|
||||
self, source: Path, sha: str, attribution_path: Path
|
||||
|
||||
@@ -90,6 +90,7 @@ class Ingester:
|
||||
platform: str,
|
||||
error_base: type[Exception],
|
||||
drift_label: str | None = None,
|
||||
body_canary: bool = True,
|
||||
):
|
||||
self.client = client
|
||||
self.downloader = downloader
|
||||
@@ -105,6 +106,12 @@ class Ingester:
|
||||
# update"). Defaults to the platform name; adapters pass a richer phrase
|
||||
# (e.g. "Patreon API", "SubscribeStar markup").
|
||||
self._drift_label = drift_label or platform
|
||||
# #862 canary opt-out: platforms whose posts legitimately have empty
|
||||
# bodies across large samples (pixiv — caption-less artists are common)
|
||||
# would false-positive the zero-bodies-means-drift alarm; their clients
|
||||
# catch drift structurally (response-shape checks) instead. The
|
||||
# "bodies X/N" summary line still surfaces the ratio either way.
|
||||
self._body_canary = body_canary
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
@@ -172,10 +179,11 @@ class Ingester:
|
||||
written: list[str] = []
|
||||
post_records: list[str] = []
|
||||
quarantined_paths: list[str] = []
|
||||
# #830 recapture: (on-disk path, CDN source_url) pairs for already-present
|
||||
# media, so phase 3 can backfill the ImageRecord's source_filehash WITHOUT
|
||||
# re-downloading or unlinking the file. Empty outside recapture mode.
|
||||
relink: list[tuple[str, str]] = []
|
||||
# #830 recapture: (on-disk path, CDN source_url, post_id) triples for
|
||||
# already-present media, so phase 3 can (a) backfill the ImageRecord's
|
||||
# source_filehash and (b) link the on-disk image to its Post (#1288) —
|
||||
# WITHOUT re-downloading or unlinking the file. Empty outside recapture.
|
||||
relink: list[tuple[str, str, str]] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
@@ -403,12 +411,15 @@ class Ingester:
|
||||
to_clear.append(key)
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
# #830 recapture: surface (on-disk path, CDN url) so phase
|
||||
# 3 can backfill source_filehash for inline-image
|
||||
# localization — a SEPARATE non-deleting channel, never the
|
||||
# import list (which would unlink the file, per above).
|
||||
# #830/#1288 recapture: surface (on-disk path, CDN url,
|
||||
# post_id) so phase 3 can backfill source_filehash AND link
|
||||
# the on-disk image to its Post — a SEPARATE non-deleting
|
||||
# channel, never the import list (which would unlink the
|
||||
# file, per above).
|
||||
if recapture and outcome.path is not None:
|
||||
relink.append((str(outcome.path), media_item.url))
|
||||
relink.append(
|
||||
(str(outcome.path), media_item.url, media_item.post_id)
|
||||
)
|
||||
elif outcome.status == "skipped_seen":
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
@@ -536,7 +547,11 @@ class Ingester:
|
||||
# creds") so the breakage screams instead of silently archiving empties.
|
||||
# Only reached on an otherwise-clean walk (timeout/stop/error returned
|
||||
# above), so it never masks a more specific failure.
|
||||
if posts_recorded >= _CANARY_MIN_SAMPLE and posts_with_body == 0:
|
||||
if (
|
||||
self._body_canary
|
||||
and posts_recorded >= _CANARY_MIN_SAMPLE
|
||||
and posts_with_body == 0
|
||||
):
|
||||
msg = (
|
||||
f"Post-body canary: extracted a body from 0 of {posts_recorded} "
|
||||
"posts — Patreon's body field shape likely changed; the ingester "
|
||||
@@ -562,73 +577,6 @@ class Ingester:
|
||||
error_type=None, error_message=None,
|
||||
)
|
||||
|
||||
# -- preview (dry-run) -------------------------------------------------
|
||||
|
||||
def preview(
|
||||
self,
|
||||
source_id: int,
|
||||
campaign_id: str,
|
||||
*,
|
||||
page_limit: int = 3,
|
||||
sample_size: int = 10,
|
||||
) -> dict:
|
||||
"""Dry-run (plan #708 B4): walk up to `page_limit` pages and count media
|
||||
NOT already in the seen/dead ledgers, WITHOUT downloading anything.
|
||||
|
||||
Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets
|
||||
an operator gauge "is this source worth a backfill?" cheaply. Returns:
|
||||
{total_new, posts_scanned, pages_scanned, has_more,
|
||||
sample: [{title, date, new}, ...]} # sample = posts with new media
|
||||
A client-level failure (auth/drift) propagates to the caller.
|
||||
"""
|
||||
total_new = 0
|
||||
posts_scanned = 0
|
||||
pages_scanned = 0
|
||||
has_more = False
|
||||
sample: list[dict] = []
|
||||
unset = object()
|
||||
last_page: object = unset
|
||||
# #874: same gated-post gate as run() — the preview must not count
|
||||
# blurred locked-preview media as "new", or it would overstate a gated
|
||||
# source's backlog (preview/apply parity, rule 93).
|
||||
post_is_gated = getattr(self.client, "post_is_gated", None)
|
||||
for post, included, page_cursor in self.client.iter_posts(
|
||||
campaign_id, cursor=None
|
||||
):
|
||||
if page_cursor != last_page:
|
||||
last_page = page_cursor
|
||||
pages_scanned += 1
|
||||
if pages_scanned > page_limit:
|
||||
has_more = True
|
||||
pages_scanned = page_limit
|
||||
break
|
||||
posts_scanned += 1
|
||||
if post_is_gated and post_is_gated(post):
|
||||
continue
|
||||
media = self.client.extract_media(post, included)
|
||||
if not media:
|
||||
continue
|
||||
keys = [self._ledger_key(m) for m in media]
|
||||
skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys)
|
||||
new_count = sum(1 for m in media if self._ledger_key(m) not in skip)
|
||||
total_new += new_count
|
||||
if new_count > 0 and len(sample) < sample_size:
|
||||
meta = self.client.post_meta(post)
|
||||
sample.append(
|
||||
{
|
||||
"title": meta.get("title") or "(untitled)",
|
||||
"date": meta.get("date"),
|
||||
"new": new_count,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"total_new": total_new,
|
||||
"posts_scanned": posts_scanned,
|
||||
"pages_scanned": pages_scanned,
|
||||
"has_more": has_more,
|
||||
"sample": sample,
|
||||
}
|
||||
|
||||
# -- failure mapping (adapter overrides) -------------------------------
|
||||
|
||||
def _failure_result(self, exc: Exception, _result) -> DownloadResult:
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Interpreter translation client (milestone 143) — a thin SYNC wrapper over the
|
||||
self-hosted Interpreter LAN service (LibreTranslate-compatible `/v1/translate`).
|
||||
|
||||
Sync (requests) because the only caller is the sync celery translate sweep, and
|
||||
it mirrors FC's other platform clients. The service knows nothing about Curator —
|
||||
all Curator logic stays here. base_url is operator-configured (empty until set;
|
||||
behind a reverse proxy). Full API contract: Scribe note #1347.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
|
||||
class InterpreterUnavailable(Exception):
|
||||
"""The translation engine is down / unreachable / draining — a connection
|
||||
error, HTTP 429, or a 5xx (commonly 502/503/504 through a reverse proxy while
|
||||
the service restarts). Retry later, don't drop the item. ``retry_after`` holds
|
||||
the server's Retry-After hint in seconds when it sent one, so the caller can
|
||||
back off exactly as long as it's asked to."""
|
||||
|
||||
def __init__(self, message: str, *, retry_after: float | None = None):
|
||||
super().__init__(message)
|
||||
self.retry_after = retry_after
|
||||
|
||||
|
||||
class InterpreterBadRequest(Exception):
|
||||
"""Bad request params (HTTP 400)."""
|
||||
|
||||
|
||||
def _url(base_url: str, path: str) -> str:
|
||||
return f"{base_url.rstrip('/')}{path}"
|
||||
|
||||
|
||||
def _parse_retry_after(resp) -> float | None:
|
||||
"""Parse a Retry-After header (RFC 7231: delta-seconds or an HTTP-date) into
|
||||
non-negative seconds, or None if absent/unparseable — so a gracefully-
|
||||
draining Interpreter can tell Curator exactly how long to wait before it
|
||||
tries again."""
|
||||
raw = (resp.headers.get("Retry-After") or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return max(0.0, float(int(raw))) # delta-seconds form
|
||||
except ValueError:
|
||||
pass
|
||||
try: # HTTP-date form
|
||||
when = parsedate_to_datetime(raw)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if when is None:
|
||||
return None
|
||||
if when.tzinfo is None:
|
||||
when = when.replace(tzinfo=UTC)
|
||||
return max(0.0, (when - datetime.now(UTC)).total_seconds())
|
||||
|
||||
|
||||
# A shared session pools the keep-alive connection across the per-post sweep
|
||||
# calls, and retries CONNECT failures only (connect=2, short backoff) — smoothing
|
||||
# the instant a reverse proxy reloads. Status codes are deliberately NOT retried
|
||||
# (status=0, raise_on_status=False): translate() maps 429/5xx → InterpreterUnavailable
|
||||
# itself, and letting urllib3 retry a draining 503 would defeat the Retry-After
|
||||
# backoff we honour upstream.
|
||||
_retry = Retry(
|
||||
total=None, connect=2, read=0, redirect=0, status=0,
|
||||
backoff_factor=0.3, raise_on_status=False,
|
||||
)
|
||||
session = requests.Session()
|
||||
_adapter = HTTPAdapter(max_retries=_retry)
|
||||
session.mount("http://", _adapter)
|
||||
session.mount("https://", _adapter)
|
||||
|
||||
|
||||
def health(base_url: str, *, timeout: float = 5.0) -> bool:
|
||||
"""True iff the Interpreter LLM engine is up. Any error (unset URL, network,
|
||||
non-200, engine down) → False, so the sweep just no-ops rather than raising."""
|
||||
if not base_url:
|
||||
return False
|
||||
try:
|
||||
r = session.get(_url(base_url, "/v1/health"), timeout=timeout)
|
||||
except requests.RequestException:
|
||||
return False
|
||||
if r.status_code != 200:
|
||||
return False
|
||||
engines = (r.json() or {}).get("engines") or {}
|
||||
return bool(engines.get("llm"))
|
||||
|
||||
|
||||
def translate(
|
||||
texts: list[str], *, base_url: str, target: str = "en",
|
||||
source: str = "auto", timeout: float = 120.0,
|
||||
) -> dict:
|
||||
"""Translate a batch. Returns::
|
||||
|
||||
{"translations": [str, ...], # SAME order & length as `texts`
|
||||
"detected_lang": str | None, # aggregate: describes the FIRST item
|
||||
"detected_confidence": float | None, # detector's confidence, if given
|
||||
"engine": str | None,
|
||||
"engine_version": str | None}
|
||||
|
||||
Interpreter batch metadata is aggregate (first item only) — fine for one
|
||||
post's ``[title, description]`` batch since they share a language. Passthrough
|
||||
items (already target-language / emoji-only) come back UNCHANGED in their
|
||||
slot. Raises InterpreterUnavailable on a connection error / 429 / 5xx (retry
|
||||
later, honouring Retry-After), InterpreterBadRequest on 400. (Scribe note
|
||||
#1347.)
|
||||
"""
|
||||
if not texts:
|
||||
return {"translations": [], "detected_lang": None,
|
||||
"detected_confidence": None,
|
||||
"engine": None, "engine_version": None}
|
||||
try:
|
||||
r = session.post(
|
||||
_url(base_url, "/v1/translate"),
|
||||
json={"q": list(texts), "source": source,
|
||||
"target": target, "engine": "auto"},
|
||||
timeout=timeout,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
raise InterpreterUnavailable(str(e)) from e
|
||||
if r.status_code == 400:
|
||||
raise InterpreterBadRequest((r.json() or {}).get("error", "bad request"))
|
||||
# A gracefully-draining service — often behind a reverse proxy — returns 429
|
||||
# or a 5xx gateway error (502/503/504), not always a clean 503. Treat every
|
||||
# one as "unavailable, retry later" and honour any Retry-After it sends, so a
|
||||
# rolling Interpreter restart interrupts the sweep cleanly (instead of raising
|
||||
# an opaque HTTPError) and resumes exactly when the service says it's ready.
|
||||
if r.status_code == 429 or r.status_code >= 500:
|
||||
raise InterpreterUnavailable(
|
||||
f"interpreter unavailable (HTTP {r.status_code})",
|
||||
retry_after=_parse_retry_after(r),
|
||||
)
|
||||
r.raise_for_status()
|
||||
body = r.json() or {}
|
||||
out = body.get("translatedText")
|
||||
# q was an array → translatedText is an array (same order & length). Guard a
|
||||
# scalar reply just in case (shouldn't happen for an array q).
|
||||
if not isinstance(out, list):
|
||||
out = [out]
|
||||
interp = body.get("interpreter") or {}
|
||||
detected = body.get("detectedLanguage") or {}
|
||||
return {
|
||||
"translations": out,
|
||||
"detected_lang": detected.get("language"),
|
||||
"detected_confidence": detected.get("confidence"),
|
||||
"engine": interp.get("engine"),
|
||||
"engine_version": interp.get("engine_version"),
|
||||
}
|
||||
@@ -12,13 +12,11 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import TagSuggestionRejection
|
||||
from ...models.tag import image_tag
|
||||
from .aliases import AliasService
|
||||
|
||||
|
||||
class AllowlistService:
|
||||
def __init__(self, session: AsyncSession):
|
||||
self.session = session
|
||||
self.aliases = AliasService(session)
|
||||
|
||||
async def _apply_image_tag(self, image_id: int, tag_id: int, source: str):
|
||||
stmt = insert(image_tag).values(
|
||||
@@ -41,18 +39,6 @@ class AllowlistService:
|
||||
await self._apply_image_tag(image_id, tag_id, source="ml_accepted")
|
||||
await self._clear_rejection(image_id, tag_id)
|
||||
|
||||
async def add_alias_and_accept(
|
||||
self,
|
||||
image_id: int,
|
||||
alias_string: str,
|
||||
alias_category: str,
|
||||
canonical_tag_id: int,
|
||||
) -> None:
|
||||
await self.aliases.create(
|
||||
alias_string, alias_category, canonical_tag_id
|
||||
)
|
||||
await self.accept(image_id, canonical_tag_id)
|
||||
|
||||
async def dismiss(self, image_id: int, tag_id: int) -> None:
|
||||
stmt = insert(TagSuggestionRejection).values(
|
||||
image_record_id=image_id, tag_id=tag_id
|
||||
|
||||
+136
-10
@@ -13,11 +13,20 @@ exact CCIP difference metric/threshold gets validated against the model during
|
||||
the hands-on eval. numpy is imported lazily (API worker has it via pgvector).
|
||||
"""
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import exists, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ...models import ImageRegion, MLSettings, Tag, TagKind
|
||||
from ...models import (
|
||||
CcipPrototypeState,
|
||||
CharacterPrototype,
|
||||
ImageRegion,
|
||||
MLSettings,
|
||||
Tag,
|
||||
TagKind,
|
||||
TagPositiveConfirmation,
|
||||
)
|
||||
from ...models.tag import image_tag
|
||||
from .training_data import _AUTO_SOURCES
|
||||
|
||||
# Cosine-similarity floor to call a figure the same character. The live setting
|
||||
# (ml_settings.ccip_match_threshold) drives it; this is only the fallback when no
|
||||
@@ -62,6 +71,18 @@ def _single_character_images():
|
||||
)
|
||||
|
||||
|
||||
def _hygiene_tagged_images():
|
||||
"""Subquery of image ids carrying any SYSTEM tag (wip / banner / editor
|
||||
screenshot). Training hygiene (#128): such images never contribute
|
||||
reference prototypes — a faceless wip's figure region would otherwise
|
||||
become an identity reference for the character it's tagged with."""
|
||||
return (
|
||||
select(image_tag.c.image_record_id)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.is_system.is_(True))
|
||||
)
|
||||
|
||||
|
||||
async def _ref_signature(session: AsyncSession) -> tuple:
|
||||
n_tags = (
|
||||
await session.execute(
|
||||
@@ -79,7 +100,28 @@ async def _ref_signature(session: AsyncSession) -> tuple:
|
||||
)
|
||||
)
|
||||
).one()
|
||||
return (n_tags, n_regs, max_id)
|
||||
# Hygiene applications must invalidate too: tagging an image `wip` changes
|
||||
# the reference set without touching character-tag or region counts.
|
||||
n_hygiene = (
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(image_tag)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.is_system.is_(True))
|
||||
)
|
||||
).scalar_one()
|
||||
return (n_tags, n_regs, max_id, n_hygiene)
|
||||
|
||||
|
||||
def _positive_char_tag():
|
||||
"""Condition on the joined character image_tag: HUMAN-applied or operator-
|
||||
confirmed — NOT an unconfirmed auto-apply. Keeps an auto-tagged character from
|
||||
self-seeding CCIP references, so a ccip_auto misfire can't reinforce itself
|
||||
(milestone 139) — mirrors the head-training positive exclusion."""
|
||||
return image_tag.c.source.not_in(_AUTO_SOURCES) | exists().where(
|
||||
TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
|
||||
TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
|
||||
)
|
||||
|
||||
|
||||
async def character_references(session: AsyncSession) -> dict[int, list]:
|
||||
@@ -99,9 +141,13 @@ async def character_references(session: AsyncSession) -> dict[int, list]:
|
||||
)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.kind == TagKind.character)
|
||||
.where(_positive_char_tag())
|
||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||
.where(ImageRegion.image_record_id.in_(_single_character_images()))
|
||||
.where(
|
||||
ImageRegion.image_record_id.not_in(_hygiene_tagged_images())
|
||||
)
|
||||
)
|
||||
).all()
|
||||
refs: dict[int, list] = {}
|
||||
@@ -123,6 +169,60 @@ async def _tag_names(session: AsyncSession, tag_ids: list[int]) -> dict[int, str
|
||||
)
|
||||
|
||||
|
||||
# Per-character normalized prototype matrices, cached per process and refreshed
|
||||
# INCREMENTALLY: only characters whose ccip_prototype_state.updated_at advanced
|
||||
# are reloaded. This replaces the request-path rebuild of the ENTIRE reference
|
||||
# blob (the ~4s stall, #1317) — the prototypes are precomputed off the request
|
||||
# path by services.ml.character_prototypes (a beat + after each retrain).
|
||||
_PROTO_CACHE: dict = {"mats": {}, "ver": {}}
|
||||
|
||||
|
||||
async def _load_prototypes(session: AsyncSession) -> dict:
|
||||
"""{tag_id: (P, D) L2-normalized prototype matrix} from character_prototype,
|
||||
served from the in-process cache and reloading ONLY the characters whose
|
||||
updated_at changed. Empty dict when the store isn't populated yet (cold start
|
||||
→ match_image falls back to the legacy on-the-fly reference build)."""
|
||||
import numpy as np
|
||||
|
||||
versions = dict(
|
||||
(
|
||||
await session.execute(
|
||||
select(CcipPrototypeState.tag_id, CcipPrototypeState.updated_at)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
mats = _PROTO_CACHE["mats"]
|
||||
ver = _PROTO_CACHE["ver"]
|
||||
# Forget characters that no longer have prototypes.
|
||||
for tag_id in [t for t in mats if t not in versions]:
|
||||
mats.pop(tag_id, None)
|
||||
ver.pop(tag_id, None)
|
||||
# Reload only the characters whose prototypes changed since we cached them.
|
||||
stale = [t for t, u in versions.items() if ver.get(t) != u]
|
||||
if stale:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
CharacterPrototype.tag_id, CharacterPrototype.ccip_embedding
|
||||
).where(CharacterPrototype.tag_id.in_(stale))
|
||||
)
|
||||
).all()
|
||||
by_tag: dict[int, list] = {}
|
||||
for tag_id, vec in rows:
|
||||
by_tag.setdefault(tag_id, []).append(
|
||||
np.asarray(vec, dtype=np.float32)
|
||||
)
|
||||
for tag_id in stale:
|
||||
vecs = by_tag.get(tag_id)
|
||||
if vecs:
|
||||
mats[tag_id] = _l2norm(np.vstack(vecs), np)
|
||||
ver[tag_id] = versions[tag_id]
|
||||
else:
|
||||
mats.pop(tag_id, None)
|
||||
ver.pop(tag_id, None)
|
||||
return mats
|
||||
|
||||
|
||||
async def match_image(
|
||||
session: AsyncSession, image_id: int, threshold: float | None = None
|
||||
) -> list[dict]:
|
||||
@@ -136,18 +236,30 @@ async def match_image(
|
||||
if threshold is None:
|
||||
threshold = await _settings_threshold(session)
|
||||
|
||||
qvecs = (
|
||||
# Keep each figure region's bbox alongside its vector so a match can point at
|
||||
# the figure that matched (#1206 grounding), not just the score.
|
||||
fig_rows = (
|
||||
await session.execute(
|
||||
select(ImageRegion.ccip_embedding).where(
|
||||
select(
|
||||
ImageRegion.ccip_embedding,
|
||||
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
|
||||
ImageRegion.kind, ImageRegion.detector_version,
|
||||
).where(
|
||||
ImageRegion.image_record_id == image_id,
|
||||
ImageRegion.kind.in_(_FIGURE_KINDS),
|
||||
ImageRegion.ccip_embedding.is_not(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
if not qvecs:
|
||||
).all()
|
||||
if not fig_rows:
|
||||
return []
|
||||
refs = await character_references(session)
|
||||
# Prefer the precomputed prototype store (fast, incremental). On a cold start
|
||||
# (store not yet populated post-deploy) fall back to the legacy on-the-fly
|
||||
# reference build so character suggestions work immediately — the background
|
||||
# refresh populates the store within ~15 min, after which this path is used
|
||||
# and the per-accept ~4s rebuild is gone (#1317).
|
||||
protos = await _load_prototypes(session)
|
||||
refs = protos if protos else await character_references(session)
|
||||
if not refs:
|
||||
return []
|
||||
applied = set(
|
||||
@@ -161,13 +273,25 @@ async def match_image(
|
||||
)
|
||||
names = await _tag_names(session, [t for t in refs if t not in applied])
|
||||
|
||||
qvecs = [r[0] for r in fig_rows]
|
||||
fig_meta = [
|
||||
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
|
||||
for _v, rx, ry, rw, rh, kind, detector in fig_rows
|
||||
]
|
||||
Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np)
|
||||
out = []
|
||||
for tag_id, vecs in refs.items():
|
||||
if tag_id in applied:
|
||||
continue
|
||||
R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np)
|
||||
best = float((Q @ R.T).max()) # best (query figure, reference) cosine
|
||||
# Prototype matrices are already L2-normalized; legacy refs are raw
|
||||
# vector lists that still need stacking + normalizing.
|
||||
R = vecs if protos else _l2norm(
|
||||
np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
|
||||
)
|
||||
sims = Q @ R.T # (n_query_figures, n_references)
|
||||
per_figure = sims.max(axis=1) # best reference cosine per figure
|
||||
best_figure = int(per_figure.argmax())
|
||||
best = float(per_figure[best_figure])
|
||||
if best >= threshold:
|
||||
out.append({
|
||||
"tag_id": tag_id,
|
||||
@@ -175,6 +299,8 @@ async def match_image(
|
||||
"category": "character",
|
||||
"score": round(best, 4),
|
||||
"source": "ccip",
|
||||
# the figure region that matched → grounds the character tag.
|
||||
"grounding": fig_meta[best_figure],
|
||||
})
|
||||
out.sort(key=lambda d: d["score"], reverse=True)
|
||||
return out
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
"""Precomputed CCIP character prototypes — incremental builder (#1317, m138).
|
||||
|
||||
Turns the per-character CCIP reference set into a precomputed artifact so the
|
||||
matcher never rebuilds it on the request path. Sync (runs in the celery ml
|
||||
worker via tasks.ml.refresh_character_prototypes); the async matcher only READS
|
||||
the character_prototype table.
|
||||
|
||||
`refresh_character_prototypes`:
|
||||
1. Cheap GLOBAL gate — a few COUNTs (`_global_signature`). Unchanged + not
|
||||
`full` → no-op (nothing that affects references changed since last refresh).
|
||||
2. Per-character fingerprint diff (one GROUP BY) → rebuild ONLY the characters
|
||||
whose references changed (or are new); drop characters that lost all refs.
|
||||
Each rebuild loads just ONE character's reference vectors, caps them to
|
||||
MLSettings.ccip_prototype_cap, and replaces that character's prototype rows — so
|
||||
cost scales with WHAT changed, not with the library size.
|
||||
|
||||
The reference PREDICATE (single-character, non-hygiene, figure CCIP) is imported
|
||||
from ccip so the prototypes match exactly what the legacy matcher selected.
|
||||
"""
|
||||
|
||||
import random
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...models import (
|
||||
CcipPrototypeState,
|
||||
CharacterPrototype,
|
||||
ImageRegion,
|
||||
MLSettings,
|
||||
Tag,
|
||||
TagKind,
|
||||
TagPositiveConfirmation,
|
||||
)
|
||||
from ...models.tag import image_tag
|
||||
from .ccip import (
|
||||
_FIGURE_KINDS,
|
||||
_hygiene_tagged_images,
|
||||
_l2norm,
|
||||
_positive_char_tag,
|
||||
_single_character_images,
|
||||
)
|
||||
|
||||
# Deterministic per-tag capping so a rebuild of an UNCHANGED reference set
|
||||
# resamples identically (stable prototypes, no churn between refreshes).
|
||||
_SAMPLE_SEED = 1317
|
||||
|
||||
|
||||
def _global_signature(session: Session) -> str:
|
||||
"""Cheap 'could any references have changed' gate: character-tag count,
|
||||
figure-CCIP region count + max id, hygiene-tag count. A few COUNTs — the same
|
||||
quantities the legacy per-request signature used, now computed once per
|
||||
refresh instead of on every /suggestions call."""
|
||||
n_tags = session.execute(
|
||||
select(func.count())
|
||||
.select_from(image_tag)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.kind == TagKind.character)
|
||||
).scalar_one()
|
||||
n_regs, max_id = session.execute(
|
||||
select(func.count(), func.max(ImageRegion.id)).where(
|
||||
ImageRegion.kind.in_(_FIGURE_KINDS),
|
||||
ImageRegion.ccip_embedding.is_not(None),
|
||||
)
|
||||
).one()
|
||||
n_hygiene = session.execute(
|
||||
select(func.count())
|
||||
.select_from(image_tag)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.is_system.is_(True))
|
||||
).scalar_one()
|
||||
# Character confirmations affect the reference set now that auto-tags only
|
||||
# seed references once confirmed (milestone 139) — so a confirm must trip the
|
||||
# gate, or the per-character diff (which reflects it) never runs.
|
||||
n_conf = session.execute(
|
||||
select(func.count())
|
||||
.select_from(TagPositiveConfirmation)
|
||||
.join(Tag, Tag.id == TagPositiveConfirmation.tag_id)
|
||||
.where(Tag.kind == TagKind.character)
|
||||
).scalar_one()
|
||||
return f"{n_tags}:{n_regs}:{max_id or 0}:{n_hygiene}:{n_conf}"
|
||||
|
||||
|
||||
def _current_fingerprints(session: Session) -> dict[int, str]:
|
||||
"""Per-character (reference count, max reference region id) over the SAME
|
||||
predicate the matcher's references use. One GROUP BY → the change detector:
|
||||
a character whose fingerprint moved (gained/lost a reference) needs a
|
||||
rebuild; everyone else is left untouched."""
|
||||
rows = session.execute(
|
||||
select(
|
||||
image_tag.c.tag_id,
|
||||
func.count(ImageRegion.id),
|
||||
func.max(ImageRegion.id),
|
||||
)
|
||||
.select_from(ImageRegion)
|
||||
.join(
|
||||
image_tag,
|
||||
image_tag.c.image_record_id == ImageRegion.image_record_id,
|
||||
)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.kind == TagKind.character)
|
||||
.where(_positive_char_tag())
|
||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||
.where(ImageRegion.image_record_id.in_(_single_character_images()))
|
||||
.where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
|
||||
.group_by(image_tag.c.tag_id)
|
||||
).all()
|
||||
return {tag_id: f"{cnt}:{mx}" for tag_id, cnt, mx in rows}
|
||||
|
||||
|
||||
def _rebuild_one(session: Session, tag_id: int, cap: int) -> int:
|
||||
"""Replace ONE character's prototype rows from its current references, capped
|
||||
to `cap`. Loads only this character's vectors (bounded by its popularity)."""
|
||||
rows = session.execute(
|
||||
select(ImageRegion.id, ImageRegion.ccip_embedding)
|
||||
.select_from(ImageRegion)
|
||||
.join(
|
||||
image_tag,
|
||||
image_tag.c.image_record_id == ImageRegion.image_record_id,
|
||||
)
|
||||
.where(image_tag.c.tag_id == tag_id)
|
||||
.where(_positive_char_tag())
|
||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||
.where(ImageRegion.image_record_id.in_(_single_character_images()))
|
||||
.where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
|
||||
).all()
|
||||
# Cap for bounded MATCH cost. A random sample (not most-recent) keeps the
|
||||
# prototypes representative of the whole reference set; a fixed per-tag seed
|
||||
# makes an unchanged set resample identically.
|
||||
if cap > 0 and len(rows) > cap:
|
||||
rows = random.Random(f"{_SAMPLE_SEED}:{tag_id}").sample(rows, cap)
|
||||
session.execute(
|
||||
delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
|
||||
)
|
||||
for region_id, vec in rows:
|
||||
session.add(
|
||||
CharacterPrototype(
|
||||
tag_id=tag_id, region_id=region_id, ccip_embedding=vec
|
||||
)
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def refresh_character_prototypes(
|
||||
session: Session, *, full: bool = False
|
||||
) -> dict[str, int | bool]:
|
||||
"""Incrementally refresh the prototype store. `full=True` rebuilds every
|
||||
character regardless of the gate/fingerprints (nightly reconcile). Returns
|
||||
{skipped, rebuilt, removed}; commits."""
|
||||
settings = session.execute(
|
||||
select(MLSettings).where(MLSettings.id == 1)
|
||||
).scalar_one()
|
||||
sig = _global_signature(session)
|
||||
if not full and settings.ccip_ref_signature == sig:
|
||||
return {"skipped": True, "rebuilt": 0, "removed": 0}
|
||||
|
||||
cap = settings.ccip_prototype_cap
|
||||
current = _current_fingerprints(session)
|
||||
stored = dict(
|
||||
session.execute(
|
||||
select(CcipPrototypeState.tag_id, CcipPrototypeState.fingerprint)
|
||||
).all()
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
rebuilt = 0
|
||||
for tag_id, fp in current.items():
|
||||
if full or stored.get(tag_id) != fp:
|
||||
_rebuild_one(session, tag_id, cap)
|
||||
state = session.get(CcipPrototypeState, tag_id)
|
||||
if state is None:
|
||||
state = CcipPrototypeState(tag_id=tag_id)
|
||||
session.add(state)
|
||||
state.fingerprint = fp
|
||||
state.updated_at = now
|
||||
rebuilt += 1
|
||||
# Characters that lost every reference (refs removed / re-kinded / image now
|
||||
# multi-character) → drop their prototypes + state so they stop matching.
|
||||
removed = 0
|
||||
for tag_id in set(stored) - set(current):
|
||||
session.execute(
|
||||
delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
|
||||
)
|
||||
session.execute(
|
||||
delete(CcipPrototypeState).where(CcipPrototypeState.tag_id == tag_id)
|
||||
)
|
||||
removed += 1
|
||||
|
||||
settings.ccip_ref_signature = sig
|
||||
session.commit()
|
||||
return {"skipped": False, "rebuilt": rebuilt, "removed": removed}
|
||||
|
||||
|
||||
def retract_auto_applied_ccip(session: Session) -> int:
|
||||
"""Soft auto-apply for CCIP character tags (milestone 139): re-score every
|
||||
standing source='ccip_auto' character tag against that character's prototypes
|
||||
and REMOVE the ones whose best figure match is now BELOW
|
||||
ccip_auto_apply_threshold. Skips operator-confirmed tags. SILENT — a low score
|
||||
isn't proof the tag was wrong (that's reserved for an operator removal). No-op
|
||||
unless ccip_auto_apply_enabled. A character with no prototypes yet, or an image
|
||||
with no figure vectors, is left alone (can't judge → keep). Returns
|
||||
n_retracted."""
|
||||
import numpy as np
|
||||
|
||||
settings = session.execute(
|
||||
select(MLSettings).where(MLSettings.id == 1)
|
||||
).scalar_one()
|
||||
if not settings.ccip_auto_apply_enabled:
|
||||
return 0
|
||||
thr = float(settings.ccip_auto_apply_threshold)
|
||||
pairs = session.execute(
|
||||
select(image_tag.c.image_record_id, image_tag.c.tag_id)
|
||||
.where(image_tag.c.source == "ccip_auto")
|
||||
).all()
|
||||
if not pairs:
|
||||
return 0
|
||||
confirmed = {
|
||||
(iid, tid) for iid, tid in session.execute(
|
||||
select(
|
||||
TagPositiveConfirmation.image_record_id,
|
||||
TagPositiveConfirmation.tag_id,
|
||||
)
|
||||
).all()
|
||||
}
|
||||
# Each involved character's normalized prototype matrix, loaded once.
|
||||
proto: dict[int, object] = {}
|
||||
for tid in {tid for _iid, tid in pairs}:
|
||||
vecs = [
|
||||
v for (v,) in session.execute(
|
||||
select(CharacterPrototype.ccip_embedding)
|
||||
.where(CharacterPrototype.tag_id == tid)
|
||||
)
|
||||
]
|
||||
if vecs:
|
||||
proto[tid] = _l2norm(
|
||||
np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
|
||||
)
|
||||
retracted = 0
|
||||
for iid, tid in pairs:
|
||||
if (iid, tid) in confirmed or tid not in proto:
|
||||
continue # confirmed / no prototypes
|
||||
qvecs = [
|
||||
v for (v,) in session.execute(
|
||||
select(ImageRegion.ccip_embedding)
|
||||
.where(ImageRegion.image_record_id == iid)
|
||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||
)
|
||||
]
|
||||
if not qvecs:
|
||||
continue # no figure vectors → keep
|
||||
Q = _l2norm(
|
||||
np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np
|
||||
)
|
||||
if float((Q @ proto[tid].T).max()) < thr:
|
||||
session.execute(
|
||||
image_tag.delete()
|
||||
.where(image_tag.c.image_record_id == iid)
|
||||
.where(image_tag.c.tag_id == tid)
|
||||
.where(image_tag.c.source == "ccip_auto")
|
||||
)
|
||||
retracted += 1
|
||||
session.commit()
|
||||
return retracted
|
||||
@@ -121,13 +121,22 @@ def triage_errored_jobs(
|
||||
def recover_defective_image(
|
||||
session: Session, image_id: int, *, images_root: Path,
|
||||
) -> dict:
|
||||
"""Delete the defective copy + record and re-poll its subscription Source.
|
||||
"""Delete the defective copy + record and queue its re-fetch — surgically
|
||||
where possible.
|
||||
|
||||
Mirrors the Layer-2 import refetch: with the bad file gone, the source's
|
||||
next gallery-dl run re-fetches a fresh copy, which re-imports as a new
|
||||
record and re-enters the GPU pipeline. The record delete cascades the
|
||||
error tombstones with it. 'no_source' when no enabled, real-URL Source is
|
||||
reachable via the image's provenance — manual remediation there."""
|
||||
Two re-fetch layers (operator 2026-07-03: deep back-catalogue items are
|
||||
NEVER re-walked by the normal cadence, so recovery can't rely on it):
|
||||
1. SURGICAL: any ExternalLink rows on the image's post(s) are reset +
|
||||
re-dispatched — this is how external-host files (the common defect
|
||||
case: big videos) come back regardless of post age. Sha-dedupe at
|
||||
import discards payload files that still exist.
|
||||
2. BROAD: a source re-check, which re-fetches gallery-dl-NATIVE files the
|
||||
walk still reaches (recent posts). A native file deeper than the walk
|
||||
needs a per-source backfill/deep scan — reported via links_reset=0 so
|
||||
the caller can say so.
|
||||
|
||||
The record delete cascades the error tombstones with it. 'no_source' when
|
||||
no enabled, real-URL Source resolves via provenance — manual there."""
|
||||
rec = session.get(ImageRecord, image_id)
|
||||
if rec is None:
|
||||
return {"status": "not_found"}
|
||||
@@ -143,14 +152,28 @@ def recover_defective_image(
|
||||
).scalars().first()
|
||||
if src_id is None:
|
||||
return {"status": "no_source"}
|
||||
# Capture the post linkage BEFORE the delete cascades provenance away.
|
||||
post_ids = session.execute(
|
||||
select(ImageProvenance.post_id)
|
||||
.where(ImageProvenance.image_record_id == image_id)
|
||||
).scalars().all()
|
||||
path = rec.path
|
||||
summary = delete_images(session, image_ids=[image_id], images_root=images_root)
|
||||
from ..external_links import refetch_links_for_post
|
||||
|
||||
links_reset = 0
|
||||
for pid in post_ids:
|
||||
links_reset += refetch_links_for_post(session, pid)["links_reset"]
|
||||
# Lazy import (services -> tasks would cycle at module load).
|
||||
from ...tasks.download import download_source
|
||||
|
||||
download_source.delay(src_id)
|
||||
log.warning(
|
||||
"gpu triage recovery: deleted defective image %s (%s) and queued a "
|
||||
"re-check of source %s to re-fetch it", image_id, path, src_id,
|
||||
"gpu triage recovery: deleted defective image %s (%s); reset %d "
|
||||
"external link(s) and queued a re-check of source %s",
|
||||
image_id, path, links_reset, src_id,
|
||||
)
|
||||
return {"status": "refetch_queued", "source_id": src_id, **summary}
|
||||
return {
|
||||
"status": "refetch_queued", "source_id": src_id,
|
||||
"links_reset": links_reset, **summary,
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy import delete, exists, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
@@ -32,14 +32,18 @@ from ...models import (
|
||||
ImageRecord,
|
||||
ImageRegion,
|
||||
MLSettings,
|
||||
PresentationReview,
|
||||
Tag,
|
||||
TagHead,
|
||||
TagKind,
|
||||
TagPositiveConfirmation,
|
||||
TagSuggestionRejection,
|
||||
)
|
||||
from ...models.tag import image_tag
|
||||
from ...models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
|
||||
from .training_data import (
|
||||
_AUTO_SOURCES,
|
||||
_auto_apply_point,
|
||||
_hygiene_excluded_ids,
|
||||
_ids_with_tag,
|
||||
_l2norm,
|
||||
_load_embeddings,
|
||||
@@ -62,6 +66,17 @@ _HEAD_KINDS = (TagKind.general, TagKind.character)
|
||||
# tag.kind -> the suggestion category the rail groups under.
|
||||
_CATEGORY = {TagKind.general: "general", TagKind.character: "character"}
|
||||
|
||||
# System-tag (wip/banner/editor screenshot) heads surface as suggestions at
|
||||
# this FLAT confidence floor instead of their auto-derived (precision-tuned)
|
||||
# suggest threshold. The auto threshold is high, so it hides the borderline /
|
||||
# false-positive guesses — which are exactly the cases the operator wants to
|
||||
# SEE and REJECT to sharpen these heads (hard-negative mining: "negatively
|
||||
# reinforce what isn't a system tag"). Operator-set 0.65 (2026-07-03): high
|
||||
# enough not to spam near-zero scores, low enough to surface real mistakes.
|
||||
# Content-tag heads keep their own thresholds; the typed-dropdown's
|
||||
# threshold_override still overrides everything (show-all mode).
|
||||
_SYSTEM_TAG_SUGGEST_FLOOR = 0.65
|
||||
|
||||
|
||||
class HeadTrainingAlreadyRunning(Exception):
|
||||
"""Raised by start_head_training_run when a run is already in flight."""
|
||||
@@ -125,42 +140,148 @@ def _embedder_version(session: Session) -> str:
|
||||
|
||||
|
||||
def _eligible_tag_ids(session: Session, min_pos: int) -> list[int]:
|
||||
"""Concept tags (general/character) with >= min_pos labelled images — the
|
||||
set that gets a head. Counts all sources; source-aware filtering (#1133) is
|
||||
a separate, optional refinement."""
|
||||
"""Concept tags (general/character) with >= min_pos POSITIVE images — the set
|
||||
that gets a head. Counts human-applied + operator-confirmed tags only;
|
||||
unconfirmed auto-applied predictions do NOT count toward eligibility (they
|
||||
don't train the head — milestone 139), so a concept can't graduate on its own
|
||||
guesses."""
|
||||
confirmed = exists().where(
|
||||
TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
|
||||
TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
|
||||
)
|
||||
rows = session.execute(
|
||||
select(Tag.id)
|
||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||
.where(Tag.kind.in_(_HEAD_KINDS))
|
||||
.where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed)
|
||||
.group_by(Tag.id)
|
||||
.having(func.count(image_tag.c.image_record_id) >= min_pos)
|
||||
).all()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
def _head_fingerprints(session: Session, tag_ids: list[int]) -> dict[int, str]:
|
||||
"""Per-tag training-data fingerprint: (positive count, latest positive
|
||||
created_at) + (rejection count, latest rejected_at). It moves whenever a tag
|
||||
gains/loses a positive or a rejection — the incremental-retrain change
|
||||
detector (#1317 p2). A newly-added positive/rejection always has the latest
|
||||
timestamp, so even a remove-one-add-one (unchanged count) is caught. The
|
||||
sampled-unlabeled negative pool + the hygiene set drift GLOBALLY and are
|
||||
reconciled by the nightly full run, not captured here."""
|
||||
if not tag_ids:
|
||||
return {}
|
||||
pos = session.execute(
|
||||
select(
|
||||
image_tag.c.tag_id,
|
||||
func.count(image_tag.c.image_record_id),
|
||||
func.max(image_tag.c.created_at),
|
||||
)
|
||||
.where(image_tag.c.tag_id.in_(tag_ids))
|
||||
.group_by(image_tag.c.tag_id)
|
||||
).all()
|
||||
pos_map = {t: (c, m) for t, c, m in pos}
|
||||
rej = session.execute(
|
||||
select(
|
||||
TagSuggestionRejection.tag_id,
|
||||
func.count(),
|
||||
func.max(TagSuggestionRejection.rejected_at),
|
||||
)
|
||||
.where(TagSuggestionRejection.tag_id.in_(tag_ids))
|
||||
.group_by(TagSuggestionRejection.tag_id)
|
||||
).all()
|
||||
rej_map = {t: (c, m) for t, c, m in rej}
|
||||
# Confirmations promote an auto-applied tag to a positive (milestone 139), so
|
||||
# a confirm must move the fingerprint too — else a manual Retrain right after
|
||||
# confirming wouldn't fold the tag in (the nightly full run would).
|
||||
conf = session.execute(
|
||||
select(TagPositiveConfirmation.tag_id, func.count())
|
||||
.where(TagPositiveConfirmation.tag_id.in_(tag_ids))
|
||||
.group_by(TagPositiveConfirmation.tag_id)
|
||||
).all()
|
||||
conf_map = dict(conf)
|
||||
out = {}
|
||||
for t in tag_ids:
|
||||
pc, pm = pos_map.get(t, (0, None))
|
||||
rc, rm = rej_map.get(t, (0, None))
|
||||
out[t] = f"{pc}:{pm}:{rc}:{rm}:{conf_map.get(t, 0)}"
|
||||
return out
|
||||
|
||||
|
||||
def _heads_needing_retrain(
|
||||
session: Session, eligible: list[int], embedding_version: str,
|
||||
fps: dict[int, str], full: bool,
|
||||
) -> list[int]:
|
||||
"""The eligible tag_ids to (re)fit: no head yet, a head trained in a DIFFERENT
|
||||
embedding space (a model swap), or a changed training-data fingerprint.
|
||||
full=True forces every eligible tag. sklearn-free (train_head itself needs
|
||||
scikit-learn) so the incremental decision is unit-testable on its own."""
|
||||
if full:
|
||||
return list(eligible)
|
||||
existing = {
|
||||
tag_id: (fp, ev)
|
||||
for tag_id, fp, ev in session.execute(
|
||||
select(
|
||||
TagHead.tag_id, TagHead.train_fingerprint,
|
||||
TagHead.embedding_version,
|
||||
)
|
||||
).all()
|
||||
}
|
||||
out = []
|
||||
for tag_id in eligible:
|
||||
prev = existing.get(tag_id)
|
||||
if (
|
||||
prev is None
|
||||
or prev[1] != embedding_version
|
||||
or prev[0] != fps.get(tag_id)
|
||||
):
|
||||
out.append(tag_id)
|
||||
return out
|
||||
|
||||
|
||||
def train_all_heads(
|
||||
session: Session, params: dict[str, Any], run: HeadTrainingRun | None = None
|
||||
) -> dict[str, int]:
|
||||
"""(Re)train a head for every eligible concept; prune heads whose tag is no
|
||||
longer eligible. Commits per head so a SIGKILL leaves trained heads durable
|
||||
(training is idempotent). Returns {n_trained, n_skipped}."""
|
||||
"""(Re)train eligible concept heads, INCREMENTALLY by default (#1317 p2):
|
||||
refit only the tags whose training data changed since last fit, so a manual
|
||||
Retrain click is fast. `params["full"]=True` (the nightly run) refits every
|
||||
head to reconcile sampled-negative + hygiene drift. Prunes heads whose tag is
|
||||
no longer eligible. Commits per head so a SIGKILL leaves trained heads durable.
|
||||
Returns {n_trained, n_skipped} (n_skipped = unchanged + too-few-examples)."""
|
||||
import numpy as np
|
||||
|
||||
cfg = _normalize_params(session, params)
|
||||
embedding_version = _embedder_version(session)
|
||||
full = bool((params or {}).get("full"))
|
||||
eligible = _eligible_tag_ids(session, cfg["min_positives"])
|
||||
eligible_set = set(eligible)
|
||||
# Computed once per run, not per head — the hygiene set is identical for
|
||||
# every non-system concept.
|
||||
hygiene = _hygiene_excluded_ids(session)
|
||||
fps = _head_fingerprints(session, eligible)
|
||||
to_train = set(
|
||||
_heads_needing_retrain(session, eligible, embedding_version, fps, full)
|
||||
)
|
||||
trained = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
for i, tag_id in enumerate(eligible):
|
||||
if tag_id not in to_train:
|
||||
continue
|
||||
try:
|
||||
ok = train_head(session, tag_id, embedding_version, cfg, np)
|
||||
ok = train_head(
|
||||
session, tag_id, embedding_version, cfg, np, hygiene=hygiene
|
||||
)
|
||||
except Exception:
|
||||
log.exception("train_head failed for tag %d", tag_id)
|
||||
ok = False
|
||||
if ok:
|
||||
# Stamp the fingerprint we trained against so an unchanged tag is
|
||||
# skipped on the next incremental run.
|
||||
head = session.get(TagHead, tag_id)
|
||||
if head is not None:
|
||||
head.train_fingerprint = fps.get(tag_id)
|
||||
session.commit()
|
||||
trained += int(ok)
|
||||
skipped += int(not ok)
|
||||
failed += int(not ok)
|
||||
if run is not None and i % 10 == 0:
|
||||
run.last_progress_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
@@ -171,30 +292,63 @@ def train_all_heads(
|
||||
else:
|
||||
session.execute(delete(TagHead))
|
||||
session.commit()
|
||||
return {"n_trained": trained, "n_skipped": skipped}
|
||||
# n_skipped = unchanged (not attempted) + failed-to-fit (too few examples).
|
||||
return {
|
||||
"n_trained": trained,
|
||||
"n_skipped": (len(eligible) - len(to_train)) + failed,
|
||||
}
|
||||
|
||||
|
||||
def head_training_ids(
|
||||
session: Session, tag_id: int, cfg: dict, hygiene: set[int] | None = None,
|
||||
) -> tuple[list[int], list[int]] | None:
|
||||
"""Select (pos_ids, neg_ids) for one head. Split out of train_head and
|
||||
kept sklearn-free so the hygiene exclusion is testable in the CI env
|
||||
(sklearn only exists in the ml image). Returns None when the concept has
|
||||
too few usable positives.
|
||||
|
||||
Training hygiene (#128): images carrying a system tag are ABSENT from
|
||||
every other concept's training — dropped as positives AND kept out of
|
||||
the rejection/sampled negative pool (see _hygiene_excluded_ids). A system
|
||||
tag's own head trains on them unfiltered: its positives ARE the hygiene
|
||||
images."""
|
||||
tag = session.get(Tag, tag_id)
|
||||
if tag is not None and tag.is_system:
|
||||
hygiene = set()
|
||||
elif hygiene is None:
|
||||
hygiene = _hygiene_excluded_ids(session)
|
||||
|
||||
pos_ids = [i for i in _ids_with_tag(session, tag_id) if i not in hygiene]
|
||||
if len(pos_ids) < cfg["min_positives"]:
|
||||
return None
|
||||
|
||||
pos_set = set(pos_ids)
|
||||
rejected = [
|
||||
i for i in _rejected_ids(session, tag_id)
|
||||
if i not in pos_set and i not in hygiene
|
||||
]
|
||||
want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4)
|
||||
sampled = _sample_unlabeled(
|
||||
session, pos_set | set(rejected) | hygiene,
|
||||
min(_UNLABELED_POOL, want_neg),
|
||||
)
|
||||
return pos_ids, rejected + [i for i in sampled if i not in pos_set]
|
||||
|
||||
|
||||
def train_head(
|
||||
session: Session, tag_id: int, embedding_version: str, cfg: dict, np
|
||||
session: Session, tag_id: int, embedding_version: str, cfg: dict, np,
|
||||
hygiene: set[int] | None = None,
|
||||
) -> bool:
|
||||
"""Fit + upsert one head. Returns True if a head was written, False if the
|
||||
concept had too few usable examples to train (the row is then removed)."""
|
||||
from sklearn.linear_model import LogisticRegression
|
||||
from sklearn.model_selection import StratifiedKFold, cross_val_predict
|
||||
|
||||
pos_ids = _ids_with_tag(session, tag_id)
|
||||
if len(pos_ids) < cfg["min_positives"]:
|
||||
ids = head_training_ids(session, tag_id, cfg, hygiene)
|
||||
if ids is None:
|
||||
session.execute(delete(TagHead).where(TagHead.tag_id == tag_id))
|
||||
return False
|
||||
|
||||
pos_set = set(pos_ids)
|
||||
rejected = [i for i in _rejected_ids(session, tag_id) if i not in pos_set]
|
||||
want_neg = max(len(pos_ids) * cfg["neg_ratio"], _EXAMPLES_MIN * 4)
|
||||
sampled = _sample_unlabeled(
|
||||
session, pos_set | set(rejected), min(_UNLABELED_POOL, want_neg)
|
||||
)
|
||||
neg_ids = rejected + [i for i in sampled if i not in pos_set]
|
||||
|
||||
pos_ids, neg_ids = ids
|
||||
emb = _load_embeddings(session, pos_ids + neg_ids)
|
||||
pos = [emb[i] for i in pos_ids if i in emb]
|
||||
neg = [emb[i] for i in neg_ids if i in emb]
|
||||
@@ -262,7 +416,7 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
TagHead.tag_id, Tag.name, Tag.kind,
|
||||
TagHead.tag_id, Tag.name, Tag.kind, Tag.is_system,
|
||||
TagHead.weights, TagHead.bias,
|
||||
TagHead.suggest_threshold, TagHead.auto_apply_threshold,
|
||||
)
|
||||
@@ -280,8 +434,12 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
|
||||
{
|
||||
"tag_id": r.tag_id,
|
||||
"name": r.name,
|
||||
"category": _CATEGORY.get(r.kind, "general"),
|
||||
# System tags (wip/banner/editor) are kind=general but group under
|
||||
# their OWN "system" suggestion category so the operator reviews
|
||||
# them apart from content tags (they still train as general heads).
|
||||
"category": "system" if r.is_system else _CATEGORY.get(r.kind, "general"),
|
||||
"auto_apply_threshold": r.auto_apply_threshold,
|
||||
"is_system": bool(r.is_system),
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
@@ -291,14 +449,70 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
|
||||
return loaded
|
||||
|
||||
|
||||
async def _image_bag(
|
||||
session: AsyncSession, image_id: int, cur_version: str,
|
||||
) -> tuple[list, list[dict | None]]:
|
||||
"""The max-over-bag inputs for one image: the whole-image SigLIP vector (when
|
||||
it's in the current model's space) PLUS every concept-region crop embedded in
|
||||
that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for
|
||||
the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag
|
||||
can point back at the crop that produced it (#1206 grounding).
|
||||
|
||||
Only current-version embeddings enter the bag: mid model-swap (#1190) an image
|
||||
still carrying an OLD-version whole-image vector is skipped rather than scored
|
||||
by heads trained in a different space; a legacy NULL version is treated as
|
||||
current (those predate per-row stamping). Shared by live scoring (score_image)
|
||||
and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4)."""
|
||||
import numpy as np
|
||||
|
||||
img = await session.get(ImageRecord, image_id)
|
||||
bag: list = []
|
||||
bag_meta: list[dict | None] = []
|
||||
if img is None:
|
||||
return bag, bag_meta
|
||||
if img.siglip_embedding is not None and img.siglip_model_version in (
|
||||
cur_version, None,
|
||||
):
|
||||
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
|
||||
bag_meta.append(None)
|
||||
region_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
ImageRegion.siglip_embedding,
|
||||
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
|
||||
ImageRegion.kind, ImageRegion.detector_version,
|
||||
)
|
||||
.where(ImageRegion.image_record_id == image_id)
|
||||
.where(ImageRegion.siglip_embedding.is_not(None))
|
||||
.where(ImageRegion.embedding_version == cur_version)
|
||||
)
|
||||
).all()
|
||||
for vec, rx, ry, rw, rh, kind, detector in region_rows:
|
||||
if vec is not None:
|
||||
bag.append(np.asarray(vec, dtype=np.float32))
|
||||
bag_meta.append(
|
||||
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
|
||||
)
|
||||
return bag, bag_meta
|
||||
|
||||
|
||||
async def score_image(
|
||||
session: AsyncSession, image_id: int, threshold_override: float | None = None,
|
||||
) -> list[dict]:
|
||||
"""Suggestions for one image from the trained heads: [{tag_id, name,
|
||||
category, score}], ranked. A concept surfaces when its score clears the
|
||||
head's own suggest_threshold — or, when threshold_override is given (the
|
||||
typed-dropdown "show everything" mode), that flat floor instead (0 → every
|
||||
head). Empty if the image has no embedding or no heads exist yet.
|
||||
category, score, above_threshold, grounding}], ranked. A concept is INCLUDED
|
||||
when its score clears the head's own suggest_threshold — or, when
|
||||
threshold_override is given (the typed-dropdown "show everything" mode), that
|
||||
flat floor instead (0 → every head). System-tag heads (wip/banner/editor)
|
||||
instead use a flat _SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface
|
||||
for rejection (still overridden by threshold_override). Empty if the image has
|
||||
no embedding or no heads exist yet.
|
||||
|
||||
``above_threshold`` is reported SEPARATELY from inclusion: it's always whether
|
||||
the score cleared the head's NATURAL cut (suggest_threshold, or the system
|
||||
floor), regardless of any override. So the single min=0 fetch returns every
|
||||
head, and the caller can split panel (above_threshold) from dropdown (all)
|
||||
without a second request.
|
||||
|
||||
MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
|
||||
vector PLUS every concept-region crop the agent embedded (same model
|
||||
@@ -308,35 +522,12 @@ async def score_image(
|
||||
always in the bag, so this can never score lower than whole-image alone."""
|
||||
import numpy as np
|
||||
|
||||
img = await session.get(ImageRecord, image_id)
|
||||
if img is None:
|
||||
return []
|
||||
settings = await _settings_async(session)
|
||||
cur_version = settings.embedder_model_version
|
||||
heads = await _current_heads(session, cur_version)
|
||||
if heads["W"] is None:
|
||||
return []
|
||||
|
||||
# Only embeddings in the CURRENT model's space enter the bag. Mid model-swap
|
||||
# (#1190), an image still carrying the OLD-version whole-image vector is
|
||||
# skipped rather than scored by heads trained in a different space; a legacy
|
||||
# NULL version is treated as current (those predate per-row stamping).
|
||||
bag = []
|
||||
if img.siglip_embedding is not None and img.siglip_model_version in (
|
||||
cur_version, None,
|
||||
):
|
||||
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
|
||||
region_vecs = (
|
||||
await session.execute(
|
||||
select(ImageRegion.siglip_embedding)
|
||||
.where(ImageRegion.image_record_id == image_id)
|
||||
.where(ImageRegion.siglip_embedding.is_not(None))
|
||||
.where(ImageRegion.embedding_version == cur_version)
|
||||
)
|
||||
).all()
|
||||
for (vec,) in region_vecs:
|
||||
if vec is not None:
|
||||
bag.append(np.asarray(vec, dtype=np.float32))
|
||||
bag, bag_meta = await _image_bag(session, image_id, cur_version)
|
||||
if not bag:
|
||||
return []
|
||||
|
||||
@@ -345,22 +536,83 @@ async def score_image(
|
||||
norms[norms == 0] = 1.0
|
||||
Xn = X / norms
|
||||
Z = Xn @ heads["W"].T + heads["b"] # (B, H)
|
||||
probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag
|
||||
probs_bag = 1.0 / (1.0 + np.exp(-Z)) # (B, H)
|
||||
probs = probs_bag.max(axis=0) # (H,) best over the bag
|
||||
# ARGMAX beside the max: WHICH bag row won each head → the region that grounds
|
||||
# the tag (bag_meta[win]); None when the whole-image vector won (#1206).
|
||||
winners = probs_bag.argmax(axis=0) # (H,)
|
||||
out = []
|
||||
for i, p in enumerate(probs):
|
||||
cut = threshold_override if threshold_override is not None else heads["thr"][i]
|
||||
m = heads["meta"][i]
|
||||
# The head's NATURAL suggest cut — system tags use the flat floor (see
|
||||
# _SYSTEM_TAG_SUGGEST_FLOOR) so their false positives show up for the
|
||||
# operator to reject; content heads use their own precision-tuned
|
||||
# threshold. This is what "above threshold" means (drives the panel).
|
||||
natural = (
|
||||
_SYSTEM_TAG_SUGGEST_FLOOR if m["is_system"] else float(heads["thr"][i])
|
||||
)
|
||||
# INCLUSION is looser under threshold_override (dropdown show-all,
|
||||
# override=0): every head comes back so a low-confidence concept can still
|
||||
# be typed + picked, each carrying its own above_threshold flag.
|
||||
cut = threshold_override if threshold_override is not None else natural
|
||||
if p >= cut:
|
||||
m = heads["meta"][i]
|
||||
out.append({
|
||||
"tag_id": m["tag_id"],
|
||||
"name": m["name"],
|
||||
"category": m["category"],
|
||||
"score": float(p),
|
||||
"above_threshold": bool(p >= natural),
|
||||
"grounding": bag_meta[int(winners[i])],
|
||||
})
|
||||
out.sort(key=lambda d: d["score"], reverse=True)
|
||||
return out
|
||||
|
||||
|
||||
async def ground_applied_tag(
|
||||
session: AsyncSession, image_id: int, tag_id: int,
|
||||
) -> tuple[dict | None, bool]:
|
||||
"""On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags
|
||||
aren't scored live, so recompute the max-over-bag argmax for just this tag's
|
||||
head — which crop region best explains the tag on this image — mirroring what
|
||||
score_image records for live suggestions. Returns (grounding, has_head):
|
||||
|
||||
- has_head False → the tag has no head in the current embedding space (manual/
|
||||
artist/meta tags, or a concept below the head floor). Nothing to localize
|
||||
with, so the UI shows no overlay (distinct from "the whole image won").
|
||||
- grounding None (has_head True) → the whole-image vector best explains it,
|
||||
not any crop; the UI shows the subtle whole-image frame.
|
||||
- grounding {bbox, kind, detector} → the winning region.
|
||||
|
||||
Character heads are covered too (character is a head kind); this deliberately
|
||||
reuses the SigLIP head bag rather than the CCIP figure path so every applied
|
||||
concept grounds through one consistent mechanism."""
|
||||
import numpy as np
|
||||
|
||||
cur_version = (await _settings_async(session)).embedder_model_version
|
||||
row = (
|
||||
await session.execute(
|
||||
select(TagHead.weights, TagHead.bias).where(
|
||||
TagHead.tag_id == tag_id,
|
||||
TagHead.embedding_version == cur_version,
|
||||
)
|
||||
)
|
||||
).one_or_none()
|
||||
if row is None:
|
||||
return None, False
|
||||
bag, bag_meta = await _image_bag(session, image_id, cur_version)
|
||||
if not bag:
|
||||
return None, True
|
||||
|
||||
X = np.vstack(bag)
|
||||
norms = np.linalg.norm(X, axis=1, keepdims=True)
|
||||
norms[norms == 0] = 1.0
|
||||
Xn = X / norms
|
||||
# The sigmoid is monotonic in the logit, so the highest-probability bag row is
|
||||
# just argmax of the raw score — no need to exponentiate to pick the winner.
|
||||
z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias) # (B,)
|
||||
return bag_meta[int(z.argmax())], True
|
||||
|
||||
|
||||
async def _settings_async(session: AsyncSession) -> MLSettings:
|
||||
return (
|
||||
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||
@@ -408,8 +660,11 @@ def start_head_auto_apply_run(session: Session, params: dict[str, Any]) -> int:
|
||||
|
||||
|
||||
def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
|
||||
"""Eligible heads to fire: graduated (auto_apply_threshold set), enough
|
||||
support, current embedding. Returns the row list (tag_id/name/weights/...)."""
|
||||
"""Eligible CONTENT heads to fire: graduated (auto_apply_threshold set),
|
||||
enough support, current embedding, NON-system. System tags never auto-apply
|
||||
via this path — `wip` never auto-applies at all, and banner/editor screenshot
|
||||
go through the presentation path at their own flat threshold (#141). Returns
|
||||
the row list (tag_id/name/weights/...)."""
|
||||
return session.execute(
|
||||
select(
|
||||
TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias,
|
||||
@@ -419,6 +674,7 @@ def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
|
||||
.where(TagHead.embedding_version == embedding_version)
|
||||
.where(TagHead.auto_apply_threshold.is_not(None))
|
||||
.where(TagHead.n_pos >= min_pos)
|
||||
.where(~Tag.is_system)
|
||||
).all()
|
||||
|
||||
|
||||
@@ -500,3 +756,321 @@ def auto_apply_sweep(
|
||||
for h in range(len(rows))
|
||||
]
|
||||
return {"n_applied": sum(applied), "concepts": concepts}
|
||||
|
||||
|
||||
_PRESENTATION_SOURCE = "presentation_auto"
|
||||
_PROCESS_SOURCE = "process_auto"
|
||||
|
||||
# System-tag auto-apply modes (#1464). Both modes run the identical sweep — apply
|
||||
# a system tag at a flat threshold with a PROVISIONAL source + a ring-loud review
|
||||
# guard — and differ ONLY in which tags, which settings knobs, and which
|
||||
# source/review-mode. 'chrome' (banner) is HIDDEN from the gallery; 'process'
|
||||
# (wip / editor screenshot) stays VISIBLE (the hide is a gallery-query effect of
|
||||
# the tag's group membership, not of this sweep).
|
||||
_SWEEP_MODES = {
|
||||
"chrome": {
|
||||
"names": CHROME_SYSTEM_TAGS,
|
||||
"enabled": "presentation_auto_apply_enabled",
|
||||
"threshold": "presentation_auto_apply_threshold",
|
||||
"conflict": "presentation_conflict_threshold",
|
||||
"source": _PRESENTATION_SOURCE,
|
||||
},
|
||||
"process": {
|
||||
"names": PROCESS_SYSTEM_TAGS,
|
||||
"enabled": "process_auto_apply_enabled",
|
||||
"threshold": "process_auto_apply_threshold",
|
||||
"conflict": "process_conflict_threshold",
|
||||
"source": _PROCESS_SOURCE,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _system_tag_heads(session: Session, embedding_version: str, names):
|
||||
"""Trained heads for a system-tag group (chrome banner / process wip+editor).
|
||||
They fire at the group's FLAT threshold regardless of graduation — a head
|
||||
exists once the operator has labelled enough (head_min_positives)."""
|
||||
return session.execute(
|
||||
select(TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias)
|
||||
.join(Tag, Tag.id == TagHead.tag_id)
|
||||
.where(TagHead.embedding_version == embedding_version)
|
||||
.where(Tag.is_system.is_(True))
|
||||
.where(Tag.name.in_(names))
|
||||
).all()
|
||||
|
||||
|
||||
def _conflict_heads(session: Session, embedding_version: str):
|
||||
"""ALL content (non-system) heads — the "does this ALSO look like real
|
||||
content" signal for the presentation conflict guard (#141)."""
|
||||
return session.execute(
|
||||
select(TagHead.tag_id, TagHead.weights, TagHead.bias)
|
||||
.join(Tag, Tag.id == TagHead.tag_id)
|
||||
.where(TagHead.embedding_version == embedding_version)
|
||||
.where(~Tag.is_system)
|
||||
).all()
|
||||
|
||||
|
||||
def _valued_image_ids(session: Session) -> set[int]:
|
||||
"""Images the operator has shown they value: carrying a HUMAN or CONFIRMED
|
||||
content (non-system) tag. The presentation sweep never auto-hides these
|
||||
(guard 1) — you tagged it, so the model doesn't get to bury it (#141)."""
|
||||
confirmed = exists().where(
|
||||
TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
|
||||
TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
|
||||
)
|
||||
rows = session.execute(
|
||||
select(image_tag.c.image_record_id)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(~Tag.is_system)
|
||||
.where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed)
|
||||
).all()
|
||||
return {r[0] for r in rows}
|
||||
|
||||
|
||||
def system_tag_auto_apply_sweep(
|
||||
session: Session, *, mode: str, dry_run: bool = False
|
||||
) -> dict:
|
||||
"""Auto-apply a system-tag group at its FLAT threshold. mode='chrome' (banner,
|
||||
#141) hides the image; mode='process' (wip / editor screenshot, #1464) keeps it
|
||||
VISIBLE — the ONLY difference is the tag group's gallery membership, not this
|
||||
sweep. Two guards keep it safe: (1) never touch an image carrying a
|
||||
human/confirmed content tag; (2) if the image ALSO scores >= the conflict
|
||||
threshold on a content head, still apply but flag it (PresentationReview,
|
||||
mode=<mode>) so the review strip surfaces "also looks like <X>". The source is
|
||||
PROVISIONAL so the head never trains on its own output. No-op unless the mode's
|
||||
enabled flag is set. numpy-only (no sklearn). Returns {n_applied, n_flagged,
|
||||
concepts}."""
|
||||
import numpy as np
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
cfg = _SWEEP_MODES[mode]
|
||||
settings = _settings(session)
|
||||
if not dry_run and not getattr(settings, cfg["enabled"]):
|
||||
return {"n_applied": 0, "n_flagged": 0, "concepts": []}
|
||||
ver = settings.embedder_model_version
|
||||
pres = _system_tag_heads(session, ver, cfg["names"])
|
||||
if not pres:
|
||||
return {"n_applied": 0, "n_flagged": 0, "concepts": []}
|
||||
thr = float(getattr(settings, cfg["threshold"]))
|
||||
conflict_thr = float(getattr(settings, cfg["conflict"]))
|
||||
source = cfg["source"]
|
||||
|
||||
Wp = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in pres])
|
||||
bp = np.asarray([r.bias for r in pres], dtype=np.float32)
|
||||
pres_tag_ids = [r.tag_id for r in pres]
|
||||
pres_names = [r.name for r in pres]
|
||||
|
||||
conf = _conflict_heads(session, ver)
|
||||
Wc = bc = conf_tag_ids = None
|
||||
if conf:
|
||||
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
|
||||
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
|
||||
conf_tag_ids = [r.tag_id for r in conf]
|
||||
|
||||
valued = _valued_image_ids(session)
|
||||
|
||||
# Skip images that already carry, or have rejected, each presentation tag.
|
||||
skip = {tid: set() for tid in pres_tag_ids}
|
||||
for tid in pres_tag_ids:
|
||||
for (iid,) in session.execute(
|
||||
select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tid)
|
||||
):
|
||||
skip[tid].add(iid)
|
||||
for (iid,) in session.execute(
|
||||
select(TagSuggestionRejection.image_record_id).where(
|
||||
TagSuggestionRejection.tag_id == tid
|
||||
)
|
||||
):
|
||||
skip[tid].add(iid)
|
||||
|
||||
applied = [0] * len(pres)
|
||||
n_flagged = 0
|
||||
scanned = 0
|
||||
all_ids = list(session.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_not(None))
|
||||
).scalars())
|
||||
for start in range(0, len(all_ids), _AUTO_APPLY_CHUNK):
|
||||
chunk = all_ids[start:start + _AUTO_APPLY_CHUNK]
|
||||
emb = _load_embeddings(session, chunk)
|
||||
cids = [i for i in chunk if i in emb]
|
||||
if not cids:
|
||||
continue
|
||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
||||
probs = 1.0 / (1.0 + np.exp(-(Xn @ Wp.T + bp))) # (N, P)
|
||||
if Wc is not None:
|
||||
cprobs = 1.0 / (1.0 + np.exp(-(Xn @ Wc.T + bc))) # (N, C)
|
||||
max_c = cprobs.max(axis=1)
|
||||
arg_c = cprobs.argmax(axis=1)
|
||||
scanned += len(cids)
|
||||
for p in range(len(pres)):
|
||||
tid = pres_tag_ids[p]
|
||||
for idx in np.where(probs[:, p] >= thr)[0]:
|
||||
iid = cids[int(idx)]
|
||||
if iid in skip[tid] or iid in valued:
|
||||
continue
|
||||
skip[tid].add(iid)
|
||||
applied[p] += 1
|
||||
if not dry_run:
|
||||
session.execute(
|
||||
pg_insert(image_tag)
|
||||
.values(
|
||||
image_record_id=iid, tag_id=tid,
|
||||
source=source,
|
||||
)
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
# Guard 2: also looks like real content → still apply, but flag it
|
||||
# for the review strip instead of silently marking (chrome hides,
|
||||
# process stays visible — either way the operator gets a heads-up).
|
||||
if Wc is not None and float(max_c[idx]) >= conflict_thr:
|
||||
n_flagged += 1
|
||||
if not dry_run:
|
||||
session.execute(
|
||||
pg_insert(PresentationReview)
|
||||
.values(
|
||||
image_record_id=iid, tag_id=tid,
|
||||
conflict_tag_id=conf_tag_ids[int(arg_c[idx])],
|
||||
conflict_score=float(max_c[idx]),
|
||||
mode=mode,
|
||||
)
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
if not dry_run:
|
||||
session.commit()
|
||||
|
||||
concepts = [
|
||||
{"tag_id": pres_tag_ids[p], "name": pres_names[p],
|
||||
"applied": applied[p], "scanned": scanned, "threshold": thr}
|
||||
for p in range(len(pres))
|
||||
]
|
||||
return {
|
||||
"n_applied": sum(applied), "n_flagged": n_flagged, "concepts": concepts,
|
||||
}
|
||||
|
||||
|
||||
def soft_wip_conflict_audit(session: Session, dry_run: bool = False) -> dict:
|
||||
"""Ring-loud audit for the SOFT WIP-title cohort (#1474). Images auto-tagged
|
||||
`wip` from a low-precision sketch/doodle title (source='wip_title_soft') that ALSO
|
||||
score >= the process conflict threshold on a content head are probably FINISHED
|
||||
art mis-tagged as process — flag them (PresentationReview, mode='process') so the
|
||||
review strip surfaces them ("also looks like <X>", Keep tag / Remove tag). Does
|
||||
NOT remove the tag; the operator decides. No-op when there are no content heads.
|
||||
numpy-only. Returns {n_scanned, n_flagged}."""
|
||||
import numpy as np
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..wip_title import WIP_TITLE_SOFT_SOURCE, resolve_wip_tag_id
|
||||
|
||||
settings = _settings(session)
|
||||
ver = settings.embedder_model_version
|
||||
conflict_thr = float(settings.process_conflict_threshold)
|
||||
conf = _conflict_heads(session, ver)
|
||||
wip_id = resolve_wip_tag_id(session)
|
||||
if not conf or wip_id is None:
|
||||
return {"n_scanned": 0, "n_flagged": 0}
|
||||
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
|
||||
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
|
||||
conf_tag_ids = [r.tag_id for r in conf]
|
||||
|
||||
soft_ids = [iid for (iid,) in session.execute(
|
||||
select(image_tag.c.image_record_id)
|
||||
.where(image_tag.c.tag_id == wip_id)
|
||||
.where(image_tag.c.source == WIP_TITLE_SOFT_SOURCE)
|
||||
)]
|
||||
# Skip images already flagged for this tag (idempotent re-runs).
|
||||
flagged = {iid for (iid,) in session.execute(
|
||||
select(PresentationReview.image_record_id)
|
||||
.where(PresentationReview.tag_id == wip_id)
|
||||
)}
|
||||
soft_ids = [i for i in soft_ids if i not in flagged]
|
||||
|
||||
n_flagged = 0
|
||||
scanned = 0
|
||||
for start in range(0, len(soft_ids), _AUTO_APPLY_CHUNK):
|
||||
chunk = soft_ids[start:start + _AUTO_APPLY_CHUNK]
|
||||
emb = _load_embeddings(session, chunk)
|
||||
cids = [i for i in chunk if i in emb]
|
||||
if not cids:
|
||||
continue
|
||||
scanned += len(cids)
|
||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
||||
cprobs = 1.0 / (1.0 + np.exp(-(Xn @ Wc.T + bc)))
|
||||
max_c = cprobs.max(axis=1)
|
||||
arg_c = cprobs.argmax(axis=1)
|
||||
for k in range(len(cids)):
|
||||
if float(max_c[k]) >= conflict_thr:
|
||||
n_flagged += 1
|
||||
if not dry_run:
|
||||
session.execute(
|
||||
pg_insert(PresentationReview)
|
||||
.values(
|
||||
image_record_id=cids[k], tag_id=wip_id,
|
||||
conflict_tag_id=conf_tag_ids[int(arg_c[k])],
|
||||
conflict_score=float(max_c[k]),
|
||||
mode="process",
|
||||
)
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
if not dry_run:
|
||||
session.commit()
|
||||
return {"n_scanned": scanned, "n_flagged": n_flagged}
|
||||
|
||||
|
||||
def retract_auto_applied_heads(session: Session) -> int:
|
||||
"""Soft auto-apply (milestone 139): re-score every standing source='head_auto'
|
||||
tag against its CURRENT head and REMOVE the ones now BELOW the head's
|
||||
auto_apply_threshold — i.e. the head sharpened (or the operator raised the bar)
|
||||
and no longer supports them. Skips operator-confirmed tags
|
||||
(TagPositiveConfirmation). SILENT: a low score isn't proof the tag was wrong,
|
||||
so no hard negative is recorded — that's reserved for an operator removal.
|
||||
No-op unless head_auto_apply_enabled. Only re-scores the images that ALREADY
|
||||
carry the auto-tag (bounded), never the whole library. Returns n_retracted."""
|
||||
import numpy as np
|
||||
|
||||
settings = _settings(session)
|
||||
if not settings.head_auto_apply_enabled:
|
||||
return 0
|
||||
heads = session.execute(
|
||||
select(
|
||||
TagHead.tag_id, TagHead.weights, TagHead.bias,
|
||||
TagHead.auto_apply_threshold,
|
||||
)
|
||||
.where(TagHead.embedding_version == settings.embedder_model_version)
|
||||
.where(TagHead.auto_apply_threshold.is_not(None))
|
||||
).all()
|
||||
retracted = 0
|
||||
for tag_id, weights, bias, thr in heads:
|
||||
auto_ids = [
|
||||
iid for (iid,) in session.execute(
|
||||
select(image_tag.c.image_record_id)
|
||||
.where(image_tag.c.tag_id == tag_id)
|
||||
.where(image_tag.c.source == "head_auto")
|
||||
)
|
||||
]
|
||||
if not auto_ids:
|
||||
continue
|
||||
confirmed = {
|
||||
iid for (iid,) in session.execute(
|
||||
select(TagPositiveConfirmation.image_record_id)
|
||||
.where(TagPositiveConfirmation.tag_id == tag_id)
|
||||
.where(TagPositiveConfirmation.image_record_id.in_(auto_ids))
|
||||
)
|
||||
}
|
||||
candidates = [i for i in auto_ids if i not in confirmed]
|
||||
emb = _load_embeddings(session, candidates)
|
||||
cids = [i for i in candidates if i in emb]
|
||||
if not cids:
|
||||
continue
|
||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
||||
w = np.asarray(weights, dtype=np.float32)
|
||||
probs = 1.0 / (1.0 + np.exp(-(Xn @ w + float(bias))))
|
||||
below = [cids[k] for k in np.where(probs < float(thr))[0]]
|
||||
for iid in below:
|
||||
session.execute(
|
||||
image_tag.delete()
|
||||
.where(image_tag.c.image_record_id == iid)
|
||||
.where(image_tag.c.tag_id == tag_id)
|
||||
.where(image_tag.c.source == "head_auto")
|
||||
)
|
||||
retracted += 1
|
||||
session.commit()
|
||||
return retracted
|
||||
|
||||
@@ -22,27 +22,30 @@ from .heads import score_image
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Suggestion:
|
||||
# canonical_tag_id is None when this is a raw Camie tag with no alias and
|
||||
# no existing Tag row — accepting it will create the tag.
|
||||
canonical_tag_id: int | None
|
||||
# Every suggestion is a canonical Tag: heads/CCIP only score EXISTING concept
|
||||
# tags (tagging-v2, #114). The old raw-model-key / creates-new / alias-remap
|
||||
# cases are gone — a suggestion always maps to a real tag id.
|
||||
canonical_tag_id: int
|
||||
display_name: str
|
||||
category: str
|
||||
score: float
|
||||
source: str # 'head' | 'ccip' | 'both' (Camie tagger/centroid removed in v2)
|
||||
creates_new_tag: bool
|
||||
# raw_name = the booru model vocab key behind this suggestion. It's the key
|
||||
# an alias MUST be stored under (resolution looks up the raw key), so the
|
||||
# modal needs it to author an alias correctly. None for centroid-only hits
|
||||
# (no underlying prediction → nothing to alias).
|
||||
raw_name: str | None = None
|
||||
# via_alias = this suggestion was surfaced because an operator alias remapped
|
||||
# the raw prediction to this canonical tag. Lets the UI mark it + offer undo.
|
||||
via_alias: bool = False
|
||||
# above_threshold = the score cleared the head's own suggest cut (or the
|
||||
# system floor). The Suggestions PANEL shows only these; the typed-tag
|
||||
# dropdown fetches ALL suggestions (every head, min=0) and just annotates each
|
||||
# matching row with its score, so a low-confidence concept can still be typed
|
||||
# and picked. CCIP character matches are always above their match threshold.
|
||||
above_threshold: bool
|
||||
# rejected = the operator dismissed this tag for this image (a stored
|
||||
# TagSuggestionRejection). It stays in the list — flagged, not dropped — so
|
||||
# the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery,
|
||||
# operator-asked 2026-06-27) instead of silently vanishing or re-suggesting.
|
||||
rejected: bool = False
|
||||
# grounding = the crop region that produced this suggestion (#1206):
|
||||
# {bbox:[x,y,w,h] normalized, kind, detector}. None when the whole-image
|
||||
# vector won (not localized) or for a CCIP-only hit (figure grounding TBD).
|
||||
# Lets the rail highlight the exact region on hover.
|
||||
grounding: dict | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -103,6 +106,8 @@ class SuggestionService:
|
||||
for h in hits:
|
||||
merged[(h["category"], h["tag_id"])] = {
|
||||
"name": h["name"], "score": h["score"], "source": "head",
|
||||
"above_threshold": h["above_threshold"],
|
||||
"grounding": h.get("grounding"),
|
||||
}
|
||||
for c in ccip_hits:
|
||||
key = ("character", c["tag_id"])
|
||||
@@ -110,9 +115,17 @@ class SuggestionService:
|
||||
if ex is not None:
|
||||
ex["source"] = "both"
|
||||
ex["score"] = max(ex["score"], c["score"])
|
||||
# CCIP only returns matches above its own threshold, so a CCIP
|
||||
# corroboration always makes the merged suggestion above-threshold.
|
||||
ex["above_threshold"] = True
|
||||
# Keep the head's localized crop if it had one; else fall back to
|
||||
# the CCIP figure so a corroborated character still grounds (#1206).
|
||||
ex["grounding"] = ex.get("grounding") or c.get("grounding")
|
||||
else:
|
||||
merged[key] = {
|
||||
"name": c["name"], "score": c["score"], "source": "ccip",
|
||||
"above_threshold": True,
|
||||
"grounding": c.get("grounding"),
|
||||
}
|
||||
|
||||
result = SuggestionList()
|
||||
@@ -126,8 +139,9 @@ class SuggestionService:
|
||||
category=cat,
|
||||
score=m["score"],
|
||||
source=m["source"],
|
||||
creates_new_tag=False,
|
||||
above_threshold=m["above_threshold"],
|
||||
rejected=tag_id in rejected,
|
||||
grounding=m.get("grounding"),
|
||||
)
|
||||
)
|
||||
for cat in result.by_category:
|
||||
@@ -146,8 +160,7 @@ class SuggestionService:
|
||||
was suggested for (or already applied to) >= threshold fraction of
|
||||
the selection AND was acceptable on >= 1 image. Confidence is the
|
||||
mean over images where it was suggested. Aggregated by
|
||||
canonical_tag_id; creates-new (no canonical id) suggestions are
|
||||
skipped (bulk Accept applies by tag id)."""
|
||||
canonical_tag_id (every suggestion is a canonical tag now)."""
|
||||
if not image_ids:
|
||||
return {}
|
||||
threshold = min(1.0, max(0.0, threshold))
|
||||
@@ -158,8 +171,6 @@ class SuggestionService:
|
||||
sl = await self.for_image(image_id)
|
||||
for category, items in sl.by_category.items():
|
||||
for s in items:
|
||||
if s.canonical_tag_id is None or s.creates_new_tag:
|
||||
continue
|
||||
# for_image keeps rejected tags (flagged) for the rail;
|
||||
# bulk consensus must still ignore them — a tag dismissed on
|
||||
# an image isn't a suggestion for that image.
|
||||
|
||||
@@ -17,14 +17,70 @@ from typing import Any
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ...models import ImageRecord, TagSuggestionRejection
|
||||
from ...models import (
|
||||
ImageRecord,
|
||||
Tag,
|
||||
TagPositiveConfirmation,
|
||||
TagSuggestionRejection,
|
||||
)
|
||||
from ...models.tag import image_tag
|
||||
|
||||
# Auto-apply sources whose tags are PROVISIONAL: they never train a head (or seed
|
||||
# a CCIP reference) unless the operator confirms them (milestone 139). Keeping
|
||||
# auto-applied predictions out of training is what makes them "soft" — a misfire
|
||||
# can't reinforce itself, so the retraction sweep can actually drop it.
|
||||
# `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
|
||||
# ALSO provisional — the head must learn only from title (`wip_title`) + manual
|
||||
# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
|
||||
# `wip_title_soft` (#1474): the soft title tier (sketch/doodle) is LOW-precision, so
|
||||
# it's provisional too — a finished piece titled "sketch" must not train the wip head.
|
||||
_AUTO_SOURCES = (
|
||||
"head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
|
||||
"wip_title_soft",
|
||||
)
|
||||
|
||||
|
||||
def _hygiene_excluded_ids(session: Session) -> set[int]:
|
||||
"""Ids of images carrying ANY system tag (wip / banner / editor
|
||||
screenshot — milestone #128). These images are excluded from OTHER
|
||||
concepts' head training entirely: not positives (a rough wip tagged as a
|
||||
character drags that head toward 'generic sketch') and not sampled or
|
||||
rejection negatives (a wip OF character X is not evidence against X) —
|
||||
simply absent. A system tag's OWN head trains on them unchanged; that is
|
||||
what makes auto-flagging banners/editor screenshots work.
|
||||
|
||||
Item-level by design: a wip-tagged process video contributes (or
|
||||
withholds) ALL its sampled frames, though some may show the finished
|
||||
piece. Operator call 2026-07-03: with enough clean data this washes out —
|
||||
no per-frame handling.
|
||||
"""
|
||||
return set(
|
||||
session.execute(
|
||||
select(image_tag.c.image_record_id)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.is_system.is_(True))
|
||||
).scalars().all()
|
||||
)
|
||||
|
||||
|
||||
def _ids_with_tag(session: Session, tag_id: int) -> list[int]:
|
||||
"""Image ids that count as POSITIVES for this tag's head: human-applied
|
||||
(manual / accepted) tags PLUS any auto-applied tag the operator explicitly
|
||||
confirmed (TagPositiveConfirmation). Unconfirmed auto-applied tags are
|
||||
EXCLUDED — they are provisional and must not train the head that judges
|
||||
them (milestone 139)."""
|
||||
confirmed = (
|
||||
select(TagPositiveConfirmation.image_record_id)
|
||||
.where(TagPositiveConfirmation.tag_id == tag_id)
|
||||
)
|
||||
return [
|
||||
r[0] for r in session.execute(
|
||||
select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tag_id)
|
||||
select(image_tag.c.image_record_id)
|
||||
.where(image_tag.c.tag_id == tag_id)
|
||||
.where(
|
||||
image_tag.c.source.not_in(_AUTO_SOURCES)
|
||||
| image_tag.c.image_record_id.in_(confirmed)
|
||||
)
|
||||
).all()
|
||||
]
|
||||
|
||||
|
||||
@@ -87,9 +87,14 @@ class PatreonIngester(Ingester):
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
request_sleep: float = 0.0,
|
||||
auth_token: str | None = None,
|
||||
client: PatreonClient | None = None,
|
||||
downloader: PatreonDownloader | None = None,
|
||||
):
|
||||
# auth_token: accepted for the uniform native-ingester construction
|
||||
# (download_backends passes it to every adapter); Patreon
|
||||
# authenticates by cookies, so it's unused here.
|
||||
del auth_token
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
# Pacing (plan #703): request_sleep paces the API page fetches,
|
||||
|
||||
@@ -139,6 +139,32 @@ def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
|
||||
return campaign_id
|
||||
|
||||
|
||||
def resolve_display_name(vanity: str, cookies_path: str | None) -> str | None:
|
||||
"""The Patreon campaign's display name for `vanity` via the campaigns API
|
||||
(`fields[campaign]=name`), used to name the Artist at add-time (#130). None
|
||||
on any failure — the caller falls back to the vanity handle. Sync: call from
|
||||
an executor."""
|
||||
jar = _load_cookie_jar(cookies_path)
|
||||
try:
|
||||
resp = requests.get(
|
||||
_CAMPAIGNS_URL,
|
||||
params={"filter[vanity]": vanity, "fields[campaign]": "name"},
|
||||
headers={"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"},
|
||||
cookies=jar,
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json().get("data")
|
||||
except (requests.RequestException, ValueError) as exc:
|
||||
log.warning("Patreon name lookup failed for vanity=%s: %s", vanity, exc)
|
||||
return None
|
||||
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
||||
return None
|
||||
name = (data[0].get("attributes") or {}).get("name")
|
||||
return name.strip() if isinstance(name, str) and name.strip() else None
|
||||
|
||||
|
||||
def _scrape_campaign_id(html: str) -> str | None:
|
||||
"""First campaign id found in creator-page HTML via the known embeddings."""
|
||||
if not isinstance(html, str):
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
"""Native Pixiv client — the Pixiv adapter's read path.
|
||||
|
||||
Pixiv has a real (if unofficial) API: the mobile app API gallery-dl drives
|
||||
(`PixivAppAPI`). Per the downloader ground rule — gallery-dl is the
|
||||
known-working base — this client mirrors gallery-dl 1.32.5's request profile
|
||||
EXACTLY: the same iOS app headers on every request, the same OAuth
|
||||
refresh-token dance against oauth.secure.pixiv.net (X-Client-Time +
|
||||
X-Client-Hash), and the same `/v1/user/illusts` walk paginated by `next_url`.
|
||||
Deviating from that profile is how the SubscribeStar/Patreon spikes broke, so
|
||||
any change here should be diffed against gallery-dl's extractor first.
|
||||
|
||||
Feed shape (characterized from gallery-dl 1.32.5, extractor/pixiv.py):
|
||||
- `GET /v1/user/illusts?user_id=<id>` returns `{"illusts": [work...],
|
||||
"next_url": "https://app-api...?user_id=..&offset=30" | null}`.
|
||||
- Pagination: re-issue the SAME endpoint with `next_url`'s query params. The
|
||||
query string doubles as our resumable page cursor (re-fetching it re-serves
|
||||
the same page — the ingest-core resume contract).
|
||||
- A work carries id/title/type(illust|manga|ugoira)/caption(HTML)/
|
||||
create_date(ISO+09:00)/tags[{name,translated_name}]/user/page_count/
|
||||
x_restrict/series/total_view/total_bookmarks/meta_single_page/meta_pages.
|
||||
- Files: multi-page → meta_pages[].image_urls.original; single page →
|
||||
meta_single_page.original_image_url; ugoira → `/v1/ugoira/metadata` zip
|
||||
(600x600 → 1920x1080 URL swap, gallery-dl's default non-original mode).
|
||||
|
||||
`campaign_id` for Pixiv is the numeric user id (extracted from the source URL
|
||||
by `user_id_from_url` — no network resolver needed).
|
||||
|
||||
Gated works: pixiv serves a `https://s.pximg.net/common/images/limit_*.png`
|
||||
placeholder as the "original" when a work is blocked for this account
|
||||
(sanity-level filter, my-pixiv lock, deleted). gallery-dl's fallback for those
|
||||
is a web-AJAX scrape that needs PHPSESSID browser cookies — FC stores only the
|
||||
OAuth refresh token, so (exactly like our previous gallery-dl configuration,
|
||||
which warned "No PHPSESSID cookie set") those works are skipped, via the
|
||||
post_is_gated seam. Auth failures are loud (rotate the refresh token); a
|
||||
response missing the fields we depend on is DRIFT (update this client).
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
import requests
|
||||
|
||||
from ..utils.paths import safe_ext
|
||||
from .native_ingest_common import (
|
||||
_MAX_429_RETRIES,
|
||||
NativeAuthError,
|
||||
NativeDriftError,
|
||||
NativeIngestError,
|
||||
make_session,
|
||||
retry_after_seconds,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_TIMEOUT_SECONDS = 30.0
|
||||
_API_ROOT = "https://app-api.pixiv.net"
|
||||
_OAUTH_URL = "https://oauth.secure.pixiv.net/auth/token"
|
||||
|
||||
# gallery-dl's public Pixiv-app credentials (PixivAppAPI, also pixivpy's) —
|
||||
# these identify the official iOS app to the API, NOT the operator; the
|
||||
# operator's identity is the OAuth refresh token.
|
||||
_CLIENT_ID = "MOBrBDS8blbauoSck0ZfDbtuzpyT"
|
||||
_CLIENT_SECRET = "lsACyCD94FhDUtGTXi3QzcFE2uU1hqtDaKeqrdwj"
|
||||
_HASH_SECRET = (
|
||||
"28c1fdd170a5204386cb1313c7077b34"
|
||||
"f83e4aaf4aa829ce78c231e05b0bae2c"
|
||||
)
|
||||
|
||||
# The exact header set gallery-dl 1.32.5 installs on its session — the proven
|
||||
# app-API request profile. The Referer also unlocks i.pximg.net media GETs
|
||||
# (403 without it), so the downloader reuses this constant.
|
||||
PIXIV_APP_HEADERS = {
|
||||
"App-OS": "ios",
|
||||
"App-OS-Version": "16.7.2",
|
||||
"App-Version": "7.19.1",
|
||||
"User-Agent": "PixivIOSApp/7.19.1 (iOS 16.7.2; iPhone12,8)",
|
||||
"Referer": "https://app-api.pixiv.net/",
|
||||
}
|
||||
|
||||
# Placeholder image prefix pixiv serves instead of a blocked work's original
|
||||
# (limit_sanity_level / limit_mypixiv / limit_unknown variants).
|
||||
_LIMIT_URL = "https://s.pximg.net/common/images/limit_"
|
||||
|
||||
# The app API reports rate-limiting as an error MESSAGE (often on HTTP 403),
|
||||
# not only as HTTP 429. gallery-dl sleeps 300s in-walk; sleeping that long
|
||||
# inside our time-boxed chunk would eat the whole budget, so we surface it as
|
||||
# a typed 429 and let download_service's cooldown machinery honor the wait.
|
||||
_RATE_LIMIT_RETRY_AFTER = 300.0
|
||||
|
||||
_TITLE_MAX = 50 # gallery-dl pixiv filename template: {title[:50]}
|
||||
|
||||
_RATINGS = {0: "General", 1: "R-18", 2: "R-18G"}
|
||||
|
||||
|
||||
class PixivAPIError(NativeIngestError):
|
||||
"""Base for native Pixiv client failures. status_code / retry_after are
|
||||
inherited from NativeIngestError."""
|
||||
|
||||
|
||||
class PixivAuthError(PixivAPIError, NativeAuthError):
|
||||
"""Auth failure — missing/expired/revoked OAuth refresh token. Fix =
|
||||
rotate the credential (Settings → Credentials → Pixiv), not update the
|
||||
client. Maps to error_type 'auth_error'."""
|
||||
|
||||
|
||||
class PixivDriftError(PixivAPIError, NativeDriftError):
|
||||
"""A response did not match the shape this client depends on (missing
|
||||
`illusts`, un-parseable JSON where JSON was promised). Fail loud so the
|
||||
run flags 'the Pixiv app API changed' instead of silently importing
|
||||
nothing. Maps to API_DRIFT."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One resolved downloadable file belonging to a Pixiv work.
|
||||
|
||||
Fields mirror the other native clients' MediaItem so the downloader and
|
||||
ledger are structurally the same. Pixiv original URLs carry no content
|
||||
hash, so `filehash` is always None and the ledger keys on
|
||||
`<post_id>:<media_id>` where media_id is `p<num>` (page) or `ugoira`
|
||||
(the frame zip) — stable across URL-shape drift.
|
||||
"""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
kind: str
|
||||
filehash: str | None
|
||||
post_id: str
|
||||
media_id: str
|
||||
|
||||
|
||||
def user_id_from_url(url: str) -> str | None:
|
||||
"""The numeric pixiv user id from a source URL, or None.
|
||||
|
||||
Handles the modern forms FC accepts as sources
|
||||
(https://www.pixiv.net/users/<id>, /en/users/<id>) plus the legacy
|
||||
member.php?id=<id>. This IS the campaign id — no network resolver.
|
||||
"""
|
||||
parts = urlsplit(url or "")
|
||||
if "pixiv.net" not in parts.netloc:
|
||||
return None
|
||||
segs = [s for s in parts.path.split("/") if s]
|
||||
if segs and segs[0] == "en":
|
||||
segs = segs[1:]
|
||||
if len(segs) >= 2 and segs[0] == "users" and segs[1].isdigit():
|
||||
return segs[1]
|
||||
if segs and segs[0] == "member.php":
|
||||
qid = dict(parse_qsl(parts.query)).get("id", "")
|
||||
if qid.isdigit():
|
||||
return qid
|
||||
return None
|
||||
|
||||
|
||||
def _work_filename(work: dict, num: int, url: str) -> str:
|
||||
"""gallery-dl layout parity: `{id}_{title[:50]}_{num:>02}.{extension}`
|
||||
(the downloader sanitizes the final segment)."""
|
||||
title = work.get("title")
|
||||
title50 = (title if isinstance(title, str) else "")[:_TITLE_MAX]
|
||||
ext = safe_ext(urlsplit(url).path.rsplit("/", 1)[-1])
|
||||
return f"{work.get('id')}_{title50}_{num:02d}{ext}"
|
||||
|
||||
|
||||
class PixivClient:
|
||||
"""Synchronous Pixiv app-API read client. Construct with the operator's
|
||||
OAuth refresh token (the same token-type Credential the gallery-dl path
|
||||
consumed as `extractor.pixiv.refresh-token`)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
refresh_token: str | None,
|
||||
*,
|
||||
request_sleep: float = 0.0,
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
self.refresh_token = refresh_token
|
||||
self._request_sleep = request_sleep or 0.0
|
||||
self._max_retries = max_retries
|
||||
# No cookies — the app API authenticates via the Bearer token _login
|
||||
# installs. make_session still supplies the retry/UA plumbing; the
|
||||
# extra_headers overwrite its browser UA with the app profile.
|
||||
self._session = (
|
||||
session if session is not None
|
||||
else make_session(None, extra_headers=PIXIV_APP_HEADERS)
|
||||
)
|
||||
self._authed_user: dict = {}
|
||||
# Monotonic deadline after which the access token must be refreshed;
|
||||
# 0 forces a refresh on first use.
|
||||
self._token_deadline = 0.0
|
||||
|
||||
# -- auth ----------------------------------------------------------------
|
||||
|
||||
def _login(self) -> None:
|
||||
"""Exchange the refresh token for a Bearer access token (gallery-dl's
|
||||
`_login_impl`, including the X-Client-Time/X-Client-Hash pair the
|
||||
endpoint validates). No-op while the current token is still fresh."""
|
||||
if time.monotonic() < self._token_deadline:
|
||||
return
|
||||
if not self.refresh_token:
|
||||
raise PixivAuthError(
|
||||
"No Pixiv refresh token configured — add the OAuth refresh "
|
||||
"token as the Pixiv credential (token type)."
|
||||
)
|
||||
# gallery-dl stamps naive-UTC with a literal +00:00 suffix.
|
||||
now = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%S+00:00")
|
||||
headers = {
|
||||
"X-Client-Time": now,
|
||||
"X-Client-Hash": hashlib.md5(
|
||||
(now + _HASH_SECRET).encode()
|
||||
).hexdigest(),
|
||||
}
|
||||
data = {
|
||||
"client_id": _CLIENT_ID,
|
||||
"client_secret": _CLIENT_SECRET,
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": self.refresh_token,
|
||||
"get_secure_url": "1",
|
||||
}
|
||||
try:
|
||||
resp = self._session.post(
|
||||
_OAUTH_URL, data=data, headers=headers,
|
||||
timeout=_TIMEOUT_SECONDS,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise PixivAPIError(f"Pixiv OAuth request failed: {exc}") from exc
|
||||
if resp.status_code >= 400:
|
||||
raise PixivAuthError(
|
||||
"Pixiv rejected the refresh token (HTTP "
|
||||
f"{resp.status_code}) — rotate the Pixiv credential.",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
try:
|
||||
payload = resp.json()["response"]
|
||||
access = payload["access_token"]
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
raise PixivDriftError(
|
||||
f"Pixiv OAuth response shape changed: {exc}"
|
||||
) from exc
|
||||
self._authed_user = payload.get("user") or {}
|
||||
self._session.headers["Authorization"] = f"Bearer {access}"
|
||||
# expires_in is 3600 today; refresh 60s early so a long walk never
|
||||
# rides an expiring token into a spurious 400.
|
||||
expires_in = payload.get("expires_in")
|
||||
lifetime = float(expires_in) if isinstance(expires_in, (int, float)) else 3600.0
|
||||
self._token_deadline = time.monotonic() + max(60.0, lifetime - 60.0)
|
||||
|
||||
# -- request -------------------------------------------------------------
|
||||
|
||||
def _call(self, endpoint: str, params: dict) -> dict:
|
||||
"""Authenticated app-API GET → parsed JSON body, with the shared 429
|
||||
backoff and the loud auth/drift/rate-limit mapping."""
|
||||
self._login()
|
||||
if self._request_sleep > 0:
|
||||
time.sleep(self._request_sleep)
|
||||
url = _API_ROOT + endpoint
|
||||
attempt = 0
|
||||
while True:
|
||||
try:
|
||||
resp = self._session.get(
|
||||
url, params=params, timeout=_TIMEOUT_SECONDS
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
raise PixivAPIError(
|
||||
f"Pixiv request failed ({endpoint}): {exc}"
|
||||
) from exc
|
||||
if resp.status_code == 429 and attempt < self._max_retries:
|
||||
attempt += 1
|
||||
delay = retry_after_seconds(resp, attempt)
|
||||
log.warning(
|
||||
"Pixiv 429 (%s) — backing off %.1fs (retry %d/%d)",
|
||||
endpoint, delay, attempt, self._max_retries,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
break
|
||||
|
||||
try:
|
||||
body = resp.json()
|
||||
except ValueError as exc:
|
||||
raise PixivDriftError(
|
||||
f"Pixiv returned non-JSON for {endpoint} "
|
||||
f"(HTTP {resp.status_code})"
|
||||
) from exc
|
||||
|
||||
error = body.get("error") if isinstance(body, dict) else None
|
||||
message = ""
|
||||
if isinstance(error, dict):
|
||||
message = str(
|
||||
error.get("user_message") or error.get("message") or ""
|
||||
)
|
||||
# Rate limiting first: the app API reports it as an error MESSAGE
|
||||
# (often on HTTP 403), which must not be mistaken for an auth failure.
|
||||
if resp.status_code == 429 or "rate limit" in message.lower():
|
||||
raise PixivAPIError(
|
||||
f"Pixiv rate limit hit ({endpoint}): {message or 'HTTP 429'}",
|
||||
status_code=429,
|
||||
retry_after=_RATE_LIMIT_RETRY_AFTER,
|
||||
)
|
||||
if resp.status_code in (400, 401, 403):
|
||||
# Invalid/expired access token surfaces as 400 invalid_grant-style
|
||||
# errors on the app API; 401/403 are straight auth rejections.
|
||||
raise PixivAuthError(
|
||||
f"Pixiv rejected the request ({endpoint}, HTTP "
|
||||
f"{resp.status_code}): {message or 'auth rejected'} — "
|
||||
"rotate the Pixiv refresh token.",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise PixivAPIError(
|
||||
f"Pixiv API error ({endpoint}, HTTP {resp.status_code}): "
|
||||
f"{message or 'unknown error'}",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
if error:
|
||||
# HTTP 200 carrying an error object — unexpected, but never
|
||||
# silently treat it as data.
|
||||
raise PixivAPIError(
|
||||
f"Pixiv API error ({endpoint}): {message or error}",
|
||||
status_code=resp.status_code,
|
||||
)
|
||||
return body
|
||||
|
||||
# -- normalization -------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _normalize(work: dict) -> dict:
|
||||
"""Wrap an app-API work in the `{"id", "attributes", ...}` post shape
|
||||
the platform-agnostic core and shared helpers read. The raw work rides
|
||||
along under `_work` for extract_media / the post record."""
|
||||
title = work.get("title")
|
||||
caption = work.get("caption")
|
||||
wtype = work.get("type")
|
||||
return {
|
||||
"id": work.get("id"),
|
||||
"attributes": {
|
||||
"title": title if isinstance(title, str) else "",
|
||||
"content": caption if isinstance(caption, str) else "",
|
||||
"published_at": work.get("create_date"),
|
||||
"post_type": wtype if isinstance(wtype, str) else "illust",
|
||||
},
|
||||
"_work": work,
|
||||
}
|
||||
|
||||
# -- post-first seams ----------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def post_record_key(post: dict) -> tuple[str, str] | None:
|
||||
"""`(ledger_key, post_id)` gating post-record capture through the seen
|
||||
ledger (`post:<id>` synthetic key), or None when the work has no id."""
|
||||
pid = post.get("id")
|
||||
pid = str(pid) if pid is not None else ""
|
||||
if not pid:
|
||||
return None
|
||||
return (f"post:{pid}", pid)
|
||||
|
||||
@staticmethod
|
||||
def post_meta(post: dict) -> dict:
|
||||
attrs = post.get("attributes") or {}
|
||||
return {"title": attrs.get("title") or None, "date": attrs.get("published_at")}
|
||||
|
||||
@staticmethod
|
||||
def post_is_gated(post: dict) -> bool:
|
||||
"""True when this account cannot fetch the work's real files: pixiv
|
||||
substitutes a `limit_*` placeholder for the original (sanity-level
|
||||
filter / my-pixiv lock / deleted), or zeroes the author (deleted
|
||||
account). Mirrors #874 semantics: gated content leaves NO trace — a
|
||||
placeholder thumbnail and an empty stub would only pollute the
|
||||
archive. (gallery-dl's PHPSESSID web-scrape fallback for these is out
|
||||
of scope: FC holds no pixiv browser cookies — module docstring.)"""
|
||||
work = post.get("_work") or {}
|
||||
user = work.get("user") or {}
|
||||
if not user.get("id"):
|
||||
return True
|
||||
if work.get("meta_pages"):
|
||||
return False
|
||||
single = work.get("meta_single_page") or {}
|
||||
original = single.get("original_image_url")
|
||||
return isinstance(original, str) and original.startswith(_LIMIT_URL)
|
||||
|
||||
# -- media ---------------------------------------------------------------
|
||||
|
||||
def extract_media(self, post: dict, included_index: dict) -> list[MediaItem]:
|
||||
"""Resolve a work's downloadable files (gallery-dl's `_extract_files`):
|
||||
multi-page originals, the single-page original, or the ugoira frame
|
||||
zip. `included_index` is unused (pixiv works are self-contained)."""
|
||||
work = post.get("_work") or {}
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid or self.post_is_gated(post):
|
||||
return []
|
||||
|
||||
if work.get("type") == "ugoira":
|
||||
return self._ugoira_media(work, pid)
|
||||
|
||||
meta_pages = work.get("meta_pages") or []
|
||||
if meta_pages:
|
||||
items = []
|
||||
for num, page in enumerate(meta_pages):
|
||||
urls = page.get("image_urls") or {}
|
||||
url = urls.get("original")
|
||||
if not isinstance(url, str) or not url:
|
||||
continue
|
||||
items.append(
|
||||
MediaItem(
|
||||
url=url,
|
||||
filename=_work_filename(work, num, url),
|
||||
kind="image",
|
||||
filehash=None,
|
||||
post_id=pid,
|
||||
media_id=f"p{num}",
|
||||
)
|
||||
)
|
||||
return items
|
||||
|
||||
single = work.get("meta_single_page") or {}
|
||||
url = single.get("original_image_url")
|
||||
if not isinstance(url, str) or not url or url.startswith(_LIMIT_URL):
|
||||
return []
|
||||
return [
|
||||
MediaItem(
|
||||
url=url,
|
||||
filename=_work_filename(work, 0, url),
|
||||
kind="image",
|
||||
filehash=None,
|
||||
post_id=pid,
|
||||
media_id="p0",
|
||||
)
|
||||
]
|
||||
|
||||
def _ugoira_meta(self, work: dict, pid: str) -> dict | None:
|
||||
"""Fetch + memoize the ugoira metadata (frames + zip urls) for a work.
|
||||
|
||||
Idempotent and cached on the work dict, so the post record and the
|
||||
media extraction share ONE `/v1/ugoira/metadata` call regardless of
|
||||
which runs first (the core writes the post record BEFORE it extracts
|
||||
media). Returns None — and caches the miss — on a non-auth failure
|
||||
(matching gallery-dl's downgrade); auth failures stay loud."""
|
||||
if "_ugoira_meta" in work:
|
||||
return work["_ugoira_meta"]
|
||||
try:
|
||||
body = self._call("/v1/ugoira/metadata", {"illust_id": pid})
|
||||
meta = body["ugoira_metadata"]
|
||||
except PixivAuthError:
|
||||
raise
|
||||
except (PixivAPIError, KeyError, TypeError) as exc:
|
||||
log.warning("Pixiv ugoira metadata failed for %s: %s", pid, exc)
|
||||
work["_ugoira_meta"] = None
|
||||
return None
|
||||
work["_ugoira_meta"] = meta
|
||||
# Frame delays: a future ugoira→video conversion needs the timings (the
|
||||
# zip alone has none), so the post record captures them.
|
||||
work["_ugoira_frames"] = meta.get("frames") or []
|
||||
return meta
|
||||
|
||||
def fetch_ugoira_frames(self, post: dict) -> None:
|
||||
"""Populate `post['_work']['_ugoira_frames']` for an ugoira post (no-op
|
||||
otherwise). The core writes the post record BEFORE extract_media, so
|
||||
without this the frame timings would never reach the record; this
|
||||
fetches (and memoizes, so extract_media reuses it) the metadata. Injected
|
||||
into the downloader by the ingester, mirroring Patreon's content_fetcher.
|
||||
Auth errors propagate; other failures leave frames unset."""
|
||||
work = post.get("_work") or {}
|
||||
if work.get("type") != "ugoira":
|
||||
return
|
||||
pid = str(post.get("id") or "")
|
||||
if pid:
|
||||
self._ugoira_meta(work, pid)
|
||||
|
||||
def _ugoira_media(self, work: dict, pid: str) -> list[MediaItem]:
|
||||
"""The ugoira frame zip (gallery-dl's default non-original mode):
|
||||
`/v1/ugoira/metadata` → zip_urls.medium with the 600x600→1920x1080
|
||||
swap. A metadata failure downgrades to 'no media' with a warning
|
||||
(matching gallery-dl) instead of failing the walk — except auth
|
||||
failures, which stay loud."""
|
||||
meta = self._ugoira_meta(work, pid)
|
||||
if meta is None:
|
||||
return []
|
||||
try:
|
||||
zip_url = meta["zip_urls"]["medium"]
|
||||
except (KeyError, TypeError) as exc:
|
||||
log.warning("Pixiv ugoira zip url missing for %s: %s", pid, exc)
|
||||
return []
|
||||
url = zip_url.replace("_ugoira600x600", "_ugoira1920x1080", 1)
|
||||
return [
|
||||
MediaItem(
|
||||
url=url,
|
||||
filename=_work_filename(work, 0, url),
|
||||
kind="ugoira",
|
||||
filehash=None,
|
||||
post_id=pid,
|
||||
media_id="ugoira",
|
||||
)
|
||||
]
|
||||
|
||||
# -- iteration -----------------------------------------------------------
|
||||
|
||||
def iter_posts(
|
||||
self, campaign_id: str, cursor: str | None = None
|
||||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||
"""Yield (post, {}, page_cursor) for every work in the user's feed.
|
||||
|
||||
`campaign_id` is the numeric pixiv user id. `cursor` is the query
|
||||
string of the app API's `next_url` (offset pagination); None fetches
|
||||
page 1. The yielded `page_cursor` is the cursor that FETCHED this
|
||||
work's page, so the core checkpoints a value that re-serves the same
|
||||
page on resume (the shared cursor contract)."""
|
||||
if not str(campaign_id or "").isdigit():
|
||||
raise PixivDriftError(
|
||||
f"Pixiv campaign id must be a numeric user id, got "
|
||||
f"{campaign_id!r}"
|
||||
)
|
||||
current = cursor
|
||||
while True:
|
||||
page_cursor = current
|
||||
if current is None:
|
||||
params: dict = {"user_id": campaign_id}
|
||||
else:
|
||||
params = dict(parse_qsl(current))
|
||||
data = self._call("/v1/user/illusts", params)
|
||||
works = data.get("illusts")
|
||||
if not isinstance(works, list):
|
||||
raise PixivDriftError(
|
||||
"Pixiv user-illusts response had no 'illusts' list "
|
||||
f"(keys: {sorted(data)[:8]})"
|
||||
)
|
||||
for work in works:
|
||||
if not isinstance(work, dict):
|
||||
continue
|
||||
yield self._normalize(work), {}, page_cursor
|
||||
next_url = data.get("next_url")
|
||||
if not next_url:
|
||||
return
|
||||
current = str(next_url).rpartition("?")[2]
|
||||
|
||||
# -- user detail ---------------------------------------------------------
|
||||
|
||||
def resolve_display_name(self, user_id: str) -> str | None:
|
||||
"""The pixiv user's display name via `/v1/user/detail` (gallery-dl's
|
||||
user_detail) — used to name the Artist when a source is added by numeric
|
||||
id. None on any failure (the caller falls back to the id)."""
|
||||
try:
|
||||
body = self._call("/v1/user/detail", {"user_id": str(user_id)})
|
||||
except PixivAPIError:
|
||||
return None
|
||||
name = (body.get("user") or {}).get("name") if isinstance(body, dict) else None
|
||||
return name if isinstance(name, str) and name.strip() else None
|
||||
|
||||
# -- verify --------------------------------------------------------------
|
||||
|
||||
def verify_auth(self) -> tuple[bool | None, str]:
|
||||
"""Cheap credential probe: run the OAuth refresh (the thing that fails
|
||||
when the token is bad) without walking any feed."""
|
||||
try:
|
||||
self._token_deadline = 0.0 # force a real refresh
|
||||
self._login()
|
||||
except PixivAuthError as exc:
|
||||
return False, f"Pixiv rejected the credential — {exc}"
|
||||
except PixivAPIError as exc:
|
||||
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||||
account = self._authed_user.get("account") or self._authed_user.get("name")
|
||||
suffix = f" as {account}" if account else ""
|
||||
return True, f"Credentials valid — Pixiv OAuth refresh succeeded{suffix}."
|
||||
|
||||
|
||||
def rating_label(x_restrict) -> str | None:
|
||||
"""Human rating from pixiv's x_restrict (0/1/2) — written into the post
|
||||
record so the archive keeps the R-18 flag without the reader needing to
|
||||
know pixiv's numeric scheme."""
|
||||
if isinstance(x_restrict, bool) or not isinstance(x_restrict, int):
|
||||
return None
|
||||
return _RATINGS.get(x_restrict)
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Native Pixiv media downloader — the Pixiv counterpart to
|
||||
patreon_downloader / subscribestar_downloader.
|
||||
|
||||
Given a normalized Pixiv work and its resolved `MediaItem`s
|
||||
(pixiv_client.extract_media), download the originals to gallery-dl's on-disk
|
||||
layout (so pre-cutover gallery-dl downloads are recognized on disk and not
|
||||
re-fetched), write the post-first sidecars the importer consumes, and report
|
||||
per-media outcomes.
|
||||
|
||||
On-disk layout (matches FC's gallery-dl pixiv config, PLATFORM_DEFAULTS:
|
||||
base-directory `<images_root>/<artist_slug>/pixiv` + `directory:
|
||||
["{category}"]` + filename `{id}_{title[:50]}_{num:>02}.{extension}`):
|
||||
|
||||
<images_root>/<artist_slug>/pixiv/pixiv/<id>_<title50>_<NN>.<ext>
|
||||
|
||||
— note the intentional DOUBLE `pixiv` segment: gallery-dl appended
|
||||
`{category}` under a base-directory that already ended in the platform name,
|
||||
and tier-2 disk-skip parity requires reproducing that exactly. The layout is
|
||||
FLAT (no per-post directory), so the post-first record is `_post_<id>.json`
|
||||
in the same directory (the id suffix prevents the collisions a bare
|
||||
`_post.json` would have here; phase 3 receives explicit post_record_paths, so
|
||||
the name is a convention, not a discovery key).
|
||||
|
||||
Simpler than Patreon (no Mux/yt-dlp video branch) — the one special file is
|
||||
the ugoira frame zip, downloaded as-is; FC's archive-containment import
|
||||
extracts the frames, and the frame DELAYS ride the post record (the zip
|
||||
carries none — a future ugoira→video conversion needs them).
|
||||
|
||||
PURE: no DB; the seen-skip is an injected predicate. FC runs on a plain-HTTP
|
||||
homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
)
|
||||
from .pixiv_client import PIXIV_APP_HEADERS, rating_label
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Control chars (0x00–0x1f + 0x7f DEL) — gallery-dl's default `path-remove`.
|
||||
_GDL_PATH_REMOVE_RE = re.compile(r"[\x00-\x1f\x7f]")
|
||||
|
||||
|
||||
def gdl_clean_filename(name: str) -> str:
|
||||
"""Reproduce gallery-dl's on-disk filename EXACTLY as it wrote it on this
|
||||
Linux host, so the tier-2 disk-skip recognizes pre-cutover files instead of
|
||||
re-downloading them.
|
||||
|
||||
gallery-dl's PathFormat.build_filename is `clean_path(clean_segment(name))`.
|
||||
On Linux (verified against gallery-dl 1.32.5 path.py) the defaults resolve to:
|
||||
- path-restrict "auto" → "/" → clean_segment replaces ONLY "/" → "_"
|
||||
- path-remove "\\x00-\\x1f\\x7f" → clean_path DELETES control chars
|
||||
- path-strip "auto" → "" → NO trailing dot/space stripping
|
||||
Crucially it does NOT touch the Windows-forbidden set (<>:"|?*) — those stay
|
||||
raw in titles on disk. A stricter sanitizer here would rename any such title,
|
||||
miss the on-disk match, and re-pull the whole work. Order mirrors gallery-dl
|
||||
(segment inner, path outer); for these disjoint char sets it's commutative.
|
||||
"""
|
||||
return _GDL_PATH_REMOVE_RE.sub("", name.replace("/", "_"))
|
||||
|
||||
# Enrichment keys copied verbatim from the app-API work dict into the post
|
||||
# record (they're already JSON scalars/objects). Everything lands in
|
||||
# Post.raw_metadata via the importer, so the archive keeps pixiv's stats and
|
||||
# structure without a schema change.
|
||||
_WORK_PASSTHROUGH_KEYS = (
|
||||
"type",
|
||||
"page_count",
|
||||
"width",
|
||||
"height",
|
||||
"total_view",
|
||||
"total_bookmarks",
|
||||
"total_comments",
|
||||
"is_bookmarked",
|
||||
"illust_ai_type",
|
||||
"series",
|
||||
)
|
||||
|
||||
|
||||
class PixivDownloader(BaseNativeDownloader):
|
||||
"""Download resolved Pixiv media to gallery-dl's on-disk layout.
|
||||
Subclasses BaseNativeDownloader for the shared streaming GET
|
||||
(transient-retry + Range-resume) and validation/quarantine. PURE: no DB."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
ugoira_frames_fetcher: Callable[[dict], None] | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
images_root, cookies_path, platform="pixiv",
|
||||
validate=validate, rate_limit=rate_limit, session=session,
|
||||
)
|
||||
# Injected by the ingester (client.fetch_ugoira_frames) so write_post_record
|
||||
# can populate frame timings — which extract_media memoizes, but the core
|
||||
# writes the post record FIRST. Mirrors Patreon's content_fetcher.
|
||||
self._ugoira_frames_fetcher = ugoira_frames_fetcher
|
||||
if session is None:
|
||||
# i.pximg.net 403s any GET without the app Referer; mirror the
|
||||
# client's full app-header profile (gallery-dl serves media off
|
||||
# the same session it drives the API with). An injected session
|
||||
# (tests) owns its own headers.
|
||||
self.session.headers.update(PIXIV_APP_HEADERS)
|
||||
|
||||
# -- public ------------------------------------------------------------
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Download every media item of one work; return per-item outcomes.
|
||||
Mirrors SubscribeStarDownloader.download_post (two-tier skip, mid-post
|
||||
time-box, recapture surfacing)."""
|
||||
flat_dir = self._flat_dir(artist_slug)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for media in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(
|
||||
self._download_one(
|
||||
post, media, flat_dir, artist_slug, is_seen,
|
||||
recapture=recapture,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"Pixiv media failed (work %s, %s): %s",
|
||||
post.get("id"), getattr(media, "media_id", "?"), exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
def _flat_dir(self, artist_slug: str) -> Path:
|
||||
# Double platform segment — gallery-dl layout parity (module docstring).
|
||||
return self.images_root / artist_slug / "pixiv" / "pixiv"
|
||||
|
||||
# -- per-item ----------------------------------------------------------
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
flat_dir: Path,
|
||||
artist_slug: str,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
# The client's filename already carries the {id}_{title50}_{NN} shape
|
||||
# (raw title, gallery-dl-template order); clean it to the byte-exact
|
||||
# name gallery-dl wrote on disk so tier-2 disk-skip matches (else a
|
||||
# re-download of the whole work). See gdl_clean_filename.
|
||||
media_path = flat_dir / gdl_clean_filename(media.filename)
|
||||
|
||||
if media_path.exists(): # tier-2: already on disk
|
||||
return MediaOutcome(
|
||||
media=media, status="skipped_disk", path=media_path, error=None
|
||||
)
|
||||
# recapture: a seen item not on disk is NOT re-downloaded (recovery's job).
|
||||
if seen:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
flat_dir.mkdir(parents=True, exist_ok=True)
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
|
||||
out_path = self._fetch_get(media.url, media_path)
|
||||
reason, quarantine_dest = self._validate_path(out_path, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
return MediaOutcome(
|
||||
media=media, status="quarantined", path=quarantine_dest, error=reason,
|
||||
)
|
||||
self._write_minimal_sidecar(post, out_path, source_url=media.url)
|
||||
return MediaOutcome(media=media, status="downloaded", path=out_path, error=None)
|
||||
|
||||
# -- post record ---------------------------------------------------------
|
||||
|
||||
def write_post_record(self, post: dict, artist_slug: str) -> PostRecordOutcome:
|
||||
"""Write the post-first `_post_<id>.json` — the sole writer of the post
|
||||
body/metadata on the native path. Beyond the standard body fields, the
|
||||
record carries pixiv's own structure (tags + EN translations, rating,
|
||||
series, view/bookmark counts, AI flag, dimensions, author, ugoira frame
|
||||
delays) so the archive keeps what the platform knows about the work."""
|
||||
attrs = post.get("attributes") or {}
|
||||
work = post.get("_work") or {}
|
||||
title = attrs.get("title") if isinstance(attrs.get("title"), str) else None
|
||||
post_type = attrs.get("post_type") if isinstance(attrs.get("post_type"), str) else None
|
||||
pid = str(post.get("id") or "")
|
||||
if not pid:
|
||||
return PostRecordOutcome(
|
||||
path=None, post_type=post_type, title=title, body_chars=0,
|
||||
)
|
||||
|
||||
content = attrs.get("content")
|
||||
content = content if isinstance(content, str) else ""
|
||||
data: dict = {
|
||||
"category": "pixiv",
|
||||
"id": pid,
|
||||
"title": title or "",
|
||||
"content": content,
|
||||
"published_at": attrs.get("published_at"),
|
||||
# The post permalink is synthesized by platforms/pixiv.py
|
||||
# derive_post_url from `id` at parse time — no url key here.
|
||||
"rating": rating_label(work.get("x_restrict")),
|
||||
}
|
||||
for key in _WORK_PASSTHROUGH_KEYS:
|
||||
if key in work:
|
||||
data[key] = work[key]
|
||||
tags = work.get("tags")
|
||||
if isinstance(tags, list):
|
||||
data["tags"] = [
|
||||
{
|
||||
"name": t.get("name"),
|
||||
"translated_name": t.get("translated_name"),
|
||||
}
|
||||
for t in tags
|
||||
if isinstance(t, dict)
|
||||
]
|
||||
user = work.get("user")
|
||||
if isinstance(user, dict):
|
||||
data["user"] = {
|
||||
"id": user.get("id"),
|
||||
"account": user.get("account"),
|
||||
"name": user.get("name"),
|
||||
}
|
||||
# Ugoira frame timings. extract_media memoizes these, but the core writes
|
||||
# the post record BEFORE extracting media, so fetch them here (shared +
|
||||
# idempotent via the client's memoization) so the record actually keeps
|
||||
# them — the zip carries no timings.
|
||||
if (
|
||||
work.get("type") == "ugoira"
|
||||
and not work.get("_ugoira_frames")
|
||||
and self._ugoira_frames_fetcher is not None
|
||||
):
|
||||
self._ugoira_frames_fetcher(post)
|
||||
frames = work.get("_ugoira_frames")
|
||||
if frames:
|
||||
data["ugoira_frames"] = frames
|
||||
|
||||
flat_dir = self._flat_dir(artist_slug)
|
||||
flat_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = flat_dir / f"_post_{pid}.json"
|
||||
path.write_text(json.dumps(data, indent=2, ensure_ascii=False))
|
||||
return PostRecordOutcome(
|
||||
path=path, post_type=post_type, title=title, body_chars=len(content),
|
||||
)
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Native Pixiv ingester — the Pixiv ADAPTER over the platform-agnostic core
|
||||
(`ingest_core.Ingester`).
|
||||
|
||||
Thin counterpart to patreon_ingester / subscribestar_ingester: wires the Pixiv
|
||||
client/downloader/ledger models/constraints/key into the core and supplies the
|
||||
Pixiv failure mapping. The modes (tick / backfill / recovery / recapture), the
|
||||
seen + dead-letter ledgers, cursor checkpointing, and the post-first capture
|
||||
all live in the core. `download_service.download_source` drives
|
||||
`PixivIngester.run` exactly as it drives the other two.
|
||||
|
||||
`campaign_id` is the numeric pixiv user id (download_backends extracts it from
|
||||
the source URL — no network resolver). Auth is the operator's OAuth refresh
|
||||
token (the token-type Credential), passed as `auth_token` — pixiv is the first
|
||||
native platform authenticating by token rather than cookies, so the uniform
|
||||
constructor accepts both and ignores what it doesn't need.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import PixivFailedMedia, PixivSeenMedia
|
||||
from .ingest_core import DEAD_LETTER_THRESHOLD, Ingester
|
||||
from .pixiv_client import MediaItem, PixivAPIError, PixivClient
|
||||
from .pixiv_downloader import PixivDownloader
|
||||
|
||||
__all__ = [
|
||||
"DEAD_LETTER_THRESHOLD",
|
||||
"PixivIngester",
|
||||
"_ledger_key",
|
||||
"verify_pixiv_credential",
|
||||
]
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_LEDGER_KEY_MAX = 128
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
"""Stable per-media identity for the cross-run seen-ledger. Pixiv original
|
||||
URLs carry no content hash, so the key is the page/zip identity scoped to
|
||||
its work: `<illust_id>:p<num>` / `<illust_id>:ugoira`. Bounded to the
|
||||
column width."""
|
||||
if media.filehash:
|
||||
return media.filehash
|
||||
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||
|
||||
|
||||
class PixivIngester(Ingester):
|
||||
"""Walk a pixiv user's works, download unseen originals, return a
|
||||
`DownloadResult`. A thin adapter over `ingest_core.Ingester`; `client` /
|
||||
`downloader` are injectable seams so unit tests run without network."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None,
|
||||
session_factory: Callable[[], object],
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
request_sleep: float = 0.0,
|
||||
auth_token: str | None = None,
|
||||
client: PixivClient | None = None,
|
||||
downloader: PixivDownloader | None = None,
|
||||
):
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
resolved_client = (
|
||||
client
|
||||
if client is not None
|
||||
else PixivClient(auth_token, request_sleep=request_sleep)
|
||||
)
|
||||
resolved_downloader = (
|
||||
downloader
|
||||
if downloader is not None
|
||||
else PixivDownloader(
|
||||
self.images_root, cookies_path, validate=validate, rate_limit=rate_limit,
|
||||
# write_post_record runs before extract_media in the core, so it
|
||||
# fetches ugoira frame timings via the SAME client (shared,
|
||||
# memoized) — else the record's ugoira_frames stays empty.
|
||||
ugoira_frames_fetcher=resolved_client.fetch_ugoira_frames,
|
||||
)
|
||||
)
|
||||
super().__init__(
|
||||
client=resolved_client,
|
||||
downloader=resolved_downloader,
|
||||
session_factory=session_factory,
|
||||
seen_model=PixivSeenMedia,
|
||||
failed_model=PixivFailedMedia,
|
||||
seen_constraint="uq_pixiv_seen_media_source_id",
|
||||
failed_constraint="uq_pixiv_failed_media_source_id",
|
||||
ledger_key=_ledger_key,
|
||||
platform="pixiv",
|
||||
error_base=PixivAPIError,
|
||||
# API_DRIFT message phrasing; the base Ingester._failure_result owns
|
||||
# the auth/drift/HTTP→error_type mapping (shared across platforms).
|
||||
drift_label="Pixiv app API",
|
||||
# Captions are legitimately empty for many pixiv artists, so the
|
||||
# zero-bodies #862 canary would false-positive here; the client's
|
||||
# response-shape checks (missing `illusts` → drift) cover the same
|
||||
# failure class structurally.
|
||||
body_canary=False,
|
||||
)
|
||||
|
||||
|
||||
async def verify_pixiv_credential(
|
||||
auth_token: str | None,
|
||||
) -> tuple[bool | None, str]:
|
||||
"""Native Pixiv credential probe — one OAuth refresh via
|
||||
PixivClient.verify_auth (the exact call that fails when the token is
|
||||
bad; no feed walk). Returns the uniform `(ok, message)` contract so
|
||||
download_backends.verify_source_credential treats it like the others."""
|
||||
client = PixivClient(auth_token)
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(None, client.verify_auth)
|
||||
@@ -1,8 +1,14 @@
|
||||
"""Pixiv — one quirk.
|
||||
|
||||
post_url: gallery-dl's `url` is the image URL on `i.pximg.net`. The
|
||||
post permalink follows /artworks/<id>. external_post_id (= `id`) was
|
||||
already correct, so no override there.
|
||||
post_url: the sidecar's `url` (legacy gallery-dl era) is the image URL
|
||||
on `i.pximg.net`, and the native post record (#129) writes no url key
|
||||
at all — the permalink is synthesized from `id` here either way:
|
||||
/artworks/<id>. external_post_id (= `id`) was already correct, so no
|
||||
override there.
|
||||
|
||||
Downloads run through the native ingester (pixiv_ingester.py), not
|
||||
gallery-dl; this registry entry still owns URL validation, sidecar
|
||||
parsing, and the credential surface (the OAuth refresh token).
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
|
||||
|
||||
@@ -14,9 +14,11 @@
|
||||
`_personalization_id` age-confirmation cookie that the user can't
|
||||
easily refresh — SubscribeStar's frontend JS uses localStorage to
|
||||
suppress the age popup once dismissed. gallery-dl's own login flow
|
||||
sidesteps this by setting `18_plus_agreement_generic=true` on
|
||||
`.subscribestar.adult`; we mirror that for cookies captured via the
|
||||
extension.
|
||||
sidesteps this by setting `18_plus_agreement_generic=true`; we mirror
|
||||
that for cookies captured via the extension — on EVERY SubscribeStar
|
||||
domain, because cookies are domain-scoped and the wall exists on all
|
||||
of them: an .adult-only line never rides along to a .art creator page
|
||||
(Elasid, event #54116 — every poll 302'd to /age_confirmation_warning).
|
||||
"""
|
||||
|
||||
from .base import GD_DEFAULTS, PlatformInfo, str_id_value
|
||||
@@ -29,20 +31,31 @@ def derive_post_url(data: dict) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
# All domains SubscribeStar serves creators from; the age wall gates each.
|
||||
_SS_DOMAINS = (".subscribestar.com", ".subscribestar.adult", ".subscribestar.art")
|
||||
|
||||
|
||||
def augment_cookies(netscape: str) -> str:
|
||||
if "18_plus_agreement_generic" in netscape:
|
||||
return netscape
|
||||
# Far-future expiry — gallery-dl's own login flow sets this with no
|
||||
# explicit expiry; the server only checks presence/value.
|
||||
expiry = 4102444800 # 2100-01-01 UTC
|
||||
line = "\t".join([
|
||||
".subscribestar.adult", "TRUE", "/", "TRUE",
|
||||
str(expiry), "18_plus_agreement_generic", "true",
|
||||
])
|
||||
body = netscape.rstrip("\n")
|
||||
if not body:
|
||||
body = "# Netscape HTTP Cookie File"
|
||||
return body + "\n" + line + "\n"
|
||||
lines = netscape.splitlines()
|
||||
for domain in _SS_DOMAINS:
|
||||
# Per-domain presence check: captured cookies carrying the age
|
||||
# cookie for one domain must not suppress injection on the others.
|
||||
if any(
|
||||
line.startswith(domain + "\t") and "18_plus_agreement_generic" in line
|
||||
for line in lines
|
||||
):
|
||||
continue
|
||||
body += "\n" + "\t".join([
|
||||
domain, "TRUE", "/", "TRUE",
|
||||
str(expiry), "18_plus_agreement_generic", "true",
|
||||
])
|
||||
return body + "\n"
|
||||
|
||||
|
||||
INFO = PlatformInfo(
|
||||
@@ -51,10 +64,11 @@ INFO = PlatformInfo(
|
||||
description="Download posts from SubscribeStar creators",
|
||||
auth_type="cookies",
|
||||
requires_auth=True,
|
||||
url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult)/",
|
||||
url_pattern=r"^https?://(www\.)?subscribestar\.(com|adult|art)/",
|
||||
url_examples=[
|
||||
"https://subscribestar.adult/example_artist",
|
||||
"https://www.subscribestar.com/example_artist",
|
||||
"https://subscribestar.art/example_artist",
|
||||
],
|
||||
default_config={**GD_DEFAULTS, "content_types": ["all"]},
|
||||
derive_post_url=derive_post_url,
|
||||
|
||||
@@ -200,6 +200,8 @@ class PostFeedService:
|
||||
atts_map = await self._attachments_for([post.id])
|
||||
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
||||
item["description_full"] = html_to_plain(post.description)
|
||||
# Full (uncapped) translated description for the detail view (#143).
|
||||
item["description_translated_full"] = post.description_translated
|
||||
# Sanitized HTML body for faithful (semantic) rendering in the post view;
|
||||
# detail-only (the feed list stays lightweight plain text). None when the
|
||||
# post has no body. Inline `<img>` sources are remapped to locally-served
|
||||
@@ -371,6 +373,12 @@ class PostFeedService:
|
||||
description_plain, truncated = None, False
|
||||
else:
|
||||
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
||||
# Translation (#143): the stored translated description is already plain
|
||||
# text; truncate it the same way for the card.
|
||||
desc_trans = post.description_translated
|
||||
desc_trans_short = (
|
||||
truncate_at_word(desc_trans, DESCRIPTION_LIMIT)[0] if desc_trans else None
|
||||
)
|
||||
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
||||
# `source` is null for filesystem-imported posts with no live
|
||||
# subscription (alembic 0030). Frontend renders that as a
|
||||
@@ -384,6 +392,13 @@ class PostFeedService:
|
||||
"downloaded_at": post.downloaded_at.isoformat(),
|
||||
"description_plain": description_plain,
|
||||
"description_truncated": truncated,
|
||||
# Translation-forward fields (#143): shown by default when present;
|
||||
# UI toggles back to the originals above. Source lang labels the toggle.
|
||||
"post_title_translated": post.post_title_translated,
|
||||
"description_translated": desc_trans_short,
|
||||
"translated_source_lang": post.translated_source_lang,
|
||||
# Sticky per-post translation choice (auto/force/original, #155).
|
||||
"translation_override": post.translation_override,
|
||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||
"source": (
|
||||
{"id": source.id, "platform": source.platform}
|
||||
|
||||
@@ -29,6 +29,13 @@ def _post_dict(p: Post) -> dict:
|
||||
"date": p.post_date.isoformat() if p.post_date else None,
|
||||
"description_html": sanitize_post_html(p.description),
|
||||
"attachment_count": p.attachment_count,
|
||||
# Translation (#143): the English title/description shown by default when
|
||||
# a translation exists; the UI toggles to the original. Source lang labels
|
||||
# the original.
|
||||
"title_translated": p.post_title_translated,
|
||||
"description_translated": p.description_translated,
|
||||
"translated_source_lang": p.translated_source_lang,
|
||||
"translation_override": p.translation_override,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -4,11 +4,18 @@ is_subscription auto-flip on first add / last delete.
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import Artist, ImportSettings, Source
|
||||
from ..models import (
|
||||
Artist,
|
||||
ImageProvenance,
|
||||
ImageRecord,
|
||||
ImportSettings,
|
||||
Post,
|
||||
Source,
|
||||
)
|
||||
from .platforms import known_platform_keys
|
||||
from .scheduler_service import compute_next_check_at
|
||||
|
||||
@@ -298,6 +305,15 @@ class SourceService:
|
||||
|
||||
for key, value in fields.items():
|
||||
setattr(source, key, value)
|
||||
# Disabling a source clears its failure state (operator: disable the subs
|
||||
# you're not paying for without them lingering as "failing"). Re-enabling
|
||||
# then starts clean; the next real run re-derives health. Only on the
|
||||
# explicit disable — an unrelated edit to an already-disabled source
|
||||
# leaves its (already-clear) state alone.
|
||||
if fields.get("enabled") is False:
|
||||
source.last_error = None
|
||||
source.error_type = None
|
||||
source.consecutive_failures = 0
|
||||
try:
|
||||
await self.session.flush()
|
||||
except IntegrityError as exc:
|
||||
@@ -411,6 +427,70 @@ class SourceService:
|
||||
await self.session.commit()
|
||||
return await self._row_to_record(source)
|
||||
|
||||
async def reassign(self, source_id: int, target_artist_id: int) -> SourceRecord:
|
||||
"""Move a Source — and the content it brought in — to another artist
|
||||
(#130). The slug/storage path is IMMUTABLE, so NO files move: only the
|
||||
artist attribution changes (reads use ImageRecord.path). Re-points the
|
||||
source's Posts and the ImageRecords it contributed (those still
|
||||
attributed to the old artist — images shared with another artist are
|
||||
left alone). If the old artist is left fully empty (no sources, images,
|
||||
or posts) it's deleted (ArtistVisit cascades); if it just lost its last
|
||||
source, its is_subscription flag clears. No-op when already on target."""
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
raise LookupError(f"source id={source_id} not found")
|
||||
target = await self._artist_or_raise(target_artist_id)
|
||||
old_artist_id = source.artist_id
|
||||
if old_artist_id == target_artist_id:
|
||||
return await self._row_to_record(source)
|
||||
|
||||
source.artist_id = target_artist_id
|
||||
target.is_subscription = True
|
||||
# Re-attribute this source's posts (Post.artist_id is denormalized).
|
||||
await self.session.execute(
|
||||
update(Post).where(Post.source_id == source_id)
|
||||
.values(artist_id=target_artist_id)
|
||||
)
|
||||
# Re-attribute the images this source contributed that are still on the
|
||||
# OLD artist. Scoping to artist_id == old avoids stealing an image that
|
||||
# a different artist's source also contributed.
|
||||
contributed = select(ImageProvenance.image_record_id).where(
|
||||
ImageProvenance.source_id == source_id
|
||||
)
|
||||
await self.session.execute(
|
||||
update(ImageRecord)
|
||||
.where(
|
||||
ImageRecord.artist_id == old_artist_id,
|
||||
ImageRecord.id.in_(contributed),
|
||||
)
|
||||
.values(artist_id=target_artist_id)
|
||||
)
|
||||
await self.session.flush()
|
||||
|
||||
old = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == old_artist_id)
|
||||
)).scalar_one_or_none()
|
||||
if old is not None:
|
||||
n_src = (await self.session.execute(
|
||||
select(func.count(Source.id)).where(Source.artist_id == old_artist_id)
|
||||
)).scalar_one()
|
||||
n_img = (await self.session.execute(
|
||||
select(func.count(ImageRecord.id)).where(
|
||||
ImageRecord.artist_id == old_artist_id
|
||||
)
|
||||
)).scalar_one()
|
||||
n_post = (await self.session.execute(
|
||||
select(func.count(Post.id)).where(Post.artist_id == old_artist_id)
|
||||
)).scalar_one()
|
||||
if n_src == 0 and n_img == 0 and n_post == 0:
|
||||
await self.session.delete(old) # ArtistVisit cascades
|
||||
elif n_src == 0:
|
||||
old.is_subscription = False
|
||||
await self.session.commit()
|
||||
return await self._row_to_record(source)
|
||||
|
||||
async def delete(self, source_id: int) -> None:
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
|
||||
@@ -226,13 +226,35 @@ def _attachment_item(
|
||||
)
|
||||
|
||||
|
||||
def _normalize_ss_host(netloc: str) -> str:
|
||||
"""Rewrite the `subscribestar.art` host to `subscribestar.adult`.
|
||||
|
||||
The age wall on the `.art` domain does not clear with the
|
||||
`18_plus_agreement_generic` cookie (unlike `.com`/`.adult`): a `.art`
|
||||
creator page keeps 302'ing to `/age_confirmation_warning` even with the
|
||||
cookie set (Elasid, event #54116). The same creator is reachable on
|
||||
`.adult`, where the cookie works — so `.art` behaves as an alias that
|
||||
doesn't honor the age gate. Normalize it to `.adult` at request time (the
|
||||
stored Source.url is left untouched). `.com`/`.adult` pass through.
|
||||
"""
|
||||
host = netloc.lower()
|
||||
if host == "subscribestar.art" or host.endswith(".subscribestar.art"):
|
||||
rewritten = netloc[: -len("art")] + "adult"
|
||||
log.info(
|
||||
"SubscribeStar: rewrote age-gated .art host %r → %r", netloc, rewritten
|
||||
)
|
||||
return rewritten
|
||||
return netloc
|
||||
|
||||
|
||||
def _split_creator_url(campaign_id: str) -> tuple[str, str]:
|
||||
"""`campaign_id` is the creator URL → (base, slug).
|
||||
|
||||
base = scheme://host (preserving .com vs .adult); slug = first path segment.
|
||||
base = scheme://host (preserving .com vs .adult; .art → .adult, see
|
||||
_normalize_ss_host); slug = first path segment.
|
||||
"""
|
||||
parts = urlsplit(campaign_id)
|
||||
base = f"{parts.scheme or 'https'}://{parts.netloc}"
|
||||
base = f"{parts.scheme or 'https'}://{_normalize_ss_host(parts.netloc)}"
|
||||
slug = parts.path.strip("/").split("/")[0] if parts.path else ""
|
||||
return base, slug
|
||||
|
||||
@@ -251,6 +273,30 @@ def _parse_ss_datetime(text: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
_OG_TITLE_RE = re.compile(
|
||||
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_TITLE_RE = re.compile(r"<title[^>]*>(.*?)</title>", re.IGNORECASE | re.DOTALL)
|
||||
# Trailing " | SubscribeStar" / " on SubscribeStar" the profile <title> carries.
|
||||
_SS_TITLE_SUFFIX_RE = re.compile(
|
||||
r"\s*[|·]\s*SubscribeStar.*$|\s+on\s+SubscribeStar.*$", re.IGNORECASE
|
||||
)
|
||||
|
||||
|
||||
def _extract_creator_name(html: str) -> str | None:
|
||||
"""The creator's display name from a SubscribeStar profile page: prefer the
|
||||
og:title meta (it's the bare creator name), else the <title> with the
|
||||
SubscribeStar suffix stripped. None when neither yields anything (#130)."""
|
||||
m = _OG_TITLE_RE.search(html)
|
||||
name = unescape(m.group(1)).strip() if m else ""
|
||||
if not name:
|
||||
t = _TITLE_RE.search(html)
|
||||
raw = unescape(t.group(1)).strip() if t else ""
|
||||
name = _SS_TITLE_SUFFIX_RE.sub("", raw).strip()
|
||||
return name or None
|
||||
|
||||
|
||||
class SubscribeStarClient:
|
||||
"""Synchronous SubscribeStar HTML-scrape read client. Construct with a path
|
||||
to a Netscape cookies.txt (the same file CredentialService.get_cookies_path
|
||||
@@ -582,6 +628,23 @@ class SubscribeStarClient:
|
||||
current = next_href
|
||||
first_page = False
|
||||
|
||||
# -- display name -------------------------------------------------------
|
||||
|
||||
def resolve_display_name(self, campaign_id: str) -> str | None:
|
||||
"""The creator's display name from their profile page, used to name the
|
||||
Artist at add-time (#130). `campaign_id` is the creator URL. None on any
|
||||
failure — the caller falls back to the URL handle. Sync: run in an
|
||||
executor."""
|
||||
base, slug = _split_creator_url(campaign_id)
|
||||
if not slug:
|
||||
return None
|
||||
self._session.headers["Referer"] = f"{base}/"
|
||||
try:
|
||||
html = self._feed_html(f"{base}/{slug}")
|
||||
except SubscribeStarAPIError:
|
||||
return None
|
||||
return _extract_creator_name(html)
|
||||
|
||||
# -- verify ------------------------------------------------------------
|
||||
|
||||
def verify_auth(self, campaign_id: str) -> tuple[bool | None, str]:
|
||||
|
||||
@@ -61,9 +61,14 @@ class SubscribeStarIngester(Ingester):
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
request_sleep: float = 0.0,
|
||||
auth_token: str | None = None,
|
||||
client: SubscribeStarClient | None = None,
|
||||
downloader: SubscribeStarDownloader | None = None,
|
||||
):
|
||||
# auth_token: accepted for the uniform native-ingester construction
|
||||
# (download_backends passes it to every adapter); SubscribeStar
|
||||
# authenticates by cookies, so it's unused here.
|
||||
del auth_token
|
||||
self.images_root = Path(images_root)
|
||||
self.cookies_path = str(cookies_path) if cookies_path else None
|
||||
resolved_client = (
|
||||
|
||||
@@ -107,6 +107,7 @@ class TagDirectoryService:
|
||||
"kind": tag.kind.value if hasattr(tag.kind, "value") else tag.kind,
|
||||
"fandom_id": tag.fandom_id,
|
||||
"fandom_name": fandom_name,
|
||||
"is_system": tag.is_system,
|
||||
"image_count": int(image_count),
|
||||
"preview_thumbnails": previews.get(tag.id, []),
|
||||
}
|
||||
|
||||
@@ -67,24 +67,35 @@ def fandom_join_alias():
|
||||
|
||||
|
||||
def tag_columns(fandom_alias):
|
||||
"""The canonical (id, name, kind, fandom_id, fandom_name) column set for a
|
||||
tag select that outerjoins `fandom_alias` (from `fandom_join_alias()`)."""
|
||||
"""The canonical (id, name, kind, fandom_id, fandom_name, is_system)
|
||||
column set for a tag select that outerjoins `fandom_alias` (from
|
||||
`fandom_join_alias()`)."""
|
||||
return [
|
||||
Tag.id,
|
||||
Tag.name,
|
||||
Tag.kind,
|
||||
Tag.fandom_id,
|
||||
fandom_alias.c.name.label("fandom_name"),
|
||||
Tag.is_system,
|
||||
]
|
||||
|
||||
|
||||
def serialize_tag(row) -> dict:
|
||||
"""Serialize a row carrying .id/.name/.kind/.fandom_id/.fandom_name to the
|
||||
canonical tag dict. `kind` may be a TagKind enum or a plain string."""
|
||||
canonical tag dict. `kind` may be a TagKind enum or a plain string.
|
||||
`is_system` defaults False for callers whose selects predate the column."""
|
||||
return {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"kind": row.kind.value if hasattr(row.kind, "value") else row.kind,
|
||||
"fandom_id": row.fandom_id,
|
||||
"fandom_name": row.fandom_name,
|
||||
"is_system": bool(getattr(row, "is_system", False)),
|
||||
# Applied-tag context: only the image-scoped selects (list_for_image /
|
||||
# get_image_with_tags) provide these; autocomplete / directory don't →
|
||||
# default. `source` drives the auto-applied badge; `confirmed` = the
|
||||
# operator affirmed the tag (a training positive, shielded from the
|
||||
# retraction sweep — milestone 139).
|
||||
"source": getattr(row, "source", None),
|
||||
"confirmed": bool(getattr(row, "confirmed", False)),
|
||||
}
|
||||
|
||||
@@ -9,7 +9,14 @@ from sqlalchemy import and_, case, exists, func, select, text, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from ..models import HeadMetric, Tag, TagHead, TagKind, image_tag
|
||||
from ..models import (
|
||||
HeadMetric,
|
||||
Tag,
|
||||
TagHead,
|
||||
TagKind,
|
||||
TagPositiveConfirmation,
|
||||
image_tag,
|
||||
)
|
||||
from .db_helpers import get_or_create
|
||||
from .tag_query import fandom_join_alias, tag_columns
|
||||
|
||||
@@ -288,8 +295,14 @@ class TagService:
|
||||
character chip with its fandom without an N+1 (mirrors the
|
||||
autocomplete/directory resolution)."""
|
||||
fandom_alias = fandom_join_alias()
|
||||
# source drives the auto-applied badge; confirmed = operator affirmed the
|
||||
# tag (positive + retraction-shielded, milestone 139).
|
||||
confirmed = exists().where(
|
||||
TagPositiveConfirmation.image_record_id == image_id,
|
||||
TagPositiveConfirmation.tag_id == Tag.id,
|
||||
).label("confirmed")
|
||||
stmt = (
|
||||
select(*tag_columns(fandom_alias))
|
||||
select(*tag_columns(fandom_alias), image_tag.c.source, confirmed)
|
||||
.select_from(
|
||||
Tag.__table__
|
||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||
@@ -330,6 +343,12 @@ class TagService:
|
||||
tag = await self.session.get(Tag, tag_id)
|
||||
if tag is None:
|
||||
raise TagValidationError(f"Tag {tag_id} not found")
|
||||
# The training-hygiene machinery keys on system tags by row; a rename
|
||||
# would silently disarm it (milestone #128).
|
||||
if tag.is_system:
|
||||
raise TagValidationError(
|
||||
f"'{tag.name}' is a system tag and cannot be renamed"
|
||||
)
|
||||
|
||||
# Case-insensitive clash (#701) — renaming onto a differently-cased tag
|
||||
# is still a merge, not a silent fork.
|
||||
@@ -528,6 +547,14 @@ class TagService:
|
||||
source = await self.session.get(Tag, source_id)
|
||||
if source is None:
|
||||
raise TagValidationError(f"Tag {source_id} not found")
|
||||
# Merge deletes the source row — the only tag-delete path — and the
|
||||
# training-hygiene machinery keys on system rows (milestone #128).
|
||||
# Merging INTO a system tag stays allowed (adopting an operator's old
|
||||
# 'wip sketch' tag into the system 'wip' is the intended move).
|
||||
if source.is_system:
|
||||
raise TagValidationError(
|
||||
f"'{source.name}' is a system tag and cannot be merged away"
|
||||
)
|
||||
target = await self.session.get(Tag, target_id)
|
||||
if target is None:
|
||||
raise TagValidationError(f"Tag {target_id} not found")
|
||||
@@ -792,6 +819,10 @@ async def normalize_existing_tags(
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Tag.id, Tag.name, Tag.kind, Tag.fandom_id)
|
||||
# System tags (#128) are exempt: their names are the keys the
|
||||
# hygiene machinery matches on, and canonicalization would recase
|
||||
# 'wip' → 'Wip' (this path bypasses TagService.rename's guard).
|
||||
.where(Tag.is_system.is_(False))
|
||||
)
|
||||
).all()
|
||||
groups = _group_existing_tags(rows)
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Title-based WIP auto-tagging (task #1458).
|
||||
|
||||
Deterministic heuristic: when a post's TITLE explicitly declares work-in-progress
|
||||
(the artist's own "WIP" / "work in progress" label), the ``wip`` system tag is
|
||||
applied to that post's images — a cheap, high-precision complement to the
|
||||
image-based ML ``wip`` head. WIP images are excluded from the Explore/gallery
|
||||
browse (see gallery_service ``excluded_system_tags``), so honouring the artist's
|
||||
own label keeps unfinished pieces out of the main browse right at import.
|
||||
|
||||
Precision over recall — a false WIP tag HIDES a finished post — so matching is
|
||||
token-anchored: ``swipe`` / ``wiped`` / ``wiping`` never trip it (a letter on the
|
||||
boundary blocks the match).
|
||||
|
||||
Sync-only: both consumers (the importer and the backfill Celery task) run on a
|
||||
sync Session. Application is idempotent-additive (ON CONFLICT DO NOTHING) and
|
||||
stamps a distinct ``image_tag.source`` so a later pass can tell where a wip tag
|
||||
came from — the "manual" / "head_auto" / "ccip_auto" / "ml_accepted" provenance
|
||||
family gains one member.
|
||||
"""
|
||||
import re
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models.tag import WIP_SYSTEM_TAG, Tag, image_tag
|
||||
|
||||
# image_tag.source stamped on title-heuristic WIP tags — distinct from the other
|
||||
# apply sources so provenance stays legible and a future undo can target only these.
|
||||
# HARD tier ("WIP"/"work in progress") is high-precision → trains the wip head.
|
||||
WIP_TITLE_SOURCE = "wip_title"
|
||||
# SOFT tier (sketch/doodle/scribble, #1474) is LOWER-precision — a finished "sketch"
|
||||
# is often not WIP. This source is PROVISIONAL (in training_data._AUTO_SOURCES) so it
|
||||
# NEVER trains the wip head; a soft-tagged image that also looks like real content is
|
||||
# surfaced by the ring-loud audit for review.
|
||||
WIP_TITLE_SOFT_SOURCE = "wip_title_soft"
|
||||
|
||||
# A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
|
||||
# (space/underscore/hyphen separated). The letter-boundary lookarounds are what
|
||||
# make this precision-first: `s|wip|e`, `|wip|ed`, `|wip|ing` all have a letter
|
||||
# abutting the token, so they're rejected. A trailing digit is allowed so
|
||||
# "WIP2" (= WIP part 2) still matches.
|
||||
_WIP_RE = re.compile(
|
||||
r"(?<![A-Za-z])(?:w\.?i\.?p\.?|work[\s_-]+in[\s_-]+progress)(?![A-Za-z])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Soft tier: sketch / doodle / scribble (+ plurals), letter-boundary anchored so
|
||||
# "sketchbook" / "kadoodle" don't trip it. Deliberately conservative — recall is
|
||||
# secondary because the soft source doesn't train the head and the ring-loud audit
|
||||
# catches false positives.
|
||||
_SOFT_WIP_RE = re.compile(
|
||||
r"(?<![A-Za-z])(?:sketch|sketches|doodle|doodles|scribble|scribbles)(?![A-Za-z])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Coarse SQL prefilters for the backfill sweep — narrow the post scan to rows that
|
||||
# COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
|
||||
# Each MUST stay a SUPERSET of its regex or the sweep would silently miss posts.
|
||||
WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
|
||||
SOFT_WIP_TITLE_SQL_PREFILTER = ("%sketch%", "%doodle%", "%scribble%")
|
||||
|
||||
# Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
|
||||
# ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
|
||||
_INSERT_CHUNK = 5000
|
||||
|
||||
|
||||
def matches_wip_title(title: str | None) -> bool:
|
||||
"""True when a post title explicitly marks it work-in-progress (HARD tier)."""
|
||||
if not title:
|
||||
return False
|
||||
return _WIP_RE.search(title) is not None
|
||||
|
||||
|
||||
def matches_soft_wip_title(title: str | None) -> bool:
|
||||
"""True when a title carries a SOFT WIP cue (sketch/doodle/scribble, #1474)."""
|
||||
if not title:
|
||||
return False
|
||||
return _SOFT_WIP_RE.search(title) is not None
|
||||
|
||||
|
||||
def resolve_wip_tag_id(session: Session) -> int | None:
|
||||
"""The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
|
||||
return session.execute(
|
||||
select(Tag.id).where(Tag.name == WIP_SYSTEM_TAG, Tag.is_system.is_(True))
|
||||
).scalar_one_or_none()
|
||||
|
||||
|
||||
def apply_wip_image_tags(
|
||||
session: Session, image_ids, tag_id: int, *, source: str = WIP_TITLE_SOURCE
|
||||
) -> int:
|
||||
"""Attach ``tag_id`` (stamped with ``source``) to each image id, idempotently —
|
||||
never disturbs an existing tag or its source. Returns the number of image_tag
|
||||
rows newly inserted. Does NOT commit.
|
||||
|
||||
The insert count is computed from a pre-SELECT of already-tagged ids rather
|
||||
than the statement's ``rowcount``: psycopg reports -1 for a multi-row
|
||||
ON CONFLICT DO NOTHING insert (it runs via an executemany path), so rowcount
|
||||
is unusable here. The SELECT is accurate within this single transaction (no
|
||||
concurrent writer touches these (image, wip) rows); ON CONFLICT DO NOTHING
|
||||
stays as a race-safety belt so a rare concurrent insert can't error."""
|
||||
ids = list({int(i) for i in image_ids})
|
||||
if not ids:
|
||||
return 0
|
||||
inserted = 0
|
||||
for start in range(0, len(ids), _INSERT_CHUNK):
|
||||
chunk = ids[start:start + _INSERT_CHUNK]
|
||||
already = set(session.execute(
|
||||
select(image_tag.c.image_record_id)
|
||||
.where(image_tag.c.tag_id == tag_id)
|
||||
.where(image_tag.c.image_record_id.in_(chunk))
|
||||
).scalars())
|
||||
to_insert = [iid for iid in chunk if iid not in already]
|
||||
if not to_insert:
|
||||
continue
|
||||
session.execute(
|
||||
pg_insert(image_tag)
|
||||
.values([
|
||||
{"image_record_id": iid, "tag_id": tag_id, "source": source}
|
||||
for iid in to_insert
|
||||
])
|
||||
.on_conflict_do_nothing(index_elements=["image_record_id", "tag_id"])
|
||||
)
|
||||
inserted += len(to_insert)
|
||||
return inserted
|
||||
@@ -76,9 +76,19 @@ DOWNLOAD_STALL_THRESHOLD_MINUTES = 30
|
||||
OLD_TASK_DAYS = 7
|
||||
PHASH_PAGE = 500
|
||||
VERIFY_PAGE = 200
|
||||
# Title-based WIP backfill (task #1458): posts scanned per keyset page.
|
||||
WIP_BACKFILL_PAGE = 500
|
||||
FFPROBE_TIMEOUT_SECONDS = 10
|
||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||
# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
|
||||
# then os.replace() into place (importer / external_fetch / native_ingest_common /
|
||||
# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
|
||||
# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
|
||||
# keeps it from deleting an in-flight download's staging file mid-write.
|
||||
IMAGES_ROOT = Path("/images")
|
||||
TEMP_STAGING_SUFFIXES = (".part", ".partial")
|
||||
ORPHAN_TEMP_MIN_AGE_HOURS = 6
|
||||
|
||||
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
||||
# > the entity's longest legitimate runtime (its task's time_limit + a
|
||||
@@ -339,6 +349,34 @@ def cleanup_old_tasks() -> int:
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
|
||||
def cleanup_orphaned_temp_files() -> int:
|
||||
"""Delete orphaned .part/.partial staging files under the images root, left by
|
||||
a download/import killed mid-write. Only removes files older than
|
||||
ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
|
||||
pulled out from under it. Returns the count removed."""
|
||||
if not IMAGES_ROOT.is_dir():
|
||||
return 0
|
||||
cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
|
||||
removed = 0
|
||||
for path in IMAGES_ROOT.rglob("*"):
|
||||
if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
|
||||
continue
|
||||
try:
|
||||
if path.stat().st_mtime >= cutoff:
|
||||
continue # too fresh — may be an active download
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError as exc:
|
||||
log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
|
||||
if removed:
|
||||
log.info(
|
||||
"cleanup_orphaned_temp_files: removed %d orphaned staging file(s)",
|
||||
removed,
|
||||
)
|
||||
return removed
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||
def recover_stalled_task_runs() -> int:
|
||||
"""Flip task_run rows stuck in 'running' past their queue-specific
|
||||
@@ -1009,6 +1047,96 @@ def cleanup_old_download_events() -> int:
|
||||
return result.rowcount or 0
|
||||
|
||||
|
||||
def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
||||
"""One keyset-paginated pass over posts whose title matches a WIP tier, applying
|
||||
`tag_id` (stamped `source`) to their images. Shared by the hard + soft tiers
|
||||
(#1458 / #1474). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
||||
`matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
|
||||
count newly applied."""
|
||||
from ..models import Post
|
||||
from ..models.image_provenance import ImageProvenance
|
||||
from ..services.wip_title import apply_wip_image_tags
|
||||
|
||||
applied = 0
|
||||
last_id = 0
|
||||
while True:
|
||||
rows = session.execute(
|
||||
select(Post.id, Post.post_title)
|
||||
.where(Post.id > last_id)
|
||||
.where(Post.post_title.is_not(None))
|
||||
.where(or_(*[Post.post_title.ilike(p) for p in prefilter]))
|
||||
.order_by(Post.id.asc())
|
||||
.limit(WIP_BACKFILL_PAGE)
|
||||
).all()
|
||||
if not rows:
|
||||
break
|
||||
last_id = rows[-1][0]
|
||||
match_ids = [pid for pid, title in rows if matcher(title)]
|
||||
if match_ids:
|
||||
image_ids = session.execute(
|
||||
select(ImageProvenance.image_record_id)
|
||||
.where(ImageProvenance.post_id.in_(match_ids))
|
||||
).scalars().all()
|
||||
applied += apply_wip_image_tags(session, image_ids, tag_id, source=source)
|
||||
session.commit()
|
||||
return applied
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.maintenance.backfill_wip_title_tags",
|
||||
# Coarse-prefiltered scan over posts; the candidate set is small on a typical
|
||||
# library, but bound it like the other full-library sweeps.
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def backfill_wip_title_tags() -> int:
|
||||
"""Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
|
||||
images — the operator-triggered back-catalogue catch-up (task #1458 hard tier +
|
||||
#1474 soft tier). New imports are tagged live by the importer; this covers the
|
||||
existing library.
|
||||
|
||||
HARD tier ("WIP"/"work in progress") always runs (the operator triggered the
|
||||
scan); the SOFT tier (sketch/doodle, provisional source) runs only when
|
||||
wip_soft_title_tagging_enabled, AFTER hard so a title matching both keeps the
|
||||
trained hard tag (ON CONFLICT DO NOTHING). Keyset-paginated, restart-safe.
|
||||
|
||||
Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
|
||||
posts and silently undo a manual WIP removal, so it stays an explicit operator
|
||||
action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
|
||||
"""
|
||||
from ..models import ImportSettings
|
||||
from ..services.wip_title import (
|
||||
SOFT_WIP_TITLE_SQL_PREFILTER,
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
WIP_TITLE_SOURCE,
|
||||
WIP_TITLE_SQL_PREFILTER,
|
||||
matches_soft_wip_title,
|
||||
matches_wip_title,
|
||||
resolve_wip_tag_id,
|
||||
)
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
tag_id = resolve_wip_tag_id(session)
|
||||
if tag_id is None:
|
||||
log.warning(
|
||||
"backfill_wip_title_tags: no `wip` system tag present; nothing to do"
|
||||
)
|
||||
return 0
|
||||
settings = ImportSettings.load_sync(session)
|
||||
applied = _backfill_wip_tier(
|
||||
session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
|
||||
WIP_TITLE_SOURCE,
|
||||
)
|
||||
if settings.wip_soft_title_tagging_enabled:
|
||||
applied += _backfill_wip_tier(
|
||||
session, tag_id, SOFT_WIP_TITLE_SQL_PREFILTER, matches_soft_wip_title,
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
)
|
||||
if applied:
|
||||
log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
|
||||
return applied
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.vacuum_analyze")
|
||||
def vacuum_analyze() -> dict:
|
||||
"""Periodic VACUUM (ANALYZE) over the high-churn tables (VACUUM_TABLES) to
|
||||
|
||||
+130
-1
@@ -328,6 +328,11 @@ def train_heads(self, run_id: int) -> str:
|
||||
run.status = "ready"
|
||||
run.finished_at = datetime.now(UTC)
|
||||
session.commit()
|
||||
# A retrain folds the day's accepts/rejects into the heads; refresh the
|
||||
# CCIP character prototypes on the SAME trigger (#1317) so the Retrain
|
||||
# button + nightly run keep character matching current too. Cheap no-op
|
||||
# when references didn't change.
|
||||
refresh_character_prototypes.delay()
|
||||
return "ready"
|
||||
|
||||
|
||||
@@ -351,7 +356,10 @@ def scheduled_train_heads() -> str:
|
||||
if running is not None:
|
||||
return "already running"
|
||||
run = HeadTrainingRun(
|
||||
params={"source": "scheduled"}, status="running",
|
||||
# Nightly = FULL reconcile (refit every head) so sampled-negative +
|
||||
# hygiene drift is folded in; the manual Retrain button stays
|
||||
# incremental (#1317 p2).
|
||||
params={"source": "scheduled", "full": True}, status="running",
|
||||
last_progress_at=datetime.now(UTC),
|
||||
)
|
||||
session.add(run)
|
||||
@@ -361,6 +369,31 @@ def scheduled_train_heads() -> str:
|
||||
return "dispatched"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.refresh_character_prototypes",
|
||||
# Global-gate no-op when idle; a rebuild touches only changed characters. The
|
||||
# initial (cold) build loads the whole reference set once, in the BACKGROUND
|
||||
# — never on a /suggestions request.
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def refresh_character_prototypes(full: bool = False) -> str:
|
||||
"""Incrementally refresh the CCIP character-prototype store (#1317, m138):
|
||||
cheap global-gate skip when nothing changed, else rebuild only the characters
|
||||
whose references moved. Triggered by a ~15-min beat, after every head retrain
|
||||
(train_heads), and nightly with full=True. Returns a short status."""
|
||||
from ..services.ml.character_prototypes import (
|
||||
refresh_character_prototypes as _refresh,
|
||||
)
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
result = _refresh(session, full=full)
|
||||
return (
|
||||
"skipped" if result["skipped"]
|
||||
else f"rebuilt={result['rebuilt']} removed={result['removed']}"
|
||||
)
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.apply_head_tags",
|
||||
bind=True,
|
||||
@@ -559,3 +592,99 @@ def scheduled_ccip_auto_apply() -> str:
|
||||
applied += 1
|
||||
session.commit()
|
||||
return f"applied={applied}"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_presentation_auto_apply",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def scheduled_presentation_auto_apply() -> str:
|
||||
"""Auto-hide presentation chrome (banner) on a daily passive sweep (#141).
|
||||
No-op unless presentation_auto_apply_enabled. Idempotent — already-tagged images
|
||||
are skipped — so an interrupted run simply re-runs next cycle (that IS the
|
||||
recovery). Wall-clock bounded by the task time limits."""
|
||||
from ..services.ml.heads import system_tag_auto_apply_sweep
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
result = system_tag_auto_apply_sweep(session, mode="chrome")
|
||||
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_process_auto_apply",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def scheduled_process_auto_apply() -> str:
|
||||
"""Auto-apply the PROCESS system tags (wip / editor screenshot) on a daily
|
||||
passive sweep (#1464) — provisional source, ring-loud review guard, image stays
|
||||
VISIBLE. No-op unless process_auto_apply_enabled (opt-in). Idempotent —
|
||||
already-tagged/rejected images are skipped — so an interrupted run just re-runs
|
||||
next cycle (the recovery). Wall-clock bounded by the task time limits."""
|
||||
from ..services.ml.heads import system_tag_auto_apply_sweep
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
result = system_tag_auto_apply_sweep(session, mode="process")
|
||||
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def scheduled_soft_wip_conflict_audit() -> str:
|
||||
"""Ring-loud audit over the SOFT WIP-title cohort (#1474) — flag sketch/doodle
|
||||
auto-tags that ALSO look like real content for review. No-op when there are no
|
||||
content heads; idempotent (already-flagged images skipped). Runs regardless of
|
||||
the process-sweep toggle, since soft-title tags come from the importer, not that
|
||||
sweep. Wall-clock bounded by the task time limits."""
|
||||
from ..services.ml.heads import soft_wip_conflict_audit
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
result = soft_wip_conflict_audit(session)
|
||||
return f"scanned={result['n_scanned']} flagged={result['n_flagged']}"
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
|
||||
def prune_presentation_reviews() -> str:
|
||||
"""Retention (rule 89): drop RESOLVED presentation-review flags older than 30
|
||||
days — the operator has acted on them, so they're just history (#141)."""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete as sa_delete
|
||||
|
||||
from ..models import PresentationReview
|
||||
|
||||
cutoff = datetime.now(UTC) - timedelta(days=30)
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
res = session.execute(
|
||||
sa_delete(PresentationReview).where(
|
||||
PresentationReview.resolved_at.is_not(None),
|
||||
PresentationReview.resolved_at < cutoff,
|
||||
)
|
||||
)
|
||||
session.commit()
|
||||
return f"pruned={res.rowcount}"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_retract_auto_tags",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def scheduled_retract_auto_tags() -> str:
|
||||
"""Soft auto-apply (milestone 139): retract standing head_auto/ccip_auto tags
|
||||
the model no longer supports (score now below the auto-apply threshold),
|
||||
skipping operator-confirmed ones. Silent (no hard negative). No-op unless the
|
||||
respective auto-apply switch is on. Returns 'head=N ccip=M'."""
|
||||
from ..services.ml.character_prototypes import retract_auto_applied_ccip
|
||||
from ..services.ml.heads import retract_auto_applied_heads
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
n_head = retract_auto_applied_heads(session)
|
||||
with SessionLocal() as session:
|
||||
n_ccip = retract_auto_applied_ccip(session)
|
||||
return f"head={n_head} ccip={n_ccip}"
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
"""Post-text translation Celery tasks (milestone 143 + re-translate m146).
|
||||
|
||||
Backfills non-English post title/description to the target language via the
|
||||
self-hosted Interpreter LAN service, storing the result + engine_version so
|
||||
viewing is instant and re-runs are cache-fast. Sync (Celery workers are sync),
|
||||
same pattern as the other backfill sweeps. No-op unless translation is enabled,
|
||||
a base URL is set, and the service is healthy.
|
||||
|
||||
Curator does NO language detection of its own: each field's translation is
|
||||
accepted or rejected purely on Interpreter's reported engine + detected language
|
||||
+ confidence (see ``_accept``) — a short English title Interpreter mis-labels as
|
||||
a European language scores below the latin-script floor and is kept as-is.
|
||||
|
||||
Two entry points:
|
||||
- ``translate_posts`` — the periodic untranslated sweep (every 8h;
|
||||
translated_source_lang IS NULL only), one bounded chunk per fire. The Settings
|
||||
"Translate now" button calls it with ``drain=True`` to chase the tail
|
||||
run-until-done and clear the whole backlog in a single press.
|
||||
- ``retranslate_posts`` — after a model change, resets the stored translation
|
||||
columns for a scoped set of posts (all, or a given set of artists) so the
|
||||
untranslated selection re-runs them, then chases the tail until drained
|
||||
(run-until-done). The Interpreter cache keys on engine_version: a changed
|
||||
model genuinely re-translates, an unchanged one is ~1ms cache-fast.
|
||||
"""
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from sqlalchemy import func, or_, select, update
|
||||
|
||||
from ..celery_app import celery
|
||||
from ..models import ImportSettings, Post
|
||||
from ..services import interpreter_client as ic
|
||||
from ..utils.text import html_to_plain
|
||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Bound one run's work so it commits progress rather than dying at the time
|
||||
# limit; the rest resumes next cycle (idempotent — only untranslated posts are
|
||||
# picked, so an interrupted run just re-runs = rule-89 recovery).
|
||||
_MAX_POSTS_PER_RUN = 300
|
||||
# Short gap between run-until-done chunks so a big re-translate finishes on its
|
||||
# own without monopolising the maintenance_long worker in one uninterrupted run.
|
||||
_RETRANSLATE_COUNTDOWN = 5
|
||||
# When Interpreter drains mid-chunk (graceful restart), resume the bulk
|
||||
# re-translate after the service's Retry-After hint if it sent one, else this
|
||||
# default; capped so a bogus/huge hint can't park the work longer than the daily
|
||||
# beat's own safety net would.
|
||||
_INTERRUPT_BACKOFF = 60
|
||||
_INTERRUPT_BACKOFF_MAX = 900
|
||||
|
||||
# --- Translation acceptance gate (Scribe rule 133; contract note #1347) -------
|
||||
# Curator does NO language detection of its own — it accepts or rejects a
|
||||
# translation purely on Interpreter's reported detection. CJK is script-detected
|
||||
# (kana/hangul/kanji) and reliably high-confidence, so ja/ko/zh are trusted
|
||||
# outright (pure-kanji Japanese can even come back labelled zh ~0.75, still
|
||||
# correctly translated). Latin-script detection is a real statistical
|
||||
# probability, so a short English title Interpreter mis-labels as a European
|
||||
# language scores low; require it to clear this floor, else keep the original.
|
||||
# Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned
|
||||
# real langdetect confidence: genuine German detected at 1.0, while a correctly-
|
||||
# detected but ambiguous latin string dipped to 0.86 — so a single constant can't win.
|
||||
# The floor is now operator-tunable (ImportSettings.translation_min_confidence,
|
||||
# milestone 155) with a stricter 0.90 default; per-post overrides
|
||||
# (Post.translation_override) rescue the false negatives it skips. Genuine German
|
||||
# and mis-flagged short English both land ~0.86, and single-word mis-flags sit at
|
||||
# a confident 1.0 no floor catches. Real langdetect fixed some cases at the
|
||||
# source (some short English titles now detect as English → passthrough), so this
|
||||
# floor is a safety net, not the primary fix. Re-tune via the "Test translation"
|
||||
# box (send fresh text — cache hits report 1.0).
|
||||
_CJK_LANGS = frozenset({"ja", "ko", "zh"})
|
||||
_DEFAULT_MIN_LATIN_CONFIDENCE = 0.90
|
||||
|
||||
|
||||
def _interrupt_backoff(retry_after) -> int:
|
||||
"""Seconds to wait before resuming after an Interpreter interruption: the
|
||||
server's Retry-After hint (capped), else the default."""
|
||||
if retry_after and retry_after > 0:
|
||||
return int(min(retry_after, _INTERRUPT_BACKOFF_MAX))
|
||||
return _INTERRUPT_BACKOFF
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.translation.translate_posts",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def translate_posts(drain: bool = False) -> str:
|
||||
"""Translate untranslated non-English post title/description (default target
|
||||
en) via Interpreter. Per-post [title, description] batch → per-post detected
|
||||
language. Passthrough / already-target posts are marked handled (source ==
|
||||
target) with no stored translation. 503 or a connection error interrupts the
|
||||
run (retry next cycle); 400 stops it (fix config); posts done so far stay
|
||||
committed. No-op unless configured + healthy. Returns a summary string.
|
||||
|
||||
``drain`` (the Settings "Translate now" button) chases the tail
|
||||
run-until-done: on a clean chunk with work still remaining it re-enqueues
|
||||
itself until the untranslated backlog is zero, like ``retranslate_posts``. The
|
||||
periodic beat leaves it False → one bounded chunk per fire (the 8h cadence
|
||||
drives the rest, enough for the trickle of newly-imported posts)."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
ready = _translation_config(session)
|
||||
if isinstance(ready, str):
|
||||
return ready
|
||||
base_url, target, min_confidence = ready
|
||||
posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
|
||||
status, translated, retry_after = _translate_batch(
|
||||
session, posts, base_url, target, min_confidence,
|
||||
)
|
||||
# Drain mode (manual "Translate now"): chase the tail until the backlog is
|
||||
# zero, so one press clears the whole pile instead of one 300-chunk.
|
||||
# Termination is guaranteed — every handled post leaves the untranslated
|
||||
# set, so the remaining count strictly decreases each chunk.
|
||||
if status == "ok" and drain:
|
||||
remaining = _count_untranslated(session, None)
|
||||
if remaining:
|
||||
translate_posts.apply_async(
|
||||
(), {"drain": True}, countdown=_RETRANSLATE_COUNTDOWN,
|
||||
)
|
||||
log.info(
|
||||
"translate_posts: draining — %d remaining → re-enqueued",
|
||||
remaining,
|
||||
)
|
||||
# Interpreter drained mid-chunk (graceful restart): don't make a manual
|
||||
# "Translate now" wait a whole beat cycle — re-enqueue after its
|
||||
# Retry-After hint (or the default backoff), preserving `drain` so a bulk
|
||||
# drain resumes cleanly. Self-terminating: once the service is down the
|
||||
# health gate returns "interpreter unavailable" early.
|
||||
elif status == "interrupted" and _count_untranslated(session, None):
|
||||
delay = _interrupt_backoff(retry_after)
|
||||
translate_posts.apply_async((), {"drain": drain}, countdown=delay)
|
||||
log.info(
|
||||
"translate_posts: interrupted (service draining) → retry in %ss",
|
||||
delay,
|
||||
)
|
||||
return _summary(status, translated, len(posts))
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.translation.retranslate_posts",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
|
||||
"""Re-translate posts after a model change. On the first call resets the
|
||||
stored translation columns to NULL for the scoped posts — all artists when
|
||||
``artist_ids`` is falsy, else ``WHERE artist_id IN artist_ids`` — so the
|
||||
normal untranslated selection re-runs them through Interpreter. Then handles
|
||||
one bounded chunk and re-enqueues itself until the scoped backlog is drained
|
||||
(run-until-done; ``_reset_done`` guards against wiping fresh work on the
|
||||
follow-up chunks). No reset happens unless the service is configured +
|
||||
healthy, so translations are never cleared when they can't be rebuilt."""
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
ready = _translation_config(session)
|
||||
if isinstance(ready, str):
|
||||
return ready
|
||||
base_url, target, min_confidence = ready
|
||||
|
||||
if not _reset_done:
|
||||
n = _reset_translations(session, artist_ids)
|
||||
log.info(
|
||||
"retranslate_posts: reset %d post(s) (artist_ids=%s)",
|
||||
n, artist_ids,
|
||||
)
|
||||
|
||||
posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN)
|
||||
status, translated, retry_after = _translate_batch(
|
||||
session, posts, base_url, target, min_confidence,
|
||||
)
|
||||
|
||||
# run-until-done: chase the tail when the chunk finished cleanly.
|
||||
# Termination is guaranteed: every handled post leaves the NULL set, so
|
||||
# the remaining count strictly decreases each chunk.
|
||||
if status == "ok":
|
||||
remaining = _count_untranslated(session, artist_ids)
|
||||
if remaining:
|
||||
retranslate_posts.apply_async(
|
||||
(),
|
||||
{"artist_ids": artist_ids, "_reset_done": True},
|
||||
countdown=_RETRANSLATE_COUNTDOWN,
|
||||
)
|
||||
log.info(
|
||||
"retranslate_posts: %d remaining → re-enqueued", remaining,
|
||||
)
|
||||
# Interpreter drained mid-chunk (graceful restart / 429 / 5xx): don't
|
||||
# stall the bulk re-translate until the daily beat — resume it after the
|
||||
# service's Retry-After hint (or the default backoff), with
|
||||
# _reset_done=True so the fresh work is never re-wiped. Self-terminating:
|
||||
# if the service is still down next run, _translation_config's health gate
|
||||
# returns "interpreter unavailable" early and nothing re-enqueues.
|
||||
elif status == "interrupted" and _count_untranslated(session, artist_ids):
|
||||
delay = _interrupt_backoff(retry_after)
|
||||
retranslate_posts.apply_async(
|
||||
(),
|
||||
{"artist_ids": artist_ids, "_reset_done": True},
|
||||
countdown=delay,
|
||||
)
|
||||
log.info(
|
||||
"retranslate_posts: interrupted (service draining) → retry in %ss",
|
||||
delay,
|
||||
)
|
||||
return _summary(status, translated, len(posts))
|
||||
|
||||
|
||||
def _translation_config(session):
|
||||
"""Resolve (base_url, target, min_confidence) if translation is enabled +
|
||||
healthy, else a short status string ("disabled" / "interpreter unavailable")
|
||||
for the caller to return. Centralises the guard so translate/retranslate agree
|
||||
exactly, including the operator-tunable acceptance floor."""
|
||||
cfg = ImportSettings.load_sync(session)
|
||||
if not cfg.translation_enabled or not cfg.interpreter_base_url.strip():
|
||||
return "disabled"
|
||||
base_url = cfg.interpreter_base_url.strip()
|
||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
||||
min_confidence = cfg.translation_min_confidence
|
||||
if not ic.health(base_url):
|
||||
return "interpreter unavailable"
|
||||
return base_url, target, min_confidence
|
||||
|
||||
|
||||
def _untranslated_filter(stmt, artist_ids):
|
||||
"""Add the untranslated-post predicate (+ optional artist scope) to a
|
||||
select/count over Post. Untranslated = translated_source_lang IS NULL with
|
||||
some text to translate."""
|
||||
stmt = stmt.where(
|
||||
Post.translated_source_lang.is_(None)
|
||||
).where(or_(
|
||||
Post.post_title.is_not(None), Post.description.is_not(None),
|
||||
))
|
||||
if artist_ids:
|
||||
stmt = stmt.where(Post.artist_id.in_(artist_ids))
|
||||
return stmt
|
||||
|
||||
|
||||
def _select_untranslated(session, artist_ids, limit):
|
||||
return session.execute(
|
||||
_untranslated_filter(select(Post), artist_ids).limit(limit)
|
||||
).scalars().all()
|
||||
|
||||
|
||||
def _count_untranslated(session, artist_ids) -> int:
|
||||
return int(session.execute(
|
||||
_untranslated_filter(select(func.count(Post.id)), artist_ids)
|
||||
).scalar_one())
|
||||
|
||||
|
||||
def _reset_translations(session, artist_ids) -> int:
|
||||
"""Clear the stored translation columns for scoped, already-handled posts so
|
||||
the untranslated sweep re-runs them. Only touches rows that were translated
|
||||
(translated_source_lang IS NOT NULL) — untranslated rows are already NULL.
|
||||
Skips 'keep original' posts (translation_override = 'original') so that choice
|
||||
survives a Re-translate-all (milestone 155). Returns the row count reset
|
||||
(commits it)."""
|
||||
stmt = (
|
||||
update(Post)
|
||||
.where(Post.translated_source_lang.is_not(None))
|
||||
.where(Post.translation_override != "original")
|
||||
.values(
|
||||
post_title_translated=None,
|
||||
description_translated=None,
|
||||
translated_source_lang=None,
|
||||
translation_engine_version=None,
|
||||
translated_at=None,
|
||||
)
|
||||
)
|
||||
if artist_ids:
|
||||
stmt = stmt.where(Post.artist_id.in_(artist_ids))
|
||||
n = session.execute(stmt).rowcount
|
||||
session.commit()
|
||||
return n
|
||||
|
||||
|
||||
def _translate_batch(session, posts, base_url: str, target: str, min_confidence: float):
|
||||
"""Translate each post in the chunk with a per-post commit (an interrupted
|
||||
run keeps progress). Returns (status, translated, retry_after) where status is
|
||||
one of "ok" / "interrupted" (unavailable/drain) / "stopped" (400) / "timeout";
|
||||
retry_after is the server's Retry-After hint in seconds on an interrupt, else
|
||||
None."""
|
||||
translated = 0
|
||||
for post in posts:
|
||||
try:
|
||||
translated += _translate_one(session, post, base_url, target, min_confidence)
|
||||
session.commit()
|
||||
except ic.InterpreterUnavailable as e:
|
||||
session.rollback()
|
||||
return "interrupted", translated, getattr(e, "retry_after", None)
|
||||
except ic.InterpreterBadRequest as e:
|
||||
session.rollback()
|
||||
log.warning("translate bad request: %s", e)
|
||||
return "stopped", translated, None
|
||||
except SoftTimeLimitExceeded:
|
||||
return "timeout", translated, None
|
||||
return "ok", translated, None
|
||||
|
||||
|
||||
def _summary(status: str, translated: int, scanned: int) -> str:
|
||||
if status == "interrupted":
|
||||
return f"interrupted (service down) — translated={translated}"
|
||||
if status == "stopped":
|
||||
return f"stopped (bad request) — translated={translated}"
|
||||
if status == "timeout":
|
||||
return f"time limit — translated={translated}"
|
||||
return f"translated={translated} scanned={scanned}"
|
||||
|
||||
|
||||
def _accept(
|
||||
language: str, confidence, min_confidence: float = _DEFAULT_MIN_LATIN_CONFIDENCE,
|
||||
) -> bool:
|
||||
"""Should curator store Interpreter's translation, or keep the original?
|
||||
Consumes ONLY Interpreter's own detection (curator does none, Scribe rule
|
||||
133): CJK languages are trusted regardless of the reported number; a
|
||||
latin-script detection must clear ``min_confidence`` (the operator-tunable
|
||||
ImportSettings.translation_min_confidence). A missing confidence fails OPEN
|
||||
(accept), so a service that omits the field never silently drops a
|
||||
translation. Per-post ``force`` overrides bypass this entirely (see
|
||||
``_translate_one``)."""
|
||||
if (language or "").split("-", 1)[0].lower() in _CJK_LANGS:
|
||||
return True
|
||||
if confidence is None:
|
||||
return True
|
||||
return confidence >= min_confidence
|
||||
|
||||
|
||||
def _translate_field(
|
||||
text: str, base_url: str, target: str, min_confidence: float, force: bool = False,
|
||||
):
|
||||
"""Translate ONE field independently. Returns (translated, source_lang,
|
||||
engine_version), or (None, None, None) when there's nothing to store — empty
|
||||
text, already the target language, a passthrough (engine "none"), or a
|
||||
latin-script detection the acceptance gate rejects as too low-confidence (kept
|
||||
as the original). ``force`` (a per-post 'force' override) bypasses the
|
||||
confidence gate so a legitimately-foreign title Interpreter is only ~0.86 sure
|
||||
about is still stored; a genuine passthrough stores nothing regardless (there
|
||||
is no translation to force). Per-field (not aggregate first-item) detection so
|
||||
a non-English description still translates when the title is already English."""
|
||||
if not text:
|
||||
return None, None, None
|
||||
res = ic.translate([text], base_url=base_url, target=target)
|
||||
detected = res["detected_lang"] or target
|
||||
if detected == target or res["engine"] == "none":
|
||||
return None, None, None
|
||||
# Trust only Interpreter's own detection (Scribe rule 133): keep the original
|
||||
# when a latin-script detection doesn't clear the confidence floor, so a short
|
||||
# English title mis-labelled as e.g. German isn't rewritten into the archive.
|
||||
# A 'force' override skips the gate — the operator is overriding a rejection.
|
||||
if not force and not _accept(
|
||||
detected, res.get("detected_confidence"), min_confidence,
|
||||
):
|
||||
return None, None, None
|
||||
return res["translations"][0], detected, res["engine_version"]
|
||||
|
||||
|
||||
def _store_translation(post, title_res, desc_res, target: str) -> int:
|
||||
"""Apply per-field (translated, source_lang, engine_version) results to a post
|
||||
in place. Returns 1 if a translation was stored, 0 when the post is all
|
||||
passthrough/empty — in which case the translated columns are CLEARED and the
|
||||
post is marked handled (translated_source_lang = target). Clearing (not just
|
||||
leaving) matters on re-apply: a 'keep original' override, or a re-translate
|
||||
that now rejects a previously-accepted mis-flag, must remove the stale
|
||||
translation, not keep it."""
|
||||
title_tr, title_lang, title_ev = title_res
|
||||
desc_tr, desc_lang, desc_ev = desc_res
|
||||
if title_tr is None and desc_tr is None:
|
||||
post.post_title_translated = None
|
||||
post.description_translated = None
|
||||
post.translated_source_lang = target
|
||||
post.translation_engine_version = None
|
||||
post.translated_at = None
|
||||
return 0
|
||||
post.post_title_translated = title_tr
|
||||
post.description_translated = desc_tr
|
||||
# Source lang = whichever field was actually non-target (for a mixed post,
|
||||
# the translated field's language, not the already-English one).
|
||||
post.translated_source_lang = title_lang or desc_lang
|
||||
post.translation_engine_version = title_ev or desc_ev
|
||||
post.translated_at = datetime.now(UTC)
|
||||
return 1
|
||||
|
||||
|
||||
def _translate_one(
|
||||
session, post, base_url: str, target: str, min_confidence: float,
|
||||
) -> int:
|
||||
"""Translate a post's title/description in place, each field independently,
|
||||
honoring the per-post override (milestone 155): 'original' keeps the original
|
||||
(clears any stored translation, no Interpreter call); 'force' translates even
|
||||
below the confidence floor; 'auto' (default) runs the acceptance gate. Returns
|
||||
1 if it stored a translation, else 0 (still marks the post handled so the sweep
|
||||
won't revisit it)."""
|
||||
if post.translation_override == "original":
|
||||
return _store_translation(post, (None, None, None), (None, None, None), target)
|
||||
force = post.translation_override == "force"
|
||||
title = (post.post_title or "").strip()
|
||||
desc = (html_to_plain(post.description) if post.description else "") or ""
|
||||
desc = desc.strip()
|
||||
title_res = _translate_field(title, base_url, target, min_confidence, force=force)
|
||||
desc_res = _translate_field(desc, base_url, target, min_confidence, force=force)
|
||||
return _store_translation(post, title_res, desc_res, target)
|
||||
@@ -8,6 +8,35 @@
|
||||
# run `docker compose up` from this directory and switches images to
|
||||
# local builds + DEBUG logging.
|
||||
|
||||
# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a
|
||||
# time, START the new task before stopping the old (zero-downtime via the ingress
|
||||
# mesh), and if the new task doesn't reach a healthy state within `monitor`, roll
|
||||
# back to the previous image automatically. `monitor` is sized above web's
|
||||
# healthcheck start_period so a broken image that never goes healthy is caught.
|
||||
# Plain `docker compose up` ignores `deploy:` (it warns + skips), so the dev
|
||||
# override is unaffected. Referenced by each long-lived service below.
|
||||
x-deploy-policy: &deploy_policy
|
||||
update_config:
|
||||
order: start-first
|
||||
failure_action: rollback
|
||||
monitor: 90s
|
||||
rollback_config:
|
||||
order: start-first
|
||||
restart_policy:
|
||||
condition: any
|
||||
delay: 10s
|
||||
|
||||
# Worker liveness: ping THIS container's celery node over the broker. Lenient
|
||||
# (60s interval, 3 retries, 60s start_period) so a transient broker blip never
|
||||
# false-flags a worker into a rollback. `$$HOSTNAME` → `$HOSTNAME` for the shell;
|
||||
# celery's default node name is celery@<hostname> (the container id).
|
||||
x-celery-healthcheck: &celery_healthcheck
|
||||
test: ["CMD-SHELL", "celery -A backend.app.celery_app:celery inspect ping -d celery@$$HOSTNAME --timeout 10 >/dev/null 2>&1"]
|
||||
interval: 60s
|
||||
timeout: 15s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
@@ -47,6 +76,24 @@ services:
|
||||
web:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
command: ["web"]
|
||||
# Graceful shutdown: give the container time to drain in-flight work on a
|
||||
# deploy (docker SIGTERMs, then SIGKILLs after this window — default is only
|
||||
# 10s, far too short for real jobs). Hypercorn/Celery both warm-shut-down on
|
||||
# SIGTERM; per-lane values sized to typical task length. Anything that still
|
||||
# outruns the window is re-queued (task_reject_on_worker_lost) and re-driven
|
||||
# by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP
|
||||
# requests + the occasional file download.
|
||||
stop_grace_period: 30s
|
||||
# Liveness for rolling deploys: /api/health is a no-DB 200 (just proves the
|
||||
# app booted + serves HTTP after `alembic upgrade head`). start_period covers
|
||||
# the migration + boot so a slow start isn't mis-flagged.
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/api/health', timeout=5).status==200 else 1)\""]
|
||||
interval: 15s
|
||||
timeout: 6s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
deploy: *deploy_policy
|
||||
ports:
|
||||
- "${PORT:-8080}:8080"
|
||||
environment: &app_env
|
||||
@@ -77,6 +124,10 @@ services:
|
||||
worker:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
command: ["worker"]
|
||||
# Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
|
||||
stop_grace_period: 90s
|
||||
healthcheck: *celery_healthcheck
|
||||
deploy: *deploy_policy
|
||||
environment:
|
||||
<<: *app_env
|
||||
CELERY_QUEUES: default,import,thumbnail,download
|
||||
@@ -93,6 +144,10 @@ services:
|
||||
scheduler:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
command: ["scheduler"]
|
||||
# Quick maintenance/scan lane + beat — short tasks, modest drain window.
|
||||
stop_grace_period: 60s
|
||||
healthcheck: *celery_healthcheck
|
||||
deploy: *deploy_policy
|
||||
environment:
|
||||
<<: *app_env
|
||||
CELERY_QUEUES: maintenance,scan
|
||||
@@ -110,6 +165,12 @@ services:
|
||||
maintenance-long:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
command: ["worker"]
|
||||
# Longest lane (DB backups, library audits, translation backfill) — give it
|
||||
# the most room to finish a chunk gracefully. Chunked + idempotent, so a job
|
||||
# that still outruns this resumes cleanly next run rather than corrupting.
|
||||
stop_grace_period: 180s
|
||||
healthcheck: *celery_healthcheck
|
||||
deploy: *deploy_policy
|
||||
environment:
|
||||
<<: *app_env
|
||||
CELERY_QUEUES: maintenance_long
|
||||
@@ -125,6 +186,10 @@ services:
|
||||
ml-worker:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev
|
||||
command: ["ml-worker"]
|
||||
# A single GPU inference pass can run tens of seconds — let it finish.
|
||||
stop_grace_period: 120s
|
||||
healthcheck: *celery_healthcheck
|
||||
deploy: *deploy_policy
|
||||
environment:
|
||||
<<: *app_env
|
||||
volumes:
|
||||
|
||||
@@ -31,6 +31,69 @@ browser.runtime.onInstalled.addListener(() => ensureInitialized());
|
||||
browser.runtime.onStartup.addListener(() => ensureInitialized());
|
||||
ensureInitialized().catch(e => console.error('init failed:', e));
|
||||
|
||||
// ---- Extension self-update check (#1489) ----
|
||||
// Installed per-instance from the operator's FC host, so Firefox's static
|
||||
// update_url can't apply (each instance has a different host). Instead ask the
|
||||
// configured backend for the latest published version and nudge the operator to
|
||||
// reinstall the freshly-signed XPI — surfaced as a popup banner (on demand) and
|
||||
// a toolbar badge (daily). /api/extension/manifest is public and returns
|
||||
// {version, latest_url, sha256}; the XPI is served from the web root (not /api).
|
||||
|
||||
function versionIsNewer(candidate, current) {
|
||||
// Dotted numeric compare so 1.0.10 > 1.0.9 (a plain string compare wouldn't).
|
||||
const a = String(candidate).split('.').map(n => parseInt(n, 10) || 0);
|
||||
const b = String(current).split('.').map(n => parseInt(n, 10) || 0);
|
||||
for (let i = 0; i < Math.max(a.length, b.length); i++) {
|
||||
if ((a[i] || 0) !== (b[i] || 0)) return (a[i] || 0) > (b[i] || 0);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
async function checkForUpdateInfo() {
|
||||
await ensureInitialized();
|
||||
if (!api.isConfigured()) return { updateAvailable: false, configured: false };
|
||||
let info;
|
||||
try {
|
||||
info = await api.getExtensionManifest();
|
||||
} catch (e) {
|
||||
return { updateAvailable: false, error: e.message };
|
||||
}
|
||||
const currentVersion = browser.runtime.getManifest().version;
|
||||
const latestVersion = info && info.version ? info.version : null;
|
||||
// latest_url is served from the web root; strip the /api suffix off baseUrl
|
||||
// (same transform as OPEN_ARTIST_PAGE).
|
||||
const base = (api.baseUrl || '').replace(/\/+$/, '').replace(/\/api$/, '');
|
||||
return {
|
||||
updateAvailable: !!latestVersion && versionIsNewer(latestVersion, currentVersion),
|
||||
currentVersion,
|
||||
latestVersion,
|
||||
xpiUrl: info && info.latest_url ? `${base}${info.latest_url}` : null,
|
||||
};
|
||||
}
|
||||
|
||||
async function refreshUpdateBadge() {
|
||||
let r;
|
||||
try { r = await checkForUpdateInfo(); } catch { return; }
|
||||
try {
|
||||
await browser.action.setBadgeText({ text: r.updateAvailable ? '↑' : '' });
|
||||
if (r.updateAvailable) {
|
||||
await browser.action.setBadgeBackgroundColor({ color: '#F4BA7A' });
|
||||
await browser.action.setTitle({ title: `FabledCurator — update available (v${r.latestVersion})` });
|
||||
} else {
|
||||
await browser.action.setTitle({ title: 'FabledCurator' });
|
||||
}
|
||||
} catch { /* action API unavailable — non-fatal */ }
|
||||
}
|
||||
|
||||
// Daily proactive check (needs the "alarms" permission). create() is idempotent
|
||||
// by name, so re-running it on each event-page load is safe.
|
||||
browser.alarms.create('fc-update-check', { periodInMinutes: 24 * 60, delayInMinutes: 1 });
|
||||
browser.alarms.onAlarm.addListener((alarm) => {
|
||||
if (alarm.name === 'fc-update-check') refreshUpdateBadge();
|
||||
});
|
||||
browser.runtime.onStartup.addListener(() => refreshUpdateBadge());
|
||||
browser.runtime.onInstalled.addListener(() => refreshUpdateBadge());
|
||||
|
||||
// ---- Discord token capture via webRequest ----
|
||||
|
||||
browser.webRequest.onBeforeSendHeaders.addListener(
|
||||
@@ -298,6 +361,9 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
}
|
||||
}
|
||||
|
||||
case 'CHECK_UPDATE':
|
||||
return await checkForUpdateInfo();
|
||||
|
||||
default:
|
||||
return { error: `Unknown message type: ${msg.type}` };
|
||||
}
|
||||
|
||||
@@ -89,6 +89,12 @@ class FabledCuratorAPI {
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
return this.request('GET', `/extension/probe?${qs}`);
|
||||
}
|
||||
// Latest published extension version on this instance — drives the in-app
|
||||
// update prompt. Public endpoint (no key needed, but request() sends it
|
||||
// harmlessly). Returns {version, xpi_url, latest_url, sha256}.
|
||||
getExtensionManifest() {
|
||||
return this.request('GET', '/extension/manifest');
|
||||
}
|
||||
|
||||
// Connection test = the cheapest read with auth.
|
||||
testConnection() {
|
||||
|
||||
@@ -86,7 +86,16 @@ const PLATFORMS = {
|
||||
* script to decide whether to show the floating "Add as source" button.
|
||||
*/
|
||||
const PLATFORM_ARTIST_PATTERNS = {
|
||||
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?!home$|search\b|messages\b|notifications\b|library\b|settings\b|posts\b|c\/)[^/?#]+\/?$/i,
|
||||
// Patreon serves the same creator under three URL shapes (see backend
|
||||
// patreon_resolver._VANITY_RE): bare `patreon.com/Atole`, `c/` prefix, and
|
||||
// `cw/` "creator workspace" — the last is the URL you land on once you're
|
||||
// SUBSCRIBED, which is exactly when the button matters. Match all three, and
|
||||
// drop the single-segment end-anchor so a creator's inner page
|
||||
// (…/cw/Atole/posts, …/Atole/membership) also injects the button. Nav pages
|
||||
// (home/search/…/posts permalink) stay excluded. Mirrors extension_service
|
||||
// ._PLATFORM_PATTERNS — keep in sync (operator-flagged 2026-07-13: button
|
||||
// vanished once subscribed because the old pattern only matched the bare root).
|
||||
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i,
|
||||
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
||||
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
||||
deviantart: /^https?:\/\/(www\.)?deviantart\.com\/(?!home$|watch\b|tag\b|browse\b)[^/?#]+\/?$/i,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"manifest_version": 3,
|
||||
"name": "FabledCurator",
|
||||
"version": "1.0.7",
|
||||
"version": "1.0.9",
|
||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||
|
||||
"browser_specific_settings": {
|
||||
@@ -22,7 +22,8 @@
|
||||
"tabs",
|
||||
"activeTab",
|
||||
"webRequest",
|
||||
"webRequestBlocking"
|
||||
"webRequestBlocking",
|
||||
"alarms"
|
||||
],
|
||||
|
||||
"host_permissions": [
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "fabledcurator-extension",
|
||||
"version": "1.0.7",
|
||||
"version": "1.0.9",
|
||||
"private": true,
|
||||
"description": "Firefox extension for FabledCurator",
|
||||
"scripts": {
|
||||
@@ -10,6 +10,6 @@
|
||||
"sign": "web-ext sign --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||
},
|
||||
"devDependencies": {
|
||||
"web-ext": "^8.0.0"
|
||||
"web-ext": "^10.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +72,17 @@ body {
|
||||
.btn.block { display: block; width: 100%; margin-top: 8px; }
|
||||
.btn.link { background: none; color: var(--on-surface-variant); padding: 4px; }
|
||||
.btn.link:hover { color: var(--accent); }
|
||||
.btn.small { padding: 6px 12px; font-size: 13px; }
|
||||
|
||||
/* In-app update prompt (accent-tinted so it reads as an actionable notice). */
|
||||
.update-banner {
|
||||
display: flex; align-items: center; gap: 10px;
|
||||
margin: 10px 10px 0; padding: 10px 12px;
|
||||
background: rgba(244, 186, 122, 0.12);
|
||||
border: 1px solid rgba(244, 186, 122, 0.4);
|
||||
border-radius: 6px;
|
||||
}
|
||||
#update-text { flex: 1; font-size: 13px; }
|
||||
|
||||
.source-row .play {
|
||||
background: none; border: none; color: var(--on-surface-variant);
|
||||
|
||||
@@ -20,6 +20,11 @@
|
||||
</section>
|
||||
|
||||
<section id="main-content" class="main hidden">
|
||||
<div id="update-banner" class="update-banner hidden">
|
||||
<span id="update-text"></span>
|
||||
<button id="update-btn" class="btn primary small">Update</button>
|
||||
</div>
|
||||
|
||||
<nav class="tabs">
|
||||
<button class="tab active" data-tab="platforms">Platforms</button>
|
||||
<button class="tab" data-tab="sources">Sources</button>
|
||||
|
||||
@@ -14,6 +14,7 @@ async function init() {
|
||||
setupEventListeners();
|
||||
showPlatformsLoading();
|
||||
testConnectionIfNeeded();
|
||||
checkForUpdate();
|
||||
loadPlatformStatus().catch(e => showError(`Failed to load platforms: ${e.message}`));
|
||||
} catch (e) {
|
||||
showSetupRequired();
|
||||
@@ -63,6 +64,26 @@ function updateConnectionDot(connected) {
|
||||
d.title = connected ? 'Connected to FabledCurator' : 'Disconnected';
|
||||
}
|
||||
|
||||
// Nudge to reinstall when the configured instance publishes a newer signed XPI
|
||||
// (the extension is self-hosted, so there's no Firefox auto-update). Never
|
||||
// blocks the popup — a failed check just leaves the banner hidden.
|
||||
async function checkForUpdate() {
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_UPDATE' });
|
||||
if (r && r.updateAvailable && r.xpiUrl) showUpdateBanner(r);
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
|
||||
function showUpdateBanner(r) {
|
||||
document.getElementById('update-text').textContent =
|
||||
`Update available — v${r.latestVersion} (installed v${r.currentVersion})`;
|
||||
// Opening the signed XPI triggers Firefox's native install prompt.
|
||||
document.getElementById('update-btn').addEventListener('click', () => {
|
||||
browser.tabs.create({ url: r.xpiUrl });
|
||||
});
|
||||
document.getElementById('update-banner').classList.remove('hidden');
|
||||
}
|
||||
|
||||
async function loadPlatformStatus() {
|
||||
const status = await browser.runtime.sendMessage({ type: 'GET_PLATFORM_STATUS' });
|
||||
const c = document.getElementById('platforms-list');
|
||||
|
||||
+11
-13
@@ -4,30 +4,28 @@
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
"node": ">=24"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"test:unit": "vitest run",
|
||||
"check": "vue-tsc --noEmit"
|
||||
"test:unit": "vitest run"
|
||||
},
|
||||
"dependencies": {
|
||||
"vue": "^3.4.0",
|
||||
"vue-router": "^4.3.0",
|
||||
"pinia": "^2.1.0",
|
||||
"vuetify": "^3.5.0",
|
||||
"vue": "^3.5.0",
|
||||
"vue-router": "^5.0.0",
|
||||
"pinia": "^3.0.0",
|
||||
"vuetify": "^4.0.0",
|
||||
"@mdi/font": "^7.4.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@vitejs/plugin-vue": "^5.0.0",
|
||||
"vite": "^5.2.0",
|
||||
"vue-tsc": "^2.0.0",
|
||||
"vite-plugin-vuetify": "^2.0.0",
|
||||
"@vitejs/plugin-vue": "^6.0.0",
|
||||
"vite": "^8.0.0",
|
||||
"vite-plugin-vuetify": "^2.1.0",
|
||||
"sass": "^1.71.0",
|
||||
"vitest": "^2.1.0",
|
||||
"vitest": "^4.0.0",
|
||||
"@vue/test-utils": "^2.4.0",
|
||||
"happy-dom": "^15.0.0"
|
||||
"happy-dom": "^20.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,11 +18,11 @@ const route = useRoute()
|
||||
<style scoped>
|
||||
.fc-content {
|
||||
min-height: 100vh;
|
||||
/* Push initial viewport content below the sticky TopNav. Without
|
||||
this, some views' first rows / form fields / table headers can
|
||||
end up obscured by the navbar (depending on parent overflow
|
||||
context interacting with position: sticky). Scrolled-down content
|
||||
still slides under the nav — the gradient-fade design is intact. */
|
||||
padding-top: 64px;
|
||||
/* NO padding-top: the TopNav is position:sticky, so it already reserves its
|
||||
own space in the v-app flex column — content flows directly below it. The
|
||||
old 64px padding-top was a leftover from a FIXED navbar and double-counted
|
||||
that space, leaving a large empty band at the top of EVERY view (and pushing
|
||||
the full-height calc(100vh - 64px) views down so they overflowed). Removed
|
||||
2026-07-13. Scrolled content still slides under the sticky nav as before. */
|
||||
}
|
||||
</style>
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
<template>
|
||||
<v-snackbar
|
||||
v-model="show" :color="color" location="bottom right" timeout="4000"
|
||||
multi-line elevation="4"
|
||||
min-height="68" elevation="4"
|
||||
>
|
||||
{{ message }}
|
||||
<template #actions>
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<header class="fc-topnav">
|
||||
<header ref="navEl" class="fc-topnav" :class="{ 'fc-topnav--chrome': hasStickyChrome }">
|
||||
<div class="fc-nav-left">
|
||||
<RouterLink :to="FRONT_DOOR" class="fc-brand" aria-label="FabledCurator home">
|
||||
<img src="/favicon.svg" alt="" class="fc-brand__glyph" width="22" height="22" />
|
||||
@@ -64,13 +64,39 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed, onMounted } from 'vue'
|
||||
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
|
||||
import { useRoute } from 'vue-router'
|
||||
import router, { FRONT_DOOR } from '../router.js'
|
||||
import { useSystemStore } from '../stores/system.js'
|
||||
import PipelineStatusChip from './PipelineStatusChip.vue'
|
||||
|
||||
const system = useSystemStore()
|
||||
onMounted(() => system.refreshHealth())
|
||||
|
||||
// Publish the nav's REAL height as --fc-nav-h so full-height workspaces
|
||||
// (Explore/Subscriptions) and sticky sub-headers pin to it exactly instead of a
|
||||
// hardcoded 64px that Vuetify 4's MD3 sizing broke — the Explore breadcrumb was
|
||||
// tucking under a taller nav (#1481). ResizeObserver keeps it live as the nav
|
||||
// reflows (per-view teleported actions, mobile breakpoint, chip state changes).
|
||||
const navEl = ref(null)
|
||||
let navRO = null
|
||||
onMounted(() => {
|
||||
system.refreshHealth()
|
||||
if (navEl.value && 'ResizeObserver' in window) {
|
||||
navRO = new ResizeObserver(() => {
|
||||
const h = navEl.value?.offsetHeight
|
||||
if (h) document.documentElement.style.setProperty('--fc-nav-h', `${h}px`)
|
||||
})
|
||||
navRO.observe(navEl.value)
|
||||
}
|
||||
})
|
||||
onBeforeUnmount(() => { navRO?.disconnect() })
|
||||
|
||||
// Views that pin a sticky sub-header (filter bar / tabs) directly under the nav
|
||||
// declare `meta.stickyChrome`. On those, the nav doesn't fade to transparent at
|
||||
// its bottom — it hands off at the shared seam alpha so the sub-header can
|
||||
// continue the SAME fade (see .fc-chrome-continues in app.css). One gradient.
|
||||
const route = useRoute()
|
||||
const hasStickyChrome = computed(() => !!route.meta?.stickyChrome)
|
||||
|
||||
// Every route with a meta.title is a nav entry. Order by meta.navOrder —
|
||||
// router.getRoutes() does NOT guarantee declaration order, so explicit numbers
|
||||
@@ -119,16 +145,35 @@ const health = computed(() => {
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
padding: 0.75rem 1rem;
|
||||
/* Obsidian (#14171A = 20,23,26) gradient fade — content scrolls under it. */
|
||||
/* Obsidian (#14171A) fade — content scrolls under it. Holds high (0.92 →
|
||||
0.84) through the top half, then eases to transparent over the bottom
|
||||
quarter so it tails off softly instead of a straight line to a hard edge
|
||||
(operator 2026-07-13). Shared --fc-chrome-rgb keeps it in sync with the
|
||||
sub-header continuation. */
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
rgba(20, 23, 26, 0.92) 0%,
|
||||
rgba(20, 23, 26, 0.65) 60%,
|
||||
rgba(20, 23, 26, 0) 100%
|
||||
rgba(var(--fc-chrome-rgb), 0.92) 0%,
|
||||
rgba(var(--fc-chrome-rgb), 0.84) 50%,
|
||||
rgba(var(--fc-chrome-rgb), 0.55) 75%,
|
||||
rgba(var(--fc-chrome-rgb), 0) 100%
|
||||
);
|
||||
backdrop-filter: blur(2px);
|
||||
-webkit-backdrop-filter: blur(2px);
|
||||
}
|
||||
/* On a view with a sticky sub-header pinned beneath (meta.stickyChrome), the nav
|
||||
stops fading at the shared seam alpha instead of going fully transparent — the
|
||||
sub-header (.fc-chrome-continues) picks the fade up from there, so the two read
|
||||
as one continuous gradient. Compound selector out-specifies .fc-topnav so it
|
||||
wins regardless of Vite's production CSS ordering. --fc-chrome-* come from the
|
||||
global :root in app.css (custom props inherit into scoped styles). */
|
||||
.fc-topnav.fc-topnav--chrome {
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
rgba(var(--fc-chrome-rgb), 0.92) 0%,
|
||||
rgba(var(--fc-chrome-rgb), 0.84) 60%,
|
||||
rgba(var(--fc-chrome-rgb), var(--fc-chrome-seam)) 100%
|
||||
);
|
||||
}
|
||||
|
||||
.fc-brand {
|
||||
display: flex;
|
||||
|
||||
@@ -1,8 +1,21 @@
|
||||
<template>
|
||||
<header class="fc-artist-header">
|
||||
<div class="fc-artist-header__left">
|
||||
<h1 class="fc-artist-header__name">{{ name }}</h1>
|
||||
<span v-if="stats" class="fc-artist-header__stats">{{ stats }}</span>
|
||||
<template v-if="!editing">
|
||||
<h1 class="fc-artist-header__name">{{ name }}</h1>
|
||||
<v-icon
|
||||
class="fc-artist-header__edit" size="16" icon="mdi-pencil"
|
||||
title="Rename artist (display name only)" @click="startEdit"
|
||||
/>
|
||||
</template>
|
||||
<v-text-field
|
||||
v-else
|
||||
v-model="draft"
|
||||
density="compact" variant="outlined" hide-details autofocus
|
||||
class="fc-artist-header__edit-field"
|
||||
@keydown.enter="submit" @keydown.esc="cancel" @blur="cancel"
|
||||
/>
|
||||
<span v-if="stats && !editing" class="fc-artist-header__stats">{{ stats }}</span>
|
||||
</div>
|
||||
<v-tabs
|
||||
:model-value="modelValue"
|
||||
@@ -34,7 +47,7 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, ref } from 'vue'
|
||||
import { formatLocalDate } from '../../utils/date.js'
|
||||
|
||||
const props = defineProps({
|
||||
@@ -45,7 +58,23 @@ const props = defineProps({
|
||||
modelValue: { type: String, required: true },
|
||||
})
|
||||
|
||||
defineEmits(['update:modelValue'])
|
||||
const emit = defineEmits(['update:modelValue', 'rename'])
|
||||
|
||||
// Inline rename (display name only — #130). Mirrors TagCard's pencil-edit.
|
||||
const editing = ref(false)
|
||||
const draft = ref('')
|
||||
function startEdit () {
|
||||
draft.value = props.name
|
||||
editing.value = true
|
||||
}
|
||||
function cancel () {
|
||||
editing.value = false
|
||||
}
|
||||
function submit () {
|
||||
const next = draft.value.trim()
|
||||
editing.value = false
|
||||
if (next && next !== props.name) emit('rename', next)
|
||||
}
|
||||
|
||||
const stats = computed(() => {
|
||||
const parts = []
|
||||
@@ -111,6 +140,20 @@ const stats = computed(() => {
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.fc-artist-header__edit {
|
||||
flex: 0 0 auto;
|
||||
opacity: 0;
|
||||
cursor: pointer;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
transition: opacity .15s ease;
|
||||
}
|
||||
.fc-artist-header__left:hover .fc-artist-header__edit { opacity: .7; }
|
||||
.fc-artist-header__edit:hover { opacity: 1; }
|
||||
.fc-artist-header__edit-field {
|
||||
max-width: 340px;
|
||||
font-family: 'Fraunces', Georgia, serif;
|
||||
}
|
||||
|
||||
.fc-artist-header__tabs {
|
||||
flex: 0 0 auto;
|
||||
}
|
||||
|
||||
@@ -47,18 +47,101 @@
|
||||
</div>
|
||||
<v-table density="compact">
|
||||
<thead>
|
||||
<tr><th>Platform</th><th>URL</th><th class="text-right">Images</th></tr>
|
||||
<tr>
|
||||
<th>Platform</th><th>URL</th>
|
||||
<th class="text-right">Images</th><th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr v-for="s in overview.sources" :key="s.id">
|
||||
<td>{{ s.platform }}</td>
|
||||
<td class="fc-artist-mgmt__url">{{ s.url }}</td>
|
||||
<td class="text-right">{{ s.image_count }}</td>
|
||||
<td class="text-right">
|
||||
<v-btn
|
||||
size="x-small" variant="text" prepend-icon="mdi-account-arrow-right"
|
||||
@click="openMove(s)"
|
||||
>Move…</v-btn>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
</section>
|
||||
|
||||
<!-- #130: move a source into a different (existing) artist. The slug is
|
||||
immutable so no files move — just re-attribution. -->
|
||||
<v-dialog v-model="moveOpen" max-width="480">
|
||||
<v-card>
|
||||
<v-card-title class="fc-h2">Move source to another artist</v-card-title>
|
||||
<v-card-text>
|
||||
<p class="mb-3 text-body-2">
|
||||
Moving <strong>{{ moveSource?.platform }}</strong> and the posts/images
|
||||
it brought in to another artist. Files aren't moved — only the
|
||||
attribution changes. If <strong>{{ overview.name }}</strong> is left
|
||||
empty it will be removed.
|
||||
</p>
|
||||
<v-autocomplete
|
||||
v-model="moveTarget"
|
||||
:items="artistItems"
|
||||
:loading="searching"
|
||||
item-title="name" item-value="id" return-object
|
||||
label="Target artist" density="compact" variant="outlined"
|
||||
no-filter hide-details autofocus clearable
|
||||
@update:search="onSearch"
|
||||
/>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="moveOpen = false">Cancel</v-btn>
|
||||
<v-btn
|
||||
color="accent" :disabled="!moveTarget || moving" :loading="moving"
|
||||
@click="confirmMove"
|
||||
>Move</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<section class="fc-artist-mgmt__sec">
|
||||
<h2 class="fc-h2">Translation</h2>
|
||||
<p class="fc-muted text-body-2 mb-3">
|
||||
Re-translate every post by {{ overview.name }} through the current
|
||||
Interpreter model — clears the stored translations and rebuilds them.
|
||||
Use this after switching translation models to refresh just this artist.
|
||||
</p>
|
||||
<div class="d-flex align-center flex-wrap" style="gap: 10px;">
|
||||
<v-btn
|
||||
size="small" variant="tonal" rounded="pill"
|
||||
prepend-icon="mdi-translate"
|
||||
:loading="retranslating" :disabled="!translationEnabled"
|
||||
@click="confirmRetranslate = true"
|
||||
>Re-translate posts</v-btn>
|
||||
<span v-if="!translationEnabled" class="fc-muted text-caption">
|
||||
Translation is off — enable it in Settings → Maintenance.
|
||||
</span>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- m146: per-artist re-translate — clears + rebuilds this artist's stored
|
||||
translations (used after a translation-model change). -->
|
||||
<v-dialog v-model="confirmRetranslate" max-width="440">
|
||||
<v-card>
|
||||
<v-card-title class="fc-h2">Re-translate {{ overview.name }}?</v-card-title>
|
||||
<v-card-text class="text-body-2">
|
||||
Clears the stored translation for every post by
|
||||
<strong>{{ overview.name }}</strong> and re-runs them through your
|
||||
current Interpreter model. Unchanged text comes back from the cache
|
||||
instantly. Runs in the background until done.
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="confirmRetranslate = false">Cancel</v-btn>
|
||||
<v-btn color="accent" :loading="retranslating" @click="onRetranslate">
|
||||
Re-translate
|
||||
</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
</v-dialog>
|
||||
|
||||
<section class="fc-artist-mgmt__sec">
|
||||
<h2 class="fc-h2">Danger zone</h2>
|
||||
<ArtistDangerZone
|
||||
@@ -71,9 +154,13 @@
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, onMounted, ref } from 'vue'
|
||||
import { useRouter, RouterLink } from 'vue-router'
|
||||
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import { useImportStore } from '../../stores/import.js'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
import { toast } from '../../utils/toast.js'
|
||||
import ArtistDangerZone from './ArtistDangerZone.vue'
|
||||
|
||||
const props = defineProps({
|
||||
@@ -81,6 +168,83 @@ const props = defineProps({
|
||||
})
|
||||
|
||||
const router = useRouter()
|
||||
const api = useApi()
|
||||
const importStore = useImportStore()
|
||||
const sources = useSourcesStore()
|
||||
|
||||
// m146: per-artist re-translate. Gated on translation being enabled globally
|
||||
// (the endpoint 400s otherwise) — read the shared import settings once.
|
||||
const translationEnabled = ref(false)
|
||||
const retranslating = ref(false)
|
||||
const confirmRetranslate = ref(false)
|
||||
|
||||
onMounted(async () => {
|
||||
try {
|
||||
await importStore.loadSettings()
|
||||
translationEnabled.value = !!importStore.settings?.translation_enabled
|
||||
} catch { /* non-fatal — the button just stays disabled */ }
|
||||
})
|
||||
|
||||
async function onRetranslate () {
|
||||
retranslating.value = true
|
||||
try {
|
||||
await api.post('/api/settings/translation/retranslate', {
|
||||
body: { artist_id: props.overview.id },
|
||||
})
|
||||
confirmRetranslate.value = false
|
||||
toast({ text: `Re-translating ${props.overview.name}…`, type: 'success' })
|
||||
} catch (e) {
|
||||
toast({ text: `Re-translate failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
retranslating.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// #130: move a source into another artist.
|
||||
const moveOpen = ref(false)
|
||||
const moveSource = ref(null)
|
||||
const moveTarget = ref(null)
|
||||
const artistItems = ref([])
|
||||
const searching = ref(false)
|
||||
const moving = ref(false)
|
||||
|
||||
function openMove (s) {
|
||||
moveSource.value = s
|
||||
moveTarget.value = null
|
||||
artistItems.value = []
|
||||
moveOpen.value = true
|
||||
}
|
||||
|
||||
let searchSeq = 0
|
||||
async function onSearch (q) {
|
||||
const mine = ++searchSeq
|
||||
if (!q || !q.trim()) { artistItems.value = []; return }
|
||||
searching.value = true
|
||||
try {
|
||||
const rows = await sources.autocompleteArtist(q)
|
||||
// Drop the current artist — you can't move a source onto itself.
|
||||
if (mine === searchSeq) artistItems.value = rows.filter(a => a.id !== props.overview.id)
|
||||
} finally {
|
||||
if (mine === searchSeq) searching.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function confirmMove () {
|
||||
if (!moveTarget.value || !moveSource.value) return
|
||||
moving.value = true
|
||||
try {
|
||||
await sources.reassign(moveSource.value.id, moveTarget.value.id)
|
||||
moveOpen.value = false
|
||||
toast({ text: `Moved to ${moveTarget.value.name}`, type: 'success' })
|
||||
// The current artist may now be empty (deleted); route to the target,
|
||||
// whose slug is stable (immutable).
|
||||
router.push({ name: 'artist', params: { slug: moveTarget.value.slug } })
|
||||
} catch (e) {
|
||||
toast({ text: `Move failed: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
moving.value = false
|
||||
}
|
||||
}
|
||||
|
||||
const sparkW = 600
|
||||
const sparkH = 80
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
filter, applied retroactively to the existing library.
|
||||
</p>
|
||||
|
||||
<v-row dense>
|
||||
<v-row density="compact">
|
||||
<v-col cols="6">
|
||||
<v-text-field
|
||||
v-model.number="minW" label="Min width (px)" type="number"
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
cadence as the transparency audit.
|
||||
</p>
|
||||
|
||||
<v-row dense>
|
||||
<v-row density="compact">
|
||||
<v-col cols="6">
|
||||
<v-text-field
|
||||
v-model.number="threshold" label="Threshold (0–1)"
|
||||
|
||||
@@ -14,10 +14,10 @@
|
||||
@update:search="onSearch"
|
||||
@update:model-value="onPick"
|
||||
>
|
||||
<template #item="{ props: itemProps, item }">
|
||||
<v-list-item v-bind="itemProps" :title="item.raw.name">
|
||||
<template #item="{ props: itemProps, internalItem }">
|
||||
<v-list-item v-bind="itemProps" :title="internalItem.raw.name">
|
||||
<template #subtitle>
|
||||
{{ item.raw.fandom_name ? `character · ${item.raw.fandom_name}` : item.raw.kind }}
|
||||
{{ internalItem.raw.fandom_name ? `character · ${internalItem.raw.fandom_name}` : internalItem.raw.kind }}
|
||||
</template>
|
||||
</v-list-item>
|
||||
</template>
|
||||
|
||||
@@ -13,9 +13,15 @@
|
||||
<div class="fc-tagcard__name">
|
||||
<template v-if="!editing">
|
||||
{{ card.name }}
|
||||
<v-icon
|
||||
v-if="card.is_system" size="14" icon="mdi-shield-outline"
|
||||
class="fc-tagcard__system"
|
||||
title="System tag — tagged items are excluded from training other concepts"
|
||||
/>
|
||||
<span v-if="card.kind === 'character' && card.fandom_name"
|
||||
class="fc-tagcard__fandom">— {{ card.fandom_name }}</span>
|
||||
<v-icon
|
||||
v-if="!card.is_system"
|
||||
class="fc-tagcard__edit" size="14" icon="mdi-pencil"
|
||||
@click.stop="startEdit"
|
||||
/>
|
||||
@@ -58,12 +64,18 @@
|
||||
prepend-icon="mdi-tag-multiple"
|
||||
@click="$emit('aliases', card)"
|
||||
/>
|
||||
<!-- System tags (#128): merge-away and delete are refused
|
||||
server-side — the hygiene machinery keys on the row — so
|
||||
don't offer them. Aliases stay (mapping model outputs ONTO
|
||||
a system tag is useful). -->
|
||||
<v-list-item
|
||||
v-if="!card.is_system"
|
||||
title="Merge with…"
|
||||
prepend-icon="mdi-call-merge"
|
||||
@click="$emit('merge-with', card)"
|
||||
/>
|
||||
<v-list-item
|
||||
v-if="!card.is_system"
|
||||
title="Delete tag"
|
||||
prepend-icon="mdi-delete"
|
||||
base-color="error"
|
||||
@@ -133,6 +145,7 @@ function submit() {
|
||||
.fc-tagcard__body { padding: 8px 12px; }
|
||||
.fc-tagcard__name { font-weight: 600; }
|
||||
.fc-tagcard__fandom { font-weight: 400; opacity: 0.7; }
|
||||
.fc-tagcard__system { opacity: 0.6; margin-left: 2px; }
|
||||
.fc-tagcard__meta {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
margin-top: 4px;
|
||||
|
||||
@@ -34,6 +34,15 @@
|
||||
:color="store.filter.no_artist ? 'accent' : undefined"
|
||||
@click="toggleFlag('no_artist')"
|
||||
>No artist<span class="fc-facets__count">{{ facetCount('no_artist') }}</span></v-chip>
|
||||
<!-- Reveal presentation chrome (banner / editor screenshot) the gallery
|
||||
hides by default (milestone 141). A browse-mode toggle, not a count. -->
|
||||
<v-chip
|
||||
size="small" label
|
||||
:variant="store.filter.include_hidden ? 'flat' : 'tonal'"
|
||||
:color="store.filter.include_hidden ? 'accent' : undefined"
|
||||
title="Show hidden — reveal banners / editor screenshots the gallery sets aside by default"
|
||||
@click="toggleFlag('include_hidden')"
|
||||
>Show hidden</v-chip>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
<template>
|
||||
<div class="fc-filterbar-wrap">
|
||||
<div class="fc-filterbar-wrap fc-chrome-continues">
|
||||
<div class="fc-filterbar">
|
||||
<v-autocomplete
|
||||
v-model="selected"
|
||||
@@ -13,14 +13,14 @@
|
||||
@update:search="onSearch"
|
||||
@update:model-value="onPick"
|
||||
>
|
||||
<template #item="{ props: itemProps, item }">
|
||||
<v-list-item v-bind="itemProps" :title="item.raw.name">
|
||||
<template #item="{ props: itemProps, internalItem }">
|
||||
<v-list-item v-bind="itemProps" :title="internalItem.raw.name">
|
||||
<template #prepend>
|
||||
<v-icon size="small">{{ iconFor(item.raw) }}</v-icon>
|
||||
<v-icon size="small">{{ iconFor(internalItem.raw) }}</v-icon>
|
||||
</template>
|
||||
<template #subtitle>
|
||||
{{ item.raw.kind === 'artist' ? 'artist'
|
||||
: (item.raw.fandom_name ? `character · ${item.raw.fandom_name}` : item.raw.kind) }}
|
||||
{{ internalItem.raw.kind === 'artist' ? 'artist'
|
||||
: (internalItem.raw.fandom_name ? `character · ${internalItem.raw.fandom_name}` : internalItem.raw.kind) }}
|
||||
</template>
|
||||
</v-list-item>
|
||||
</template>
|
||||
@@ -160,7 +160,7 @@ let debounce = null
|
||||
const refineCount = computed(() => {
|
||||
const f = store.filter
|
||||
return (f.platform ? 1 : 0) + (f.untagged ? 1 : 0) + (f.no_artist ? 1 : 0)
|
||||
+ (f.date_from ? 1 : 0) + (f.date_to ? 1 : 0)
|
||||
+ (f.include_hidden ? 1 : 0) + (f.date_from ? 1 : 0) + (f.date_to ? 1 : 0)
|
||||
})
|
||||
const hasRefineFilters = computed(() => refineCount.value > 0)
|
||||
|
||||
@@ -306,27 +306,17 @@ function pushFilter(mutate) {
|
||||
frosted block pinned directly under the 64px TopNav and continuous with it. */
|
||||
.fc-filterbar-wrap {
|
||||
position: sticky;
|
||||
top: 64px;
|
||||
top: var(--fc-nav-h, 64px); /* pins at the nav's real measured bottom (#1481) */
|
||||
z-index: 5;
|
||||
/* Attach to the TopNav: cancel the v-container's top padding (pt-2 = 8px)
|
||||
so the bar sits flush at 64px even at scroll 0 — without this it detaches
|
||||
and a gap shows through when scrolled to the top. */
|
||||
margin-top: -8px;
|
||||
margin-bottom: 12px;
|
||||
/* EXACT same gradiated obsidian (#14171A = 20,23,26) frost as the TopNav so
|
||||
the two read as one continuous piece of chrome — images scroll visibly
|
||||
under both. The nav's gradient fades to transparent at ITS bottom; this
|
||||
bar re-darkens at its top, so a faint seam (the page/image showing through
|
||||
the nav's transparent edge) separates them when scrolled to the very top,
|
||||
while under-scroll they frost as one. */
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
rgba(20, 23, 26, 0.92) 0%,
|
||||
rgba(20, 23, 26, 0.65) 60%,
|
||||
rgba(20, 23, 26, 0) 100%
|
||||
);
|
||||
backdrop-filter: blur(2px);
|
||||
-webkit-backdrop-filter: blur(2px);
|
||||
/* The frost itself (obsidian fade + blur) is the shared .fc-chrome-continues
|
||||
primitive: it CONTINUES the TopNav's fade from the seam alpha to transparent
|
||||
rather than re-darkening, so the nav + bar read as one gradient (operator
|
||||
2026-07-13). This block only owns the sticky positioning now. */
|
||||
}
|
||||
.fc-filterbar {
|
||||
display: flex;
|
||||
@@ -341,6 +331,20 @@ function pushFilter(mutate) {
|
||||
.fc-filterbar-wrap :deep(.v-btn-group) {
|
||||
background-color: rgba(20, 23, 26, 0.72);
|
||||
}
|
||||
/* Media toggle (All / Images / Videos) as ONE cohesive segmented control.
|
||||
FC's global VBtn { rounded: 'pill' } default made Vuetify 4 pill-round each
|
||||
SEGMENT individually, so the rounded ends collided at the joins — the shapes
|
||||
landed awkwardly on the button edges (operator 2026-07-13). Square the inner
|
||||
segments (over the pill utility's !important) and clip the group to a single
|
||||
8px outline (matches the chips/tiles rounding elsewhere in the app). Radius
|
||||
only — no height change, so the bar height and nav offset are untouched. */
|
||||
.fc-filterbar-wrap :deep(.v-btn-toggle) {
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.fc-filterbar-wrap :deep(.v-btn-toggle .v-btn) {
|
||||
border-radius: 0 !important;
|
||||
}
|
||||
.fc-filterbar__search { max-width: 320px; min-width: 200px; }
|
||||
.fc-filterbar__chips { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
|
||||
/* The tag chips' bodies toggle include/exclude — signal they're clickable. */
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
<template>
|
||||
<!-- System-tag auto-applies (chrome hides / process WIP tags) that ALSO looked
|
||||
like real content — surfaced PROACTIVELY atop the gallery whenever there's
|
||||
something to review (NOT gated on the Show-hidden toggle, so misfires can't
|
||||
go unnoticed), most-concerning first, with keep / remove (#141, #1464).
|
||||
Renders nothing when there's nothing to review. -->
|
||||
<section v-if="items.length" class="fc-review" aria-label="Auto-tagged images to review">
|
||||
<div class="fc-review__head">
|
||||
<v-icon size="18" color="warning">mdi-alert-outline</v-icon>
|
||||
<span class="fc-review__title">
|
||||
{{ items.length }} auto-tagged {{ items.length === 1 ? 'image' : 'images' }}
|
||||
may be real content — review
|
||||
</span>
|
||||
</div>
|
||||
<div class="fc-review__cards">
|
||||
<div
|
||||
v-for="it in items" :key="keyOf(it)" class="fc-review-card"
|
||||
>
|
||||
<img
|
||||
:src="it.thumbnail_url" :alt="it.tag_name"
|
||||
class="fc-review-card__thumb" loading="lazy"
|
||||
>
|
||||
<div class="fc-review-card__body">
|
||||
<div
|
||||
class="fc-review-card__conflict"
|
||||
:title="`Scored ${Math.round(it.conflict_score * 100)}% on “${it.conflict_name || 'a content tag'}”`"
|
||||
>
|
||||
also looks like <strong>{{ it.conflict_name || 'content' }}</strong>
|
||||
</div>
|
||||
<div class="fc-review-card__tag">{{ tagLine(it) }}</div>
|
||||
<div class="fc-review-card__acts">
|
||||
<button
|
||||
type="button" class="fc-review-btn fc-review-btn--keep"
|
||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')"
|
||||
>{{ keepLabel(it) }}</button>
|
||||
<button
|
||||
type="button" class="fc-review-btn fc-review-btn--unhide"
|
||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')"
|
||||
>{{ removeLabel(it) }}</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { onMounted, ref } from 'vue'
|
||||
import { useApi } from '../../composables/useApi.js'
|
||||
import { toast } from '../../utils/toast.js'
|
||||
|
||||
const api = useApi()
|
||||
|
||||
const items = ref([])
|
||||
const busy = ref([])
|
||||
|
||||
function keyOf(it) { return `${it.image_id}:${it.tag_id}` }
|
||||
// Chrome flags hide the image (keep-hidden / un-hide); process flags leave it
|
||||
// visible and just tagged (keep-tag / remove-tag). Same endpoints, different words.
|
||||
function tagLine(it) { return (it.mode === 'process' ? 'auto-tagged ' : 'hidden as ') + it.tag_name }
|
||||
function keepLabel(it) { return it.mode === 'process' ? 'Keep tag' : 'Keep hidden' }
|
||||
function removeLabel(it) { return it.mode === 'process' ? 'Remove tag' : 'Un-hide' }
|
||||
|
||||
async function load() {
|
||||
// Fetched unconditionally on mount — the strip prompts for pending misfires
|
||||
// even when the operator is browsing normally (Show-hidden off).
|
||||
try {
|
||||
const body = await api.get('/api/gallery/hidden-review')
|
||||
items.value = body.items || []
|
||||
} catch { items.value = [] }
|
||||
}
|
||||
|
||||
async function resolve(it, action) {
|
||||
const k = keyOf(it)
|
||||
busy.value = [...busy.value, k]
|
||||
try {
|
||||
await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`)
|
||||
items.value = items.value.filter((x) => keyOf(x) !== k)
|
||||
if (action === 'unhide') {
|
||||
const verb = it.mode === 'process' ? 'Removed' : 'Un-hidden'
|
||||
toast({ text: `${verb} — “${it.tag_name}” removed; it'll train the head`, type: 'success' })
|
||||
}
|
||||
} catch (e) {
|
||||
toast({
|
||||
text: `Could not ${action === 'keep' ? 'keep hidden' : 'un-hide'}: ${e.message}`,
|
||||
type: 'error',
|
||||
})
|
||||
} finally {
|
||||
busy.value = busy.value.filter((x) => x !== k)
|
||||
}
|
||||
}
|
||||
|
||||
onMounted(load)
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-review {
|
||||
margin-bottom: 14px;
|
||||
padding: 12px;
|
||||
border: 1px solid rgb(var(--v-theme-warning), 0.4);
|
||||
background: rgb(var(--v-theme-warning), 0.06);
|
||||
border-radius: 8px;
|
||||
}
|
||||
.fc-review__head {
|
||||
display: flex; align-items: center; gap: 8px; margin-bottom: 10px;
|
||||
}
|
||||
.fc-review__title {
|
||||
font-size: 14px; font-weight: 600;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
.fc-review__cards {
|
||||
display: flex; gap: 10px; overflow-x: auto; padding-bottom: 4px;
|
||||
}
|
||||
.fc-review-card {
|
||||
flex: 0 0 auto; width: 150px;
|
||||
display: flex; flex-direction: column;
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
border-radius: 6px; overflow: hidden;
|
||||
background: rgb(var(--v-theme-surface));
|
||||
}
|
||||
.fc-review-card__thumb {
|
||||
width: 100%; height: 96px; object-fit: cover; display: block;
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
}
|
||||
.fc-review-card__body { padding: 6px 8px; }
|
||||
.fc-review-card__conflict {
|
||||
font-size: 11px; color: rgb(var(--v-theme-on-surface));
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-review-card__conflict strong { color: rgb(var(--v-theme-warning)); }
|
||||
.fc-review-card__tag {
|
||||
font-size: 10px; color: rgb(var(--v-theme-on-surface-variant));
|
||||
margin: 1px 0 6px;
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-review-card__acts { display: flex; gap: 4px; }
|
||||
.fc-review-btn {
|
||||
flex: 1; font-size: 11px; padding: 3px 4px; border-radius: 4px;
|
||||
cursor: pointer; border: 1px solid transparent;
|
||||
}
|
||||
.fc-review-btn:disabled { opacity: 0.5; cursor: default; }
|
||||
.fc-review-btn--keep {
|
||||
background: transparent;
|
||||
color: rgb(var(--v-theme-on-surface-variant));
|
||||
border-color: rgb(var(--v-theme-on-surface-variant), 0.4);
|
||||
}
|
||||
.fc-review-btn--keep:hover:not(:disabled) {
|
||||
background: rgb(var(--v-theme-on-surface-variant), 0.1);
|
||||
}
|
||||
.fc-review-btn--unhide {
|
||||
background: rgb(var(--v-theme-accent)); color: #fff;
|
||||
}
|
||||
.fc-review-btn--unhide:hover:not(:disabled) { opacity: 0.9; }
|
||||
.fc-review-btn:focus-visible {
|
||||
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 1px;
|
||||
}
|
||||
</style>
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user