Compare commits
192 Commits
dev
..
ebac34e17b
| Author | SHA1 | Date | |
|---|---|---|---|
| ebac34e17b | |||
| 4c3ba20198 | |||
| d2acec61ae | |||
| 15ae5ef4fa | |||
| e3ceefc820 | |||
| e52090a4cf | |||
| bfba8045e4 | |||
| a708357436 | |||
| d9354ac1e1 | |||
| 1d48770793 | |||
| 489e6aaaee | |||
| ed20df905b | |||
| 5ba9871ef0 | |||
| 2a820d0848 | |||
| 2f9aa3d86c | |||
| 560a5000a2 | |||
| 7ddad231f8 | |||
| a78f7eaace | |||
| 71337b0ba4 | |||
| 3996205f3b | |||
| 9f9db01456 | |||
| 216b7fc743 | |||
| fbb76e6f36 | |||
| 5ef1478ade | |||
| 88ff4147e1 | |||
| 1ea02ad44c | |||
| 937cfb65b4 | |||
| baac851220 | |||
| a28c33281a | |||
| 40be0a9323 | |||
| 2c6bf26bfc | |||
| 3c1e76bd44 | |||
| 831c5b1c10 | |||
| 76b6af4903 | |||
| 92494ec4ed | |||
| bdfc17477c | |||
| 6cd3153bf4 | |||
| 5f2853168a | |||
| 9a979ee808 | |||
| 3138f912fd | |||
| 9df874e396 | |||
| 6915a7590a | |||
| 5e8c28236a | |||
| 7e3c0f0b74 | |||
| 5d0c7ba706 | |||
| 18300e1f8a | |||
| d52ac0a0e2 | |||
| 401fe8213e | |||
| e8774d7953 | |||
| bc0f00c51b | |||
| 1bef68aa29 | |||
| 1a4bc2f981 | |||
| 862ace69d6 | |||
| abf88b1a15 | |||
| c05dcafbea | |||
| 55e8632dab | |||
| 825e6b90bf | |||
| fb012c557c | |||
| 66593ab895 | |||
| b266a54ad3 | |||
| ad803b646f | |||
| 1f5da3d283 | |||
| 93034f580d | |||
| 9b9b12f410 | |||
| 376d310693 | |||
| bc69495a16 | |||
| 478f898e72 | |||
| 38a5e7f332 | |||
| 57fe15c267 | |||
| eb3231ef10 | |||
| e9af459c0d | |||
| 6f02806aec | |||
| a1d19bd96a | |||
| 26827ff38f | |||
| 26dcfaf6c2 | |||
| 9b1b0369cc | |||
| 18123fb9cb | |||
| 2e806f202f | |||
| 18d5c05639 | |||
| 11ddfc3876 | |||
| 2b8ce86622 | |||
| 49bee77cdc | |||
| c209e3b37e | |||
| cffdd93418 | |||
| fd84be40dd | |||
| 79f510d7f8 | |||
| 59181069da | |||
| 428ecd8642 | |||
| ed1e04b831 | |||
| f5156bd847 | |||
| dfc3922d24 | |||
| 3eb08e926b | |||
| 9e81ced359 | |||
| 11e9f5af60 | |||
| 909fa37b15 | |||
| dfab8f65ff | |||
| 618f7cdc36 | |||
| 028ea33a7c | |||
| 444c1fb075 | |||
| 26c68b0a75 | |||
| e75427b19a | |||
| 5447fab987 | |||
| bad37e07b2 | |||
| 2bfc9936a1 | |||
| 4c6406ee18 | |||
| bb47e80b3e | |||
| dc1083b5e0 | |||
| e46893fefd | |||
| 0666e15211 | |||
| 747390631d | |||
| e0d2a20588 | |||
| 1d84f67418 | |||
| 91265df3d6 | |||
| 11acdb0322 | |||
| 2eb9fd5dd0 | |||
| 01e5ce1410 | |||
| 3bb94674cf | |||
| a75c602175 | |||
| ef8f4f7193 | |||
| ec3d27b219 | |||
| 03bd3b2eda | |||
| 7395e77d75 | |||
| 575d817919 | |||
| 2a8f7cd8b6 | |||
| 83f8af8090 | |||
| 9a2617c1a2 | |||
| 81688815a0 | |||
| 773128c3bf | |||
| ce7b154ae9 | |||
| 9430a9d9c3 | |||
| 23aee56ce3 | |||
| 711abea567 | |||
| 844bb86802 | |||
| a8f6a464aa | |||
| ab9922ad2e | |||
| 0533807669 | |||
| 279dff3fb6 | |||
| 37e66cddc4 | |||
| 9cf6b2d363 | |||
| 6ef0fed41f | |||
| 89b48f8f35 | |||
| d60e0b9494 | |||
| 9c27a2d3c7 | |||
| 93e37681b7 | |||
| 64ca858574 | |||
| 9d0c0b7da8 | |||
| 8e4d252ae4 | |||
| fdd3e01f56 | |||
| c82fb308b6 | |||
| 8cf8d2ca4d | |||
| b1d58bc3b8 | |||
| 65386f02a0 | |||
| 667b05f14e | |||
| 856e9104b4 | |||
| 0397642b21 | |||
| 237575447d | |||
| ed358757dc | |||
| d181f4afb8 | |||
| 2886fa4997 | |||
| f256f587ee | |||
| 384d8d5e50 | |||
| 319e8c1d18 | |||
| 9075d8eadd | |||
| 88e53e5b86 | |||
| 37e8b796a1 | |||
| 4e82208926 | |||
| 52fff00353 | |||
| c14338cbce | |||
| 8c36dd28b0 | |||
| 88cfb3dd02 | |||
| 5d4f223b71 | |||
| 05090c6e85 | |||
| 3a577d5ade | |||
| f4fe02e346 | |||
| e766197d99 | |||
| 3872e1dda9 | |||
| 9814f3dbaf | |||
| b214460fdb | |||
| ac55d0e8d8 | |||
| 89a89e0ded | |||
| 4e9aac2c05 | |||
| 2879ac6f2b | |||
| b8dce6c483 | |||
| d1c0b82a22 | |||
| 5526b8dc78 | |||
| 16eb7075c4 | |||
| 885dcf64f3 | |||
| f2f6b6d25e | |||
| 0822240fde | |||
| 27f7f3fd01 | |||
| c5bf564f53 | |||
| 602c7d275d |
@@ -9,15 +9,10 @@ name: CI
|
|||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches: [dev, main]
|
branches: [dev, main]
|
||||||
# Renovate opens PRs from `renovate/*` branches into `dev`. Those branches
|
# pull_request trigger intentionally absent — with branches: [dev, main]
|
||||||
# never push to dev/main, so the push trigger above gives them NO pre-merge
|
# above, every PR commit already fires CI via the push event on dev. Adding
|
||||||
# CI — a bump could only be validated after it was already merged. This
|
# pull_request would duplicate runs on dev→main PRs. FC has no fork PRs
|
||||||
# pull_request trigger (base `dev` only) validates Renovate PRs before merge.
|
# (single-operator Forgejo repo) so push coverage is complete.
|
||||||
# It deliberately does NOT fire on dev→main PRs (base `main`), which still
|
|
||||||
# rely on the dev push run — so no duplicate runs. FC has no fork PRs
|
|
||||||
# (single-operator Forgejo repo), so secrets-on-PR is not a concern.
|
|
||||||
pull_request:
|
|
||||||
branches: [dev]
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
|
# Fast-fail lint lane. ruff is pre-installed in the ci-python image, so
|
||||||
@@ -97,10 +92,10 @@ jobs:
|
|||||||
# If we want strict lockfile-based reproducibility later, commit a
|
# If we want strict lockfile-based reproducibility later, commit a
|
||||||
# package-lock.json and flip this back to `npm ci`.
|
# package-lock.json and flip this back to `npm ci`.
|
||||||
- run: npm install --no-audit --no-fund
|
- run: npm install --no-audit --no-fund
|
||||||
# No type-check step: the frontend is pure JS (no .ts files, no JSDoc),
|
# `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS
|
||||||
# so a type-checker has nothing to do. The vue-tsc devDep + its `check`
|
# with no .ts files and no JSDoc annotations, so vue-tsc has nothing
|
||||||
# script were dropped 2026-07-11 rather than bumped to v3. If we add
|
# to type-check. Re-enable once we add a tsconfig.json and either
|
||||||
# TS/JSDoc later, re-add a tsconfig.json + vue-tsc + a type-check step.
|
# convert to TS or add JSDoc.
|
||||||
- run: npm run test:unit
|
- run: npm run test:unit
|
||||||
- run: npm run build
|
- run: npm run build
|
||||||
|
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ jobs:
|
|||||||
lint:
|
lint:
|
||||||
runs-on: python-ci
|
runs-on: python-ci
|
||||||
container:
|
container:
|
||||||
image: node:24-bookworm-slim
|
image: node:22-bookworm-slim
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- name: Install web-ext
|
- name: Install web-ext
|
||||||
|
|||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
# syntax=docker/dockerfile:1.25
|
# syntax=docker/dockerfile:1.7
|
||||||
|
|
||||||
FROM node:24-alpine AS frontend-builder
|
FROM node:22-alpine AS frontend-builder
|
||||||
WORKDIR /build
|
WORKDIR /build
|
||||||
COPY frontend/package.json frontend/package-lock.json* ./
|
COPY frontend/package.json frontend/package-lock.json* ./
|
||||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
# syntax=docker/dockerfile:1.25
|
# syntax=docker/dockerfile:1.7
|
||||||
|
|
||||||
FROM python:3.14-slim
|
FROM python:3.14-slim
|
||||||
ENV PYTHONUNBUFFERED=1 \
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
|||||||
+10
-51
@@ -265,8 +265,6 @@ class Worker:
|
|||||||
self._embedder = None
|
self._embedder = None
|
||||||
self._embedder_lock = threading.Lock()
|
self._embedder_lock = threading.Lock()
|
||||||
self._proposers = None
|
self._proposers = None
|
||||||
self._proposers_sig = None # detector-config signature the current
|
|
||||||
# proposers were built for (#134)
|
|
||||||
self._proposers_lock = threading.Lock()
|
self._proposers_lock = threading.Lock()
|
||||||
|
|
||||||
# --- held-lease bookkeeping --------------------------------------------
|
# --- held-lease bookkeeping --------------------------------------------
|
||||||
@@ -798,52 +796,14 @@ class Worker:
|
|||||||
self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype)
|
self._embedder = CropEmbedder(model_name, self.cfg.embed_dtype)
|
||||||
return self._embedder
|
return self._embedder
|
||||||
|
|
||||||
def _effective_cfg(self, det: dict | None):
|
def _ensure_proposers(self):
|
||||||
"""The detector config for a job: the LEASE-ANNOUNCED block (the DB/UI
|
if self._proposers is not None:
|
||||||
source of truth, #134) overlaid on the env cfg, or the env cfg unchanged
|
|
||||||
when the server announced nothing (bootstrap / older backend). An enabled
|
|
||||||
toggle off maps to empty weights — the 'weights == "" → proposer skipped'
|
|
||||||
contract in detectors.py. detector_level + ccip_model stay env (separate
|
|
||||||
settings, not part of the crop-proposer config)."""
|
|
||||||
if not det:
|
|
||||||
return self.cfg
|
|
||||||
|
|
||||||
def _w(p: str) -> str:
|
|
||||||
slot = det.get(p) or {}
|
|
||||||
return (slot.get("weights") or "") if slot.get("enabled", True) else ""
|
|
||||||
|
|
||||||
from dataclasses import replace
|
|
||||||
return replace(
|
|
||||||
self.cfg,
|
|
||||||
person_weights=_w("person"),
|
|
||||||
person_conf=float(det["person"]["conf"]),
|
|
||||||
anatomy_weights=_w("anatomy"),
|
|
||||||
anatomy_conf=float(det["anatomy"]["conf"]),
|
|
||||||
panel_weights=_w("panel"),
|
|
||||||
panel_conf=float(det["panel"]["conf"]),
|
|
||||||
max_figures=int(det["max_figures"]),
|
|
||||||
max_components=int(det["max_components"]),
|
|
||||||
max_panels=int(det["max_panels"]),
|
|
||||||
max_regions=int(det["max_regions"]),
|
|
||||||
dedupe_iou=float(det["dedupe_iou"]),
|
|
||||||
)
|
|
||||||
|
|
||||||
def _proposers_for(self, eff):
|
|
||||||
"""Proposers built from the effective detector config, rebuilt ONLY when
|
|
||||||
that config changes — so an operator's UI edit takes effect on the next
|
|
||||||
lease with no restart, and steady-state reuses the loaded YOLO models."""
|
|
||||||
sig = (
|
|
||||||
eff.person_weights, eff.person_conf,
|
|
||||||
eff.anatomy_weights, eff.anatomy_conf,
|
|
||||||
eff.panel_weights, eff.panel_conf,
|
|
||||||
eff.max_figures, eff.max_components, eff.max_panels,
|
|
||||||
)
|
|
||||||
with self._proposers_lock:
|
|
||||||
if self._proposers is None or self._proposers_sig != sig:
|
|
||||||
from .detectors import Proposers
|
|
||||||
self._proposers = Proposers(eff)
|
|
||||||
self._proposers_sig = sig
|
|
||||||
return self._proposers
|
return self._proposers
|
||||||
|
with self._proposers_lock:
|
||||||
|
if self._proposers is None:
|
||||||
|
from .detectors import Proposers
|
||||||
|
self._proposers = Proposers(self.cfg)
|
||||||
|
return self._proposers
|
||||||
|
|
||||||
def _consume(self, job: dict, frames: list, stop_evt: threading.Event) -> bool:
|
def _consume(self, job: dict, frames: list, stop_evt: threading.Event) -> bool:
|
||||||
"""Detect + embed the decoded frames and submit the result. Returns True
|
"""Detect + embed the decoded frames and submit the result. Returns True
|
||||||
@@ -896,8 +856,7 @@ class Worker:
|
|||||||
else ["figure", "face", "concept", "panel"]
|
else ["figure", "face", "concept", "panel"]
|
||||||
)
|
)
|
||||||
embedder = self._ensure_embedder(model_name) if want_siglip else None
|
embedder = self._ensure_embedder(model_name) if want_siglip else None
|
||||||
eff = self._effective_cfg(job.get("detectors"))
|
proposers = self._ensure_proposers()
|
||||||
proposers = self._proposers_for(eff)
|
|
||||||
|
|
||||||
regions = []
|
regions = []
|
||||||
ccip_ev = self.cfg.ccip_model or "ccip-default"
|
ccip_ev = self.cfg.ccip_model or "ccip-default"
|
||||||
@@ -963,7 +922,7 @@ class Worker:
|
|||||||
# we never SigLIP the same region twice — saves GPU and a
|
# we never SigLIP the same region twice — saves GPU and a
|
||||||
# slot against max_regions. High-IoU + kind-aware, so
|
# slot against max_regions. High-IoU + kind-aware, so
|
||||||
# intentional nested crops (figure ⊃ head) survive.
|
# intentional nested crops (figure ⊃ head) survive.
|
||||||
pending = dedupe_crops(pending, eff.dedupe_iou)
|
pending = dedupe_crops(pending, self.cfg.dedupe_iou)
|
||||||
vecs = embedder.embed_batch([c for c, _ in pending])
|
vecs = embedder.embed_batch([c for c, _ in pending])
|
||||||
for (_c, tmpl), vec in zip(pending, vecs, strict=True):
|
for (_c, tmpl), vec in zip(pending, vecs, strict=True):
|
||||||
tmpl["siglip_embedding"] = vec
|
tmpl["siglip_embedding"] = vec
|
||||||
@@ -973,7 +932,7 @@ class Worker:
|
|||||||
# mp4, 64 sampled frames × ~32 regions) would otherwise burn
|
# mp4, 64 sampled frames × ~32 regions) would otherwise burn
|
||||||
# ~38s of GPU across every frame before the submit is even
|
# ~38s of GPU across every frame before the submit is even
|
||||||
# truncated. Bounds the WORK, not just the POST body.
|
# truncated. Bounds the WORK, not just the POST body.
|
||||||
if len(regions) >= eff.max_regions:
|
if len(regions) >= self.cfg.max_regions:
|
||||||
break
|
break
|
||||||
|
|
||||||
# A Stop mid-frame-loop leaves partial regions — don't submit those;
|
# A Stop mid-frame-loop leaves partial regions — don't submit those;
|
||||||
|
|||||||
@@ -1,83 +0,0 @@
|
|||||||
"""ml_settings crop-proposer / detector config (#134)
|
|
||||||
|
|
||||||
Move the WHERE-to-crop detector config (per-proposer enable + weights + conf,
|
|
||||||
plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU
|
|
||||||
agent in the lease (like the embedder model) — no restart, agent env is now
|
|
||||||
bootstrap-only. All server_defaults are the working values so existing rows +
|
|
||||||
fresh installs crop out-of-the-box with all three proposers ON.
|
|
||||||
|
|
||||||
Revision ID: 0078
|
|
||||||
Revises: 0077
|
|
||||||
Create Date: 2026-07-05
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0078"
|
|
||||||
down_revision: Union[str, None] = "0077"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
_ANATOMY_DEFAULT = (
|
|
||||||
"https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt"
|
|
||||||
)
|
|
||||||
_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt"
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_person_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.true()))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_person_weights", sa.String(512), nullable=False,
|
|
||||||
server_default="yolo11n.pt"))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_person_conf", sa.Float(), nullable=False,
|
|
||||||
server_default=sa.text("0.35")))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_anatomy_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.true()))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_anatomy_weights", sa.String(512), nullable=False,
|
|
||||||
server_default=_ANATOMY_DEFAULT))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_anatomy_conf", sa.Float(), nullable=False,
|
|
||||||
server_default=sa.text("0.30")))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_panel_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.true()))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_panel_weights", sa.String(512), nullable=False,
|
|
||||||
server_default=_PANEL_DEFAULT))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_panel_conf", sa.Float(), nullable=False,
|
|
||||||
server_default=sa.text("0.30")))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_max_figures", sa.Integer(), nullable=False,
|
|
||||||
server_default=sa.text("8")))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_max_components", sa.Integer(), nullable=False,
|
|
||||||
server_default=sa.text("8")))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_max_panels", sa.Integer(), nullable=False,
|
|
||||||
server_default=sa.text("8")))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_max_regions", sa.Integer(), nullable=False,
|
|
||||||
server_default=sa.text("128")))
|
|
||||||
op.add_column("ml_settings", sa.Column(
|
|
||||||
"detector_dedupe_iou", sa.Float(), nullable=False,
|
|
||||||
server_default=sa.text("0.85")))
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
for col in (
|
|
||||||
"detector_person_enabled", "detector_person_weights", "detector_person_conf",
|
|
||||||
"detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf",
|
|
||||||
"detector_panel_enabled", "detector_panel_weights", "detector_panel_conf",
|
|
||||||
"detector_max_figures", "detector_max_components", "detector_max_panels",
|
|
||||||
"detector_max_regions", "detector_dedupe_iou",
|
|
||||||
):
|
|
||||||
op.drop_column("ml_settings", col)
|
|
||||||
@@ -1,77 +0,0 @@
|
|||||||
"""character prototype store (#1317) — precomputed, incremental CCIP references
|
|
||||||
|
|
||||||
New tables character_prototype + ccip_prototype_state, plus MLSettings columns
|
|
||||||
ccip_ref_signature (cheap global change gate) + ccip_prototype_cap (per-character
|
|
||||||
reference cap). The reference set the CCIP matcher uses becomes a precomputed
|
|
||||||
artifact refreshed incrementally off the request path. See milestone 138 /
|
|
||||||
backend.app.services.ml.character_prototypes.
|
|
||||||
|
|
||||||
Revision ID: 0079
|
|
||||||
Revises: 0078
|
|
||||||
Create Date: 2026-07-06
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
from pgvector.sqlalchemy import Vector
|
|
||||||
|
|
||||||
revision: str = "0079"
|
|
||||||
down_revision: Union[str, None] = "0078"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
# Matches models.image_region.CCIP_DIM (the CCIP figure-embedding width).
|
|
||||||
_CCIP_DIM = 768
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.create_table(
|
|
||||||
"character_prototype",
|
|
||||||
sa.Column("id", sa.Integer(), primary_key=True),
|
|
||||||
sa.Column(
|
|
||||||
"tag_id", sa.Integer(),
|
|
||||||
sa.ForeignKey("tag.id", ondelete="CASCADE"), nullable=False,
|
|
||||||
),
|
|
||||||
sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"region_id", sa.Integer(),
|
|
||||||
sa.ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.create_index(
|
|
||||||
"ix_character_prototype_tag_id", "character_prototype", ["tag_id"]
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"ccip_prototype_state",
|
|
||||||
sa.Column(
|
|
||||||
"tag_id", sa.Integer(),
|
|
||||||
sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
|
|
||||||
),
|
|
||||||
sa.Column("fingerprint", sa.String(64), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
|
||||||
server_default=sa.func.now(),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column("ccip_ref_signature", sa.String(128), nullable=True),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column(
|
|
||||||
"ccip_prototype_cap", sa.Integer(), nullable=False,
|
|
||||||
server_default=sa.text("64"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("ml_settings", "ccip_prototype_cap")
|
|
||||||
op.drop_column("ml_settings", "ccip_ref_signature")
|
|
||||||
op.drop_table("ccip_prototype_state")
|
|
||||||
op.drop_index(
|
|
||||||
"ix_character_prototype_tag_id", table_name="character_prototype"
|
|
||||||
)
|
|
||||||
op.drop_table("character_prototype")
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
"""tag_head.train_fingerprint (#1317 phase 2) — incremental head retraining
|
|
||||||
|
|
||||||
A per-head training-data fingerprint (positive + rejection count/latest-timestamp)
|
|
||||||
so a manual Retrain refits only the tags whose data changed; the nightly run
|
|
||||||
ignores it (full reconcile). Nullable — a NULL fingerprint (existing heads) forces
|
|
||||||
a refit on the first incremental run, then it's stamped.
|
|
||||||
|
|
||||||
Revision ID: 0080
|
|
||||||
Revises: 0079
|
|
||||||
Create Date: 2026-07-06
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0080"
|
|
||||||
down_revision: Union[str, None] = "0079"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"tag_head",
|
|
||||||
sa.Column("train_fingerprint", sa.String(128), nullable=True),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("tag_head", "train_fingerprint")
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
"""stricter auto-apply defaults (milestone 139) — cut auto-apply misfires
|
|
||||||
|
|
||||||
head_auto_apply_min_positives 30→50 and ccip_auto_apply_threshold 0.92→0.95
|
|
||||||
(operator-asked 2026-07-06). The head graduation precision bar stays 0.97 — the
|
|
||||||
operator confirmed the general-tag confidence was already well tuned; only the
|
|
||||||
support floor + the CCIP match confidence are raised. The model defaults change
|
|
||||||
for fresh installs; here we bump the existing singleton row IFF it is still at
|
|
||||||
the previous default, so a deliberate operator change is NOT clobbered.
|
|
||||||
|
|
||||||
Revision ID: 0081
|
|
||||||
Revises: 0080
|
|
||||||
Create Date: 2026-07-06
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0081"
|
|
||||||
down_revision: Union[str, None] = "0080"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.execute(
|
|
||||||
"UPDATE ml_settings SET head_auto_apply_min_positives = 50 "
|
|
||||||
"WHERE head_auto_apply_min_positives = 30"
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"UPDATE ml_settings SET ccip_auto_apply_threshold = 0.95 "
|
|
||||||
"WHERE ccip_auto_apply_threshold = 0.92"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.execute(
|
|
||||||
"UPDATE ml_settings SET head_auto_apply_min_positives = 30 "
|
|
||||||
"WHERE head_auto_apply_min_positives = 50"
|
|
||||||
)
|
|
||||||
op.execute(
|
|
||||||
"UPDATE ml_settings SET ccip_auto_apply_threshold = 0.92 "
|
|
||||||
"WHERE ccip_auto_apply_threshold = 0.95"
|
|
||||||
)
|
|
||||||
@@ -1,85 +0,0 @@
|
|||||||
"""presentation-chrome auto-hide (#141) — settings knobs + review table
|
|
||||||
|
|
||||||
MLSettings gains presentation_auto_apply_enabled / _threshold and
|
|
||||||
presentation_conflict_threshold: banner + editor-screenshot auto-hide on the
|
|
||||||
sweep with a FLAT threshold (decoupled from content-head graduation), and a
|
|
||||||
conflict threshold that flags an auto-hide that "also looks like content".
|
|
||||||
|
|
||||||
New table presentation_review records an auto-hidden chrome image that also
|
|
||||||
scored high on a content head, surfaced in the Hidden view for a keep-hidden /
|
|
||||||
un-hide decision. Resolved rows are pruned by retention.
|
|
||||||
|
|
||||||
Revision ID: 0082
|
|
||||||
Revises: 0081
|
|
||||||
Create Date: 2026-07-07
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0082"
|
|
||||||
down_revision: Union[str, None] = "0081"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column(
|
|
||||||
"presentation_auto_apply_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.text("true"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column(
|
|
||||||
"presentation_auto_apply_threshold", sa.Float(), nullable=False,
|
|
||||||
server_default=sa.text("0.90"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column(
|
|
||||||
"presentation_conflict_threshold", sa.Float(), nullable=False,
|
|
||||||
server_default=sa.text("0.50"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.create_table(
|
|
||||||
"presentation_review",
|
|
||||||
sa.Column(
|
|
||||||
"image_record_id", sa.Integer(),
|
|
||||||
sa.ForeignKey("image_record.id", ondelete="CASCADE"),
|
|
||||||
primary_key=True,
|
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"tag_id", sa.Integer(),
|
|
||||||
sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True,
|
|
||||||
),
|
|
||||||
sa.Column(
|
|
||||||
"conflict_tag_id", sa.Integer(),
|
|
||||||
sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True,
|
|
||||||
),
|
|
||||||
sa.Column("conflict_score", sa.Float(), nullable=False),
|
|
||||||
sa.Column(
|
|
||||||
"created_at", sa.DateTime(timezone=True), nullable=False,
|
|
||||||
server_default=sa.func.now(),
|
|
||||||
),
|
|
||||||
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
|
||||||
)
|
|
||||||
# The review list queries the unresolved flags (resolved_at IS NULL).
|
|
||||||
op.create_index(
|
|
||||||
"ix_presentation_review_resolved_at", "presentation_review",
|
|
||||||
["resolved_at"],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_index(
|
|
||||||
"ix_presentation_review_resolved_at", table_name="presentation_review"
|
|
||||||
)
|
|
||||||
op.drop_table("presentation_review")
|
|
||||||
op.drop_column("ml_settings", "presentation_conflict_threshold")
|
|
||||||
op.drop_column("ml_settings", "presentation_auto_apply_threshold")
|
|
||||||
op.drop_column("ml_settings", "presentation_auto_apply_enabled")
|
|
||||||
@@ -1,73 +0,0 @@
|
|||||||
"""post-text translation via Interpreter (milestone 143) — Post columns + settings
|
|
||||||
|
|
||||||
Post gains the translated title/description + the detected source language,
|
|
||||||
Interpreter engine_version (cache key), and translated_at — filled by the
|
|
||||||
translate sweep. ImportSettings gains translation_enabled (OFF by default),
|
|
||||||
interpreter_base_url (EMPTY — the operator sets their own, behind a reverse
|
|
||||||
proxy), and translation_target_lang (en).
|
|
||||||
|
|
||||||
Revision ID: 0083
|
|
||||||
Revises: 0082
|
|
||||||
Create Date: 2026-07-07
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0083"
|
|
||||||
down_revision: Union[str, None] = "0082"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"post", sa.Column("post_title_translated", sa.Text(), nullable=True)
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"post", sa.Column("description_translated", sa.Text(), nullable=True)
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"post",
|
|
||||||
sa.Column("translated_source_lang", sa.String(8), nullable=True),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"post",
|
|
||||||
sa.Column("translation_engine_version", sa.String(128), nullable=True),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"post",
|
|
||||||
sa.Column("translated_at", sa.DateTime(timezone=True), nullable=True),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"import_settings",
|
|
||||||
sa.Column(
|
|
||||||
"translation_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.text("false"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"import_settings",
|
|
||||||
sa.Column(
|
|
||||||
"interpreter_base_url", sa.Text(), nullable=False, server_default="",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"import_settings",
|
|
||||||
sa.Column(
|
|
||||||
"translation_target_lang", sa.Text(), nullable=False,
|
|
||||||
server_default="en",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("import_settings", "translation_target_lang")
|
|
||||||
op.drop_column("import_settings", "interpreter_base_url")
|
|
||||||
op.drop_column("import_settings", "translation_enabled")
|
|
||||||
op.drop_column("post", "translated_at")
|
|
||||||
op.drop_column("post", "translation_engine_version")
|
|
||||||
op.drop_column("post", "translated_source_lang")
|
|
||||||
op.drop_column("post", "description_translated")
|
|
||||||
op.drop_column("post", "post_title_translated")
|
|
||||||
@@ -1,51 +0,0 @@
|
|||||||
"""translation strictness setting + per-post translation override (milestone 155)
|
|
||||||
|
|
||||||
ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance
|
|
||||||
floor, now operator-tunable in the UI; default 0.9 — stricter than the old
|
|
||||||
hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at
|
|
||||||
~0.86). Post gains ``translation_override`` — a sticky per-post choice of
|
|
||||||
auto / force / original so the operator can force a skipped translation on, or
|
|
||||||
knock a wrongly-translated one back to the original, and have it survive a
|
|
||||||
Re-translate-all.
|
|
||||||
|
|
||||||
Revision ID: 0084
|
|
||||||
Revises: 0083
|
|
||||||
Create Date: 2026-07-10
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0084"
|
|
||||||
down_revision: Union[str, None] = "0083"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"import_settings",
|
|
||||||
sa.Column(
|
|
||||||
"translation_min_confidence", sa.Float(), nullable=False,
|
|
||||||
server_default=sa.text("0.9"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"post",
|
|
||||||
sa.Column(
|
|
||||||
"translation_override", sa.String(16), nullable=False,
|
|
||||||
server_default="auto",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.create_check_constraint(
|
|
||||||
"ck_post_translation_override",
|
|
||||||
"post",
|
|
||||||
"translation_override IN ('auto', 'force', 'original')",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_constraint("ck_post_translation_override", "post", type_="check")
|
|
||||||
op.drop_column("post", "translation_override")
|
|
||||||
op.drop_column("import_settings", "translation_min_confidence")
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
"""title-based WIP auto-tagging (task #1458) — ImportSettings toggle
|
|
||||||
|
|
||||||
ImportSettings gains wip_title_tagging_enabled (ON by default): when a freshly
|
|
||||||
imported post's title explicitly declares work-in-progress ("WIP" / "work in
|
|
||||||
progress"), the importer applies the `wip` system tag to its images. No new
|
|
||||||
table — the tag itself is the seeded `wip` system tag (migration 0075) and the
|
|
||||||
application reuses image_tag with source='wip_title'.
|
|
||||||
|
|
||||||
Revision ID: 0085
|
|
||||||
Revises: 0084
|
|
||||||
Create Date: 2026-07-12
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0085"
|
|
||||||
down_revision: Union[str, None] = "0084"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"import_settings",
|
|
||||||
sa.Column(
|
|
||||||
"wip_title_tagging_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.text("true"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("import_settings", "wip_title_tagging_enabled")
|
|
||||||
@@ -1,61 +0,0 @@
|
|||||||
"""process auto-apply settings + review mode (#1464) — system-tag refactor
|
|
||||||
|
|
||||||
The system-tag behavior refactor gives `wip` / `editor screenshot` (the PROCESS
|
|
||||||
group) their own provisional auto-apply, parallel to the presentation (chrome)
|
|
||||||
sweep. MLSettings gains three knobs: enabled (OFF by default — a new whole-library
|
|
||||||
auto-tagger is opt-in), the flat apply threshold, and the ring-loud conflict
|
|
||||||
threshold. presentation_review gains a `mode` column so one review surface serves
|
|
||||||
both chrome and process flags (existing rows backfill 'chrome'). server_defaults
|
|
||||||
so the existing rows fill cleanly.
|
|
||||||
|
|
||||||
Revision ID: 0086
|
|
||||||
Revises: 0085
|
|
||||||
Create Date: 2026-07-13
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0086"
|
|
||||||
down_revision: Union[str, None] = "0085"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column(
|
|
||||||
"process_auto_apply_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.text("false"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column(
|
|
||||||
"process_auto_apply_threshold", sa.Float(), nullable=False,
|
|
||||||
server_default="0.90",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"ml_settings",
|
|
||||||
sa.Column(
|
|
||||||
"process_conflict_threshold", sa.Float(), nullable=False,
|
|
||||||
server_default="0.50",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
op.add_column(
|
|
||||||
"presentation_review",
|
|
||||||
sa.Column(
|
|
||||||
"mode", sa.String(16), nullable=False,
|
|
||||||
server_default="chrome",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("presentation_review", "mode")
|
|
||||||
op.drop_column("ml_settings", "process_conflict_threshold")
|
|
||||||
op.drop_column("ml_settings", "process_auto_apply_threshold")
|
|
||||||
op.drop_column("ml_settings", "process_auto_apply_enabled")
|
|
||||||
@@ -1,33 +0,0 @@
|
|||||||
"""soft WIP title tier toggle (#1474) — ImportSettings.wip_soft_title_tagging_enabled
|
|
||||||
|
|
||||||
The soft tier also tags sketch/doodle/scribble titles, but with a provisional source
|
|
||||||
that never trains the head. OFF by default (a lower-precision tier is opt-in).
|
|
||||||
server_default so the existing singleton row (id=1) fills cleanly.
|
|
||||||
|
|
||||||
Revision ID: 0087
|
|
||||||
Revises: 0086
|
|
||||||
Create Date: 2026-07-13
|
|
||||||
"""
|
|
||||||
from typing import Sequence, Union
|
|
||||||
|
|
||||||
import sqlalchemy as sa
|
|
||||||
from alembic import op
|
|
||||||
|
|
||||||
revision: str = "0087"
|
|
||||||
down_revision: Union[str, None] = "0086"
|
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
|
||||||
op.add_column(
|
|
||||||
"import_settings",
|
|
||||||
sa.Column(
|
|
||||||
"wip_soft_title_tagging_enabled", sa.Boolean(), nullable=False,
|
|
||||||
server_default=sa.text("false"),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
|
||||||
op.drop_column("import_settings", "wip_soft_title_tagging_enabled")
|
|
||||||
+3
-131
@@ -3,18 +3,9 @@
|
|||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
from sqlalchemy import delete, select, update
|
|
||||||
from sqlalchemy.orm import aliased
|
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import (
|
from ..services.gallery_service import GalleryService
|
||||||
ImageRecord,
|
|
||||||
PresentationReview,
|
|
||||||
Tag,
|
|
||||||
TagSuggestionRejection,
|
|
||||||
)
|
|
||||||
from ..models.tag import image_tag
|
|
||||||
from ..services.gallery_service import GalleryService, image_url, thumbnail_url
|
|
||||||
|
|
||||||
gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
|
gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
|
||||||
|
|
||||||
@@ -86,9 +77,6 @@ def _parse_filters():
|
|||||||
platform = request.args.get("platform") or None
|
platform = request.args.get("platform") or None
|
||||||
untagged = request.args.get("untagged") in ("1", "true", "yes")
|
untagged = request.args.get("untagged") in ("1", "true", "yes")
|
||||||
no_artist = request.args.get("no_artist") in ("1", "true", "yes")
|
no_artist = request.args.get("no_artist") in ("1", "true", "yes")
|
||||||
# Show the presentation chrome (banner / editor screenshot) that the default
|
|
||||||
# gallery hides — the Hidden view sets this (milestone 141).
|
|
||||||
include_hidden = request.args.get("include_hidden") in ("1", "true", "yes")
|
|
||||||
date_from = _parse_date(request.args.get("date_from"))
|
date_from = _parse_date(request.args.get("date_from"))
|
||||||
date_to = _parse_date(request.args.get("date_to"))
|
date_to = _parse_date(request.args.get("date_to"))
|
||||||
if date_to is not None:
|
if date_to is not None:
|
||||||
@@ -100,7 +88,6 @@ def _parse_filters():
|
|||||||
"platform": platform,
|
"platform": platform,
|
||||||
"untagged": untagged, "no_artist": no_artist,
|
"untagged": untagged, "no_artist": no_artist,
|
||||||
"date_from": date_from, "date_to": date_to,
|
"date_from": date_from, "date_to": date_to,
|
||||||
"include_hidden": include_hidden,
|
|
||||||
}
|
}
|
||||||
return filters, sort
|
return filters, sort
|
||||||
|
|
||||||
@@ -145,32 +132,12 @@ async def similar():
|
|||||||
filters, _sort = _parse_filters()
|
filters, _sort = _parse_filters()
|
||||||
except (KeyError, ValueError):
|
except (KeyError, ValueError):
|
||||||
return jsonify({"error": "similar_to query param required"}), 400
|
return jsonify({"error": "similar_to query param required"}), 400
|
||||||
# Explore passes exclude_wip=1 to also drop work-in-progress from the
|
|
||||||
# rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274).
|
|
||||||
exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True")
|
|
||||||
# Explore reach (#1476): 0 = nearest (gallery default), →1 reaches into farther
|
|
||||||
# distance bands so the walk can escape a dense cluster. exclude_ids = the
|
|
||||||
# breadcrumb, so already-walked images aren't re-served as neighbours.
|
|
||||||
try:
|
|
||||||
reach = max(0.0, min(1.0, float(request.args.get("reach", "0"))))
|
|
||||||
except ValueError:
|
|
||||||
reach = 0.0
|
|
||||||
exclude_ids = [
|
|
||||||
int(x) for x in request.args.get("exclude_ids", "").split(",")
|
|
||||||
if x.strip().isdigit()
|
|
||||||
] or None
|
|
||||||
# post_id is the exclusive post-detail view — not a similarity scope.
|
# post_id is the exclusive post-detail view — not a similarity scope.
|
||||||
# include_hidden is a gallery-browse flag; similar() has its OWN presentation
|
scope = {k: v for k, v in filters.items() if k != "post_id"}
|
||||||
# exclusion (a similarity-quality concern, #1274), so drop it here (#141).
|
|
||||||
scope = {
|
|
||||||
k: v for k, v in filters.items() if k not in ("post_id", "include_hidden")
|
|
||||||
}
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
svc = GalleryService(session)
|
svc = GalleryService(session)
|
||||||
try:
|
try:
|
||||||
images = await svc.similar(
|
images = await svc.similar(image_id=similar_to, limit=limit, **scope)
|
||||||
image_id=similar_to, limit=limit, exclude_wip=exclude_wip,
|
|
||||||
reach=reach, exclude_ids=exclude_ids, **scope)
|
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return jsonify({"error": str(exc)}), 400
|
return jsonify({"error": str(exc)}), 400
|
||||||
if images is None:
|
if images is None:
|
||||||
@@ -244,101 +211,6 @@ async def jump():
|
|||||||
return jsonify({"cursor": cursor})
|
return jsonify({"cursor": cursor})
|
||||||
|
|
||||||
|
|
||||||
# -- Hidden-view review (#141): auto-hidden chrome flagged "also looks like
|
|
||||||
# content", surfaced in the gallery's Show-hidden review strip. -----------
|
|
||||||
@gallery_bp.route("/hidden-review", methods=["GET"])
|
|
||||||
async def hidden_review():
|
|
||||||
"""Unresolved system-tag auto-apply review flags (chrome + process, #1464),
|
|
||||||
most-concerning first (highest content score) — for the review strip. `mode`
|
|
||||||
tells the client whether the flagged tag hid the image ('chrome') or left it
|
|
||||||
visible ('process'), which decides the resolve labels (un-hide vs remove-tag)."""
|
|
||||||
ptag = aliased(Tag)
|
|
||||||
ctag = aliased(Tag)
|
|
||||||
async with get_session() as session:
|
|
||||||
rows = (await session.execute(
|
|
||||||
select(
|
|
||||||
PresentationReview.image_record_id,
|
|
||||||
PresentationReview.tag_id,
|
|
||||||
PresentationReview.conflict_tag_id,
|
|
||||||
PresentationReview.conflict_score,
|
|
||||||
PresentationReview.mode,
|
|
||||||
ImageRecord.path, ImageRecord.thumbnail_path,
|
|
||||||
ImageRecord.sha256, ImageRecord.mime,
|
|
||||||
ptag.name.label("tag_name"),
|
|
||||||
ctag.name.label("conflict_name"),
|
|
||||||
)
|
|
||||||
.join(ImageRecord, ImageRecord.id == PresentationReview.image_record_id)
|
|
||||||
.join(ptag, ptag.id == PresentationReview.tag_id)
|
|
||||||
.outerjoin(ctag, ctag.id == PresentationReview.conflict_tag_id)
|
|
||||||
.where(PresentationReview.resolved_at.is_(None))
|
|
||||||
.order_by(PresentationReview.conflict_score.desc())
|
|
||||||
)).all()
|
|
||||||
return jsonify({"items": [
|
|
||||||
{
|
|
||||||
"image_id": r.image_record_id,
|
|
||||||
"tag_id": r.tag_id,
|
|
||||||
"tag_name": r.tag_name,
|
|
||||||
"conflict_tag_id": r.conflict_tag_id,
|
|
||||||
"conflict_name": r.conflict_name,
|
|
||||||
"conflict_score": r.conflict_score,
|
|
||||||
"mode": r.mode,
|
|
||||||
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
|
|
||||||
"image_url": image_url(r.path),
|
|
||||||
}
|
|
||||||
for r in rows
|
|
||||||
]})
|
|
||||||
|
|
||||||
|
|
||||||
@gallery_bp.route(
|
|
||||||
"/hidden-review/<int:image_id>/<int:tag_id>/keep", methods=["POST"]
|
|
||||||
)
|
|
||||||
async def hidden_review_keep(image_id, tag_id):
|
|
||||||
"""Keep the auto-hide: resolve the flag; the tag stays applied (#141)."""
|
|
||||||
async with get_session() as session:
|
|
||||||
await session.execute(
|
|
||||||
update(PresentationReview)
|
|
||||||
.where(
|
|
||||||
PresentationReview.image_record_id == image_id,
|
|
||||||
PresentationReview.tag_id == tag_id,
|
|
||||||
)
|
|
||||||
.values(resolved_at=datetime.now(UTC))
|
|
||||||
)
|
|
||||||
await session.commit()
|
|
||||||
return "", 204
|
|
||||||
|
|
||||||
|
|
||||||
@gallery_bp.route(
|
|
||||||
"/hidden-review/<int:image_id>/<int:tag_id>/unhide", methods=["POST"]
|
|
||||||
)
|
|
||||||
async def hidden_review_unhide(image_id, tag_id):
|
|
||||||
"""Un-hide: remove the presentation tag (image returns to the gallery), record
|
|
||||||
a rejection so the head LEARNS it misfired, and resolve the flag (#141)."""
|
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
||||||
|
|
||||||
async with get_session() as session:
|
|
||||||
await session.execute(
|
|
||||||
delete(image_tag).where(
|
|
||||||
image_tag.c.image_record_id == image_id,
|
|
||||||
image_tag.c.tag_id == tag_id,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
await session.execute(
|
|
||||||
pg_insert(TagSuggestionRejection)
|
|
||||||
.values(image_record_id=image_id, tag_id=tag_id)
|
|
||||||
.on_conflict_do_nothing()
|
|
||||||
)
|
|
||||||
await session.execute(
|
|
||||||
update(PresentationReview)
|
|
||||||
.where(
|
|
||||||
PresentationReview.image_record_id == image_id,
|
|
||||||
PresentationReview.tag_id == tag_id,
|
|
||||||
)
|
|
||||||
.values(resolved_at=datetime.now(UTC))
|
|
||||||
)
|
|
||||||
await session.commit()
|
|
||||||
return "", 204
|
|
||||||
|
|
||||||
|
|
||||||
@gallery_bp.route("/image/<int:image_id>", methods=["GET"])
|
@gallery_bp.route("/image/<int:image_id>", methods=["GET"])
|
||||||
async def image_detail(image_id: int):
|
async def image_detail(image_id: int):
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
|
|||||||
+3
-29
@@ -256,7 +256,9 @@ async def lease():
|
|||||||
if not await _agent_authed(session):
|
if not await _agent_authed(session):
|
||||||
return jsonify({"error": "unauthorized"}), 401
|
return jsonify({"error": "unauthorized"}), 401
|
||||||
jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
|
jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
|
||||||
ml = await MLSettings.load(session)
|
ml = (
|
||||||
|
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||||
|
).scalar_one()
|
||||||
# image rows for url/mime in one shot
|
# image rows for url/mime in one shot
|
||||||
ids = [j.image_record_id for j in jobs]
|
ids = [j.image_record_id for j in jobs]
|
||||||
imgs = {
|
imgs = {
|
||||||
@@ -267,33 +269,6 @@ async def lease():
|
|||||||
).scalars()
|
).scalars()
|
||||||
} if ids else {}
|
} if ids else {}
|
||||||
await session.commit()
|
await session.commit()
|
||||||
# Crop-proposer config, announced FROM THE SETTING like embed_model_name
|
|
||||||
# (#134): the agent builds its detectors from this, rebuilding live when
|
|
||||||
# it changes — so tuning is a DB/UI edit, never an agent restart. Same
|
|
||||||
# block for every job in the batch (it's global), built once. An enabled
|
|
||||||
# toggle off is carried through so the agent skips that proposer.
|
|
||||||
detectors = {
|
|
||||||
"person": {
|
|
||||||
"enabled": ml.detector_person_enabled,
|
|
||||||
"weights": ml.detector_person_weights,
|
|
||||||
"conf": ml.detector_person_conf,
|
|
||||||
},
|
|
||||||
"anatomy": {
|
|
||||||
"enabled": ml.detector_anatomy_enabled,
|
|
||||||
"weights": ml.detector_anatomy_weights,
|
|
||||||
"conf": ml.detector_anatomy_conf,
|
|
||||||
},
|
|
||||||
"panel": {
|
|
||||||
"enabled": ml.detector_panel_enabled,
|
|
||||||
"weights": ml.detector_panel_weights,
|
|
||||||
"conf": ml.detector_panel_conf,
|
|
||||||
},
|
|
||||||
"max_figures": ml.detector_max_figures,
|
|
||||||
"max_components": ml.detector_max_components,
|
|
||||||
"max_panels": ml.detector_max_panels,
|
|
||||||
"max_regions": ml.detector_max_regions,
|
|
||||||
"dedupe_iou": ml.detector_dedupe_iou,
|
|
||||||
}
|
|
||||||
out = []
|
out = []
|
||||||
for j in jobs:
|
for j in jobs:
|
||||||
img = imgs.get(j.image_record_id)
|
img = imgs.get(j.image_record_id)
|
||||||
@@ -315,7 +290,6 @@ async def lease():
|
|||||||
# re-embed, never an agent change.
|
# re-embed, never an agent change.
|
||||||
"embed_model_name": ml.embedder_model_name,
|
"embed_model_name": ml.embedder_model_name,
|
||||||
"embed_version": ml.embedder_model_version,
|
"embed_version": ml.embedder_model_version,
|
||||||
"detectors": detectors,
|
|
||||||
})
|
})
|
||||||
return jsonify({"jobs": out})
|
return jsonify({"jobs": out})
|
||||||
|
|
||||||
|
|||||||
+32
-65
@@ -4,31 +4,10 @@ from quart import Blueprint, jsonify, request
|
|||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import MLSettings
|
from ..models import MLSettings
|
||||||
from ..services.ml.heads import AUTO_APPLY_THRESHOLD_MAX, AUTO_APPLY_THRESHOLD_MIN
|
|
||||||
|
|
||||||
ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
||||||
|
|
||||||
|
|
||||||
# Crop-proposer / detector config (#134). Announced to the GPU agent in the lease
|
|
||||||
# → tunable here with no restart. weights = ultralytics name | URL | hf_repo::file
|
|
||||||
# (empty, or enabled off, skips that proposer).
|
|
||||||
_DETECTOR_FIELDS = (
|
|
||||||
"detector_person_enabled",
|
|
||||||
"detector_person_weights",
|
|
||||||
"detector_person_conf",
|
|
||||||
"detector_anatomy_enabled",
|
|
||||||
"detector_anatomy_weights",
|
|
||||||
"detector_anatomy_conf",
|
|
||||||
"detector_panel_enabled",
|
|
||||||
"detector_panel_weights",
|
|
||||||
"detector_panel_conf",
|
|
||||||
"detector_max_figures",
|
|
||||||
"detector_max_components",
|
|
||||||
"detector_max_panels",
|
|
||||||
"detector_max_regions",
|
|
||||||
"detector_dedupe_iou",
|
|
||||||
)
|
|
||||||
|
|
||||||
_EDITABLE = (
|
_EDITABLE = (
|
||||||
"cpu_embed_enabled",
|
"cpu_embed_enabled",
|
||||||
"video_frame_interval_seconds",
|
"video_frame_interval_seconds",
|
||||||
@@ -40,15 +19,8 @@ _EDITABLE = (
|
|||||||
"ccip_match_threshold",
|
"ccip_match_threshold",
|
||||||
"ccip_auto_apply_enabled",
|
"ccip_auto_apply_enabled",
|
||||||
"ccip_auto_apply_threshold",
|
"ccip_auto_apply_threshold",
|
||||||
"presentation_auto_apply_enabled",
|
|
||||||
"presentation_auto_apply_threshold",
|
|
||||||
"presentation_conflict_threshold",
|
|
||||||
"process_auto_apply_enabled",
|
|
||||||
"process_auto_apply_threshold",
|
|
||||||
"process_conflict_threshold",
|
|
||||||
"embedder_model_name",
|
"embedder_model_name",
|
||||||
"embedder_model_version",
|
"embedder_model_version",
|
||||||
*_DETECTOR_FIELDS,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -84,21 +56,41 @@ async def embedder_models():
|
|||||||
|
|
||||||
@ml_admin_bp.route("/settings", methods=["GET"])
|
@ml_admin_bp.route("/settings", methods=["GET"])
|
||||||
async def get_settings():
|
async def get_settings():
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
s = await MLSettings.load(session)
|
s = (
|
||||||
# Table-driven off _EDITABLE (which PATCH also writes) so a new settings field
|
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||||
# can never be silently absent from GET — the split that historically dropped
|
).scalar_one()
|
||||||
# fields. _EDITABLE already includes *_DETECTOR_FIELDS.
|
return jsonify(
|
||||||
return jsonify({f: getattr(s, f) for f in _EDITABLE})
|
{
|
||||||
|
"cpu_embed_enabled": s.cpu_embed_enabled,
|
||||||
|
"video_frame_interval_seconds": s.video_frame_interval_seconds,
|
||||||
|
"video_max_frames": s.video_max_frames,
|
||||||
|
"embedder_model_version": s.embedder_model_version,
|
||||||
|
"head_min_positives": s.head_min_positives,
|
||||||
|
"head_auto_apply_precision": s.head_auto_apply_precision,
|
||||||
|
"head_auto_apply_enabled": s.head_auto_apply_enabled,
|
||||||
|
"head_auto_apply_min_positives": s.head_auto_apply_min_positives,
|
||||||
|
"ccip_match_threshold": s.ccip_match_threshold,
|
||||||
|
"ccip_auto_apply_enabled": s.ccip_auto_apply_enabled,
|
||||||
|
"ccip_auto_apply_threshold": s.ccip_auto_apply_threshold,
|
||||||
|
"embedder_model_name": s.embedder_model_name,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@ml_admin_bp.route("/settings", methods=["PATCH"])
|
@ml_admin_bp.route("/settings", methods=["PATCH"])
|
||||||
async def patch_settings():
|
async def patch_settings():
|
||||||
|
from sqlalchemy import select
|
||||||
|
|
||||||
body = await request.get_json()
|
body = await request.get_json()
|
||||||
if not isinstance(body, dict):
|
if not isinstance(body, dict):
|
||||||
return jsonify({"error": "body must be an object"}), 400
|
return jsonify({"error": "body must be an object"}), 400
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
s = await MLSettings.load(session)
|
s = (
|
||||||
|
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
# Merge the patch over current values, then validate the result as a
|
# Merge the patch over current values, then validate the result as a
|
||||||
# whole — the store-floor invariant couples three fields, so they
|
# whole — the store-floor invariant couples three fields, so they
|
||||||
@@ -128,44 +120,19 @@ def _validate(p: dict) -> str | None:
|
|||||||
# Head training (#114).
|
# Head training (#114).
|
||||||
if int(p["head_min_positives"]) < 1:
|
if int(p["head_min_positives"]) < 1:
|
||||||
return "head_min_positives must be >= 1"
|
return "head_min_positives must be >= 1"
|
||||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["head_auto_apply_precision"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
if not (0.5 <= float(p["head_auto_apply_precision"]) <= 0.999):
|
||||||
return f"head_auto_apply_precision must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
return "head_auto_apply_precision must be between 0.5 and 0.999"
|
||||||
if int(p["head_auto_apply_min_positives"]) < 1:
|
if int(p["head_auto_apply_min_positives"]) < 1:
|
||||||
return "head_auto_apply_min_positives must be >= 1"
|
return "head_auto_apply_min_positives must be >= 1"
|
||||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["ccip_match_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
if not (0.5 <= float(p["ccip_match_threshold"]) <= 0.999):
|
||||||
return f"ccip_match_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
return "ccip_match_threshold must be between 0.5 and 0.999"
|
||||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["ccip_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
if not (0.5 <= float(p["ccip_auto_apply_threshold"]) <= 0.999):
|
||||||
return f"ccip_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
return "ccip_auto_apply_threshold must be between 0.5 and 0.999"
|
||||||
# Presentation chrome auto-hide (#141). Auto-apply runs high (hiding is
|
|
||||||
# consequential); the conflict cut is a plain probability [0,1].
|
|
||||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["presentation_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
|
||||||
return f"presentation_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
|
||||||
if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0):
|
|
||||||
return "presentation_conflict_threshold must be between 0 and 1"
|
|
||||||
# Process auto-apply (#1464). wip/editor stay VISIBLE so a false apply is
|
|
||||||
# low-harm (excludes-from-training + a review flag), but keep the same bar.
|
|
||||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["process_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
|
||||||
return f"process_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
|
||||||
if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
|
|
||||||
return "process_conflict_threshold must be between 0 and 1"
|
|
||||||
# Embedder model swap (#1190): both must be non-empty. Changing them means a
|
# Embedder model swap (#1190): both must be non-empty. Changing them means a
|
||||||
# different embedding space — the operator must re-embed + retrain after.
|
# different embedding space — the operator must re-embed + retrain after.
|
||||||
for key in ("embedder_model_name", "embedder_model_version"):
|
for key in ("embedder_model_name", "embedder_model_version"):
|
||||||
if not str(p[key]).strip():
|
if not str(p[key]).strip():
|
||||||
return f"{key} must not be empty"
|
return f"{key} must not be empty"
|
||||||
# Crop proposers (#134). Weights may be empty (that proposer is just off);
|
|
||||||
# confidences are probabilities; caps are positive counts; IoU is [0,1].
|
|
||||||
for key in ("detector_person_conf", "detector_anatomy_conf", "detector_panel_conf"):
|
|
||||||
if not (0.0 <= float(p[key]) <= 1.0):
|
|
||||||
return f"{key} must be between 0 and 1"
|
|
||||||
for key in (
|
|
||||||
"detector_max_figures", "detector_max_components",
|
|
||||||
"detector_max_panels", "detector_max_regions",
|
|
||||||
):
|
|
||||||
if int(p[key]) < 1:
|
|
||||||
return f"{key} must be >= 1"
|
|
||||||
if not (0.0 <= float(p["detector_dedupe_iou"]) <= 1.0):
|
|
||||||
return "detector_dedupe_iou must be between 0 and 1"
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -3,28 +3,12 @@
|
|||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import ImportSettings, Post
|
|
||||||
from ..services import interpreter_client as ic
|
|
||||||
from ..services.post_feed_service import PostFeedService
|
from ..services.post_feed_service import PostFeedService
|
||||||
from ..services.source_service import KNOWN_PLATFORMS
|
from ..services.source_service import KNOWN_PLATFORMS
|
||||||
from ..utils.text import html_to_plain
|
|
||||||
from ._responses import error_response as _bad
|
from ._responses import error_response as _bad
|
||||||
|
|
||||||
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
||||||
|
|
||||||
_TRANSLATION_OVERRIDES = ("auto", "force", "original")
|
|
||||||
|
|
||||||
|
|
||||||
def _queue_for_sweep(post: Post) -> None:
|
|
||||||
"""Mark a post untranslated (all translation columns NULL) so the periodic
|
|
||||||
sweep re-runs it under its new override — used when Interpreter is down and we
|
|
||||||
can't translate inline."""
|
|
||||||
post.post_title_translated = None
|
|
||||||
post.description_translated = None
|
|
||||||
post.translated_source_lang = None
|
|
||||||
post.translation_engine_version = None
|
|
||||||
post.translated_at = None
|
|
||||||
|
|
||||||
|
|
||||||
@posts_bp.route("", methods=["GET"])
|
@posts_bp.route("", methods=["GET"])
|
||||||
async def list_posts():
|
async def list_posts():
|
||||||
@@ -98,70 +82,3 @@ async def get_post(post_id: int):
|
|||||||
if item is None:
|
if item is None:
|
||||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||||
return jsonify(item)
|
return jsonify(item)
|
||||||
|
|
||||||
|
|
||||||
@posts_bp.route("/<int:post_id>/translation-override", methods=["POST"])
|
|
||||||
async def set_translation_override(post_id: int):
|
|
||||||
"""Sticky per-post translation override (milestone 155). Body:
|
|
||||||
``{"override": "auto" | "force" | "original"}``.
|
|
||||||
|
|
||||||
'original' keeps the original (clears any stored translation now — no
|
|
||||||
Interpreter needed). 'force'/'auto' translate the post immediately if the
|
|
||||||
service is up (force bypasses the acceptance floor; auto re-runs the gate);
|
|
||||||
if it's down we save the flag and mark the post untranslated so the next sweep
|
|
||||||
applies it. The override persists, so the sweep + Re-translate-all keep
|
|
||||||
honoring it. Returns the updated translation fields + an ``applied`` status."""
|
|
||||||
body = await request.get_json(silent=True) or {}
|
|
||||||
override = body.get("override")
|
|
||||||
if override not in _TRANSLATION_OVERRIDES:
|
|
||||||
return _bad(
|
|
||||||
"invalid_override",
|
|
||||||
detail=f"override must be one of {list(_TRANSLATION_OVERRIDES)}",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Lazy import (mirrors settings.py) so the API module doesn't pull the celery
|
|
||||||
# task graph at import time.
|
|
||||||
from ..tasks.translation import _store_translation, _translate_field
|
|
||||||
|
|
||||||
async with get_session() as session:
|
|
||||||
post = await session.get(Post, post_id)
|
|
||||||
if post is None:
|
|
||||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
|
||||||
post.translation_override = override
|
|
||||||
cfg = await ImportSettings.load(session)
|
|
||||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
|
||||||
|
|
||||||
if override == "original":
|
|
||||||
_store_translation(post, (None, None, None), (None, None, None), target)
|
|
||||||
applied = "cleared"
|
|
||||||
else:
|
|
||||||
base_url = (cfg.interpreter_base_url or "").strip()
|
|
||||||
if cfg.translation_enabled and base_url and ic.health(base_url):
|
|
||||||
force = override == "force"
|
|
||||||
title = (post.post_title or "").strip()
|
|
||||||
desc = (html_to_plain(post.description) if post.description else "") or ""
|
|
||||||
desc = desc.strip()
|
|
||||||
mc = cfg.translation_min_confidence
|
|
||||||
try:
|
|
||||||
title_res = _translate_field(title, base_url, target, mc, force=force)
|
|
||||||
desc_res = _translate_field(desc, base_url, target, mc, force=force)
|
|
||||||
except ic.InterpreterUnavailable:
|
|
||||||
_queue_for_sweep(post)
|
|
||||||
applied = "queued"
|
|
||||||
else:
|
|
||||||
_store_translation(post, title_res, desc_res, target)
|
|
||||||
applied = "translated"
|
|
||||||
else:
|
|
||||||
# Disabled / no URL / unhealthy → let the sweep apply it later.
|
|
||||||
_queue_for_sweep(post)
|
|
||||||
applied = "queued"
|
|
||||||
|
|
||||||
await session.commit()
|
|
||||||
return jsonify({
|
|
||||||
"id": post.id,
|
|
||||||
"translation_override": post.translation_override,
|
|
||||||
"post_title_translated": post.post_title_translated,
|
|
||||||
"description_translated": post.description_translated,
|
|
||||||
"translated_source_lang": post.translated_source_lang,
|
|
||||||
"applied": applied,
|
|
||||||
})
|
|
||||||
|
|||||||
+24
-220
@@ -1,24 +1,12 @@
|
|||||||
"""Settings API: import filters, system stats."""
|
"""Settings API: import filters, system stats."""
|
||||||
|
|
||||||
import asyncio
|
|
||||||
import secrets
|
import secrets
|
||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
from sqlalchemy import func, or_, select
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import (
|
from ..models import AppSetting, Artist, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
|
||||||
AppSetting,
|
|
||||||
Artist,
|
|
||||||
ImageRecord,
|
|
||||||
ImportBatch,
|
|
||||||
ImportSettings,
|
|
||||||
ImportTask,
|
|
||||||
Post,
|
|
||||||
Tag,
|
|
||||||
TaskRun,
|
|
||||||
)
|
|
||||||
from ..services import interpreter_client as ic
|
|
||||||
|
|
||||||
settings_bp = Blueprint("settings", __name__, url_prefix="/api")
|
settings_bp = Blueprint("settings", __name__, url_prefix="/api")
|
||||||
|
|
||||||
@@ -44,12 +32,6 @@ _EDITABLE_FIELDS = (
|
|||||||
"extdl_mediafire_enabled",
|
"extdl_mediafire_enabled",
|
||||||
"extdl_dropbox_enabled",
|
"extdl_dropbox_enabled",
|
||||||
"extdl_pixeldrain_enabled",
|
"extdl_pixeldrain_enabled",
|
||||||
"translation_enabled",
|
|
||||||
"interpreter_base_url",
|
|
||||||
"translation_target_lang",
|
|
||||||
"translation_min_confidence",
|
|
||||||
"wip_title_tagging_enabled",
|
|
||||||
"wip_soft_title_tagging_enabled",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
||||||
@@ -66,9 +48,28 @@ _EXTDL_TOGGLE_FIELDS = (
|
|||||||
async def get_import_settings():
|
async def get_import_settings():
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = await ImportSettings.load(session)
|
row = await ImportSettings.load(session)
|
||||||
# Table-driven off _EDITABLE_FIELDS (which PATCH also writes) so a new field
|
return jsonify({
|
||||||
# can't be silently absent from GET.
|
"min_width": row.min_width,
|
||||||
return jsonify({f: getattr(row, f) for f in _EDITABLE_FIELDS})
|
"min_height": row.min_height,
|
||||||
|
"skip_transparent": row.skip_transparent,
|
||||||
|
"transparency_threshold": row.transparency_threshold,
|
||||||
|
"skip_single_color": row.skip_single_color,
|
||||||
|
"single_color_threshold": row.single_color_threshold,
|
||||||
|
"single_color_tolerance": row.single_color_tolerance,
|
||||||
|
"phash_threshold": row.phash_threshold,
|
||||||
|
"download_rate_limit_seconds": row.download_rate_limit_seconds,
|
||||||
|
"download_validate_files": row.download_validate_files,
|
||||||
|
"download_schedule_default_seconds": row.download_schedule_default_seconds,
|
||||||
|
"download_event_retention_days": row.download_event_retention_days,
|
||||||
|
"download_failure_warning_threshold": row.download_failure_warning_threshold,
|
||||||
|
"series_suggest_enabled": row.series_suggest_enabled,
|
||||||
|
"series_suggest_threshold": row.series_suggest_threshold,
|
||||||
|
"extdl_mega_enabled": row.extdl_mega_enabled,
|
||||||
|
"extdl_gdrive_enabled": row.extdl_gdrive_enabled,
|
||||||
|
"extdl_mediafire_enabled": row.extdl_mediafire_enabled,
|
||||||
|
"extdl_dropbox_enabled": row.extdl_dropbox_enabled,
|
||||||
|
"extdl_pixeldrain_enabled": row.extdl_pixeldrain_enabled,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/settings/import", methods=["PATCH"])
|
@settings_bp.route("/settings/import", methods=["PATCH"])
|
||||||
@@ -127,41 +128,12 @@ async def update_import_settings():
|
|||||||
for tog in _EXTDL_TOGGLE_FIELDS:
|
for tog in _EXTDL_TOGGLE_FIELDS:
|
||||||
if tog in body and not isinstance(body[tog], bool):
|
if tog in body and not isinstance(body[tog], bool):
|
||||||
return jsonify({"error": f"{tog} must be a boolean"}), 400
|
return jsonify({"error": f"{tog} must be a boolean"}), 400
|
||||||
# Translation (#143): base URL may be empty (feature off until set — no
|
|
||||||
# default host; the operator points it at their own Interpreter proxy).
|
|
||||||
if "translation_enabled" in body and not isinstance(
|
|
||||||
body["translation_enabled"], bool
|
|
||||||
):
|
|
||||||
return jsonify({"error": "translation_enabled must be a boolean"}), 400
|
|
||||||
for key in ("interpreter_base_url", "translation_target_lang"):
|
|
||||||
if key in body and not isinstance(body[key], str):
|
|
||||||
return jsonify({"error": f"{key} must be a string"}), 400
|
|
||||||
# Acceptance floor (milestone 155): latin-script translations below this
|
|
||||||
# Interpreter confidence are kept as the original.
|
|
||||||
if "translation_min_confidence" in body:
|
|
||||||
v = body["translation_min_confidence"]
|
|
||||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
|
||||||
return jsonify(
|
|
||||||
{"error": "translation_min_confidence must be a number in [0, 1]"}
|
|
||||||
), 400
|
|
||||||
if "series_suggest_threshold" in body:
|
if "series_suggest_threshold" in body:
|
||||||
v = body["series_suggest_threshold"]
|
v = body["series_suggest_threshold"]
|
||||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||||
return jsonify(
|
return jsonify(
|
||||||
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
||||||
), 400
|
), 400
|
||||||
if "wip_title_tagging_enabled" in body and not isinstance(
|
|
||||||
body["wip_title_tagging_enabled"], bool
|
|
||||||
):
|
|
||||||
return jsonify(
|
|
||||||
{"error": "wip_title_tagging_enabled must be a boolean"}
|
|
||||||
), 400
|
|
||||||
if "wip_soft_title_tagging_enabled" in body and not isinstance(
|
|
||||||
body["wip_soft_title_tagging_enabled"], bool
|
|
||||||
):
|
|
||||||
return jsonify(
|
|
||||||
{"error": "wip_soft_title_tagging_enabled must be a boolean"}
|
|
||||||
), 400
|
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = await ImportSettings.load(session)
|
row = await ImportSettings.load(session)
|
||||||
@@ -173,18 +145,6 @@ async def update_import_settings():
|
|||||||
return await get_import_settings()
|
return await get_import_settings()
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/settings/wip-title/scan", methods=["POST"])
|
|
||||||
async def wip_title_scan():
|
|
||||||
"""Enqueue the back-catalogue WIP-title scan (task #1458 Settings button):
|
|
||||||
apply the `wip` system tag to EXISTING posts whose title declares
|
|
||||||
work-in-progress. New imports are tagged live by the importer; this catches
|
|
||||||
the existing library. Returns the Celery task id (202)."""
|
|
||||||
from ..tasks.maintenance import backfill_wip_title_tags
|
|
||||||
|
|
||||||
r = backfill_wip_title_tags.delay()
|
|
||||||
return jsonify({"celery_task_id": r.id}), 202
|
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/system/stats", methods=["GET"])
|
@settings_bp.route("/system/stats", methods=["GET"])
|
||||||
async def system_stats():
|
async def system_stats():
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
@@ -310,159 +270,3 @@ async def rotate_extension_api_key():
|
|||||||
row.value = new_value
|
row.value = new_value
|
||||||
await session.commit()
|
await session.commit()
|
||||||
return jsonify({"key": new_value})
|
return jsonify({"key": new_value})
|
||||||
|
|
||||||
|
|
||||||
# --- Translation (#143): live status + manual "Translate now" --------------
|
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/settings/translation/status", methods=["GET"])
|
|
||||||
async def translation_status():
|
|
||||||
"""For the Settings card: is it on, is a URL set, is the service reachable,
|
|
||||||
and how many posts still await translation. Health runs the sync client in a
|
|
||||||
thread so the event loop isn't blocked."""
|
|
||||||
translation_tasks = (
|
|
||||||
"backend.app.tasks.translation.translate_posts",
|
|
||||||
"backend.app.tasks.translation.retranslate_posts",
|
|
||||||
)
|
|
||||||
async with get_session() as session:
|
|
||||||
cfg = await ImportSettings.load(session)
|
|
||||||
untranslated = (await session.execute(
|
|
||||||
select(func.count(Post.id))
|
|
||||||
.where(Post.translated_source_lang.is_(None))
|
|
||||||
.where(or_(
|
|
||||||
Post.post_title.is_not(None), Post.description.is_not(None),
|
|
||||||
))
|
|
||||||
)).scalar_one()
|
|
||||||
# Live progress: is a sweep running now, and what did the last one do?
|
|
||||||
# (run-until-done re-enqueues itself, so `active` stays true across a
|
|
||||||
# bulk re-translate; `last_run` surfaces a completed run's outcome.)
|
|
||||||
active = (await session.execute(
|
|
||||||
select(func.count(TaskRun.id))
|
|
||||||
.where(TaskRun.task_name.in_(translation_tasks))
|
|
||||||
.where(TaskRun.status == "running")
|
|
||||||
)).scalar_one()
|
|
||||||
last = (await session.execute(
|
|
||||||
select(TaskRun.task_name, TaskRun.status, TaskRun.finished_at)
|
|
||||||
.where(TaskRun.task_name.in_(translation_tasks))
|
|
||||||
.where(TaskRun.finished_at.is_not(None))
|
|
||||||
.order_by(TaskRun.finished_at.desc())
|
|
||||||
.limit(1)
|
|
||||||
)).first()
|
|
||||||
base_url = (cfg.interpreter_base_url or "").strip()
|
|
||||||
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
|
||||||
return jsonify({
|
|
||||||
"enabled": cfg.translation_enabled,
|
|
||||||
"base_url_set": bool(base_url),
|
|
||||||
"healthy": healthy,
|
|
||||||
"untranslated_count": int(untranslated),
|
|
||||||
"active": int(active) > 0,
|
|
||||||
"last_run": {
|
|
||||||
"task": last[0].rsplit(".", 1)[-1],
|
|
||||||
"status": last[1],
|
|
||||||
"finished_at": last[2].isoformat() if last[2] else None,
|
|
||||||
} if last else None,
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/settings/translation/test", methods=["POST"])
|
|
||||||
async def translation_test():
|
|
||||||
"""On-demand reachability check for a GIVEN Interpreter base URL (the Settings
|
|
||||||
'Test connection' button) — pings /v1/health without saving, so the operator
|
|
||||||
can verify a URL before enabling. Health runs in a thread (sync client)."""
|
|
||||||
body = await request.get_json()
|
|
||||||
base_url = ""
|
|
||||||
if isinstance(body, dict):
|
|
||||||
base_url = (body.get("base_url") or "").strip()
|
|
||||||
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
|
||||||
return jsonify({"healthy": healthy})
|
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/settings/translation/probe", methods=["POST"])
|
|
||||||
async def translation_probe():
|
|
||||||
"""Diagnostic for the Settings 'Test translation' box: translate a pasted
|
|
||||||
snippet WITHOUT saving anything, returning what Interpreter *detected*
|
|
||||||
(language + confidence) alongside the result. Lets the operator see why a
|
|
||||||
given string was (mis-)detected — e.g. a short English title flagged as
|
|
||||||
another language — so a detection guard can be tuned from real numbers.
|
|
||||||
Read-only: no post is touched. Uses the currently-saved base URL + target."""
|
|
||||||
body = await request.get_json(silent=True)
|
|
||||||
body = body if isinstance(body, dict) else {}
|
|
||||||
text = (body.get("text") or "").strip()
|
|
||||||
if not text:
|
|
||||||
return jsonify({"error": "provide text to translate"}), 400
|
|
||||||
async with get_session() as session:
|
|
||||||
cfg = await ImportSettings.load(session)
|
|
||||||
base_url = (cfg.interpreter_base_url or "").strip()
|
|
||||||
if not base_url:
|
|
||||||
return jsonify({"error": "no Interpreter base URL is set"}), 400
|
|
||||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
|
||||||
try:
|
|
||||||
res = await asyncio.to_thread(
|
|
||||||
ic.translate, [text], base_url=base_url, target=target,
|
|
||||||
)
|
|
||||||
except ic.InterpreterUnavailable as e:
|
|
||||||
return jsonify({"error": f"Interpreter unavailable: {e}"}), 503
|
|
||||||
except ic.InterpreterBadRequest as e:
|
|
||||||
return jsonify({"error": f"Interpreter rejected the request: {e}"}), 400
|
|
||||||
translations = res.get("translations") or []
|
|
||||||
return jsonify({
|
|
||||||
"target": target,
|
|
||||||
"detected_lang": res.get("detected_lang"),
|
|
||||||
"detected_confidence": res.get("detected_confidence"),
|
|
||||||
"engine": res.get("engine"),
|
|
||||||
"engine_version": res.get("engine_version"),
|
|
||||||
"translated": translations[0] if translations else None,
|
|
||||||
})
|
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/settings/translation/run", methods=["POST"])
|
|
||||||
async def translation_run():
|
|
||||||
"""Enqueue the translate sweep now (the Settings 'Translate now' button).
|
|
||||||
Runs in drain mode — run-until-done — so one press chases the whole
|
|
||||||
untranslated backlog to zero rather than a single 300-post chunk."""
|
|
||||||
async with get_session() as session:
|
|
||||||
cfg = await ImportSettings.load(session)
|
|
||||||
if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
|
|
||||||
return jsonify(
|
|
||||||
{"error": "translation is disabled or no base URL is set"}
|
|
||||||
), 400
|
|
||||||
from ..tasks.translation import translate_posts
|
|
||||||
|
|
||||||
r = translate_posts.delay(drain=True)
|
|
||||||
return jsonify({"celery_task_id": r.id}), 202
|
|
||||||
|
|
||||||
|
|
||||||
@settings_bp.route("/settings/translation/retranslate", methods=["POST"])
|
|
||||||
async def translation_retranslate():
|
|
||||||
"""Re-translate stored translations after a model change (m146). Body:
|
|
||||||
``{"artist_id": <int>}`` aims at one artist; ``{"all": true}`` re-runs every
|
|
||||||
artist. ``all`` must be explicit so an empty/typo body can't wipe everything.
|
|
||||||
Clears the scoped translations and enqueues the run-until-done retranslate
|
|
||||||
sweep (the Interpreter cache re-translates on a changed model, is cache-fast
|
|
||||||
otherwise). Same enabled + base-URL guard as 'Translate now'."""
|
|
||||||
body = await request.get_json(silent=True)
|
|
||||||
body = body if isinstance(body, dict) else {}
|
|
||||||
artist_id = body.get("artist_id")
|
|
||||||
do_all = bool(body.get("all"))
|
|
||||||
if artist_id is None and not do_all:
|
|
||||||
return jsonify(
|
|
||||||
{"error": "provide artist_id, or all=true to re-translate everything"}
|
|
||||||
), 400
|
|
||||||
if artist_id is not None:
|
|
||||||
try:
|
|
||||||
artist_id = int(artist_id)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return jsonify({"error": "artist_id must be an integer"}), 400
|
|
||||||
|
|
||||||
async with get_session() as session:
|
|
||||||
cfg = await ImportSettings.load(session)
|
|
||||||
if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
|
|
||||||
return jsonify(
|
|
||||||
{"error": "translation is disabled or no base URL is set"}
|
|
||||||
), 400
|
|
||||||
from ..tasks.translation import retranslate_posts
|
|
||||||
|
|
||||||
# artist_id wins when both are sent; otherwise all=true → None (every artist).
|
|
||||||
artist_ids = [artist_id] if artist_id is not None else None
|
|
||||||
r = retranslate_posts.delay(artist_ids=artist_ids)
|
|
||||||
return jsonify({"celery_task_id": r.id}), 202
|
|
||||||
|
|||||||
@@ -230,6 +230,52 @@ async def set_backfill(source_id: int):
|
|||||||
return jsonify(record.to_dict())
|
return jsonify(record.to_dict())
|
||||||
|
|
||||||
|
|
||||||
|
@sources_bp.route("/<int:source_id>/preview", methods=["POST"])
|
||||||
|
async def preview_source_endpoint(source_id: int):
|
||||||
|
"""Plan #708 B4: dry-run — count what a backfill WOULD download for a native
|
||||||
|
platform (Patreon today), without downloading. Walks the first few feed pages
|
||||||
|
and counts media not already in the seen/dead ledgers. Returns
|
||||||
|
{total_new, posts_scanned, pages_scanned, has_more, sample[]} or 409 + reason
|
||||||
|
(unresolvable campaign id / auth / drift). 400 for gallery-dl platforms (no
|
||||||
|
cheap dry-run — their verify is a slow --simulate)."""
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from ..services.credential_service import CredentialService
|
||||||
|
from ..services.download_backends import preview_source, uses_native_ingester
|
||||||
|
from ..tasks._sync_engine import sync_session_factory
|
||||||
|
from .credentials import _get_crypto
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
rec = await SourceService(session).get(source_id)
|
||||||
|
if rec is None:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
if not uses_native_ingester(rec.platform):
|
||||||
|
return _bad(
|
||||||
|
"unsupported",
|
||||||
|
detail="Preview is only available for native-ingester platforms.",
|
||||||
|
status=400,
|
||||||
|
)
|
||||||
|
cred = CredentialService(session, _get_crypto())
|
||||||
|
cookies_path = await cred.get_cookies_path(rec.platform)
|
||||||
|
auth_token = await cred.get_token(rec.platform)
|
||||||
|
|
||||||
|
# The walk + ledger reads are sync (run off the request loop); the process
|
||||||
|
# sync engine is the same one the download task uses.
|
||||||
|
result = await preview_source(
|
||||||
|
platform=rec.platform,
|
||||||
|
url=rec.url,
|
||||||
|
source_id=source_id,
|
||||||
|
config_overrides=rec.config_overrides or {},
|
||||||
|
cookies_path=str(cookies_path) if cookies_path else None,
|
||||||
|
images_root=Path("/images"),
|
||||||
|
sync_session_factory=sync_session_factory(),
|
||||||
|
auth_token=auth_token,
|
||||||
|
)
|
||||||
|
if "error" in result:
|
||||||
|
return _bad("preview_failed", detail=result["error"], status=409)
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
||||||
async def check_source(source_id: int):
|
async def check_source(source_id: int):
|
||||||
"""FC-3c: enqueue a download for this source.
|
"""FC-3c: enqueue a download for this source.
|
||||||
|
|||||||
@@ -11,11 +11,10 @@ suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
|
|||||||
|
|
||||||
@suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
|
@suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
|
||||||
async def get_suggestions(image_id: int):
|
async def get_suggestions(image_id: int):
|
||||||
# ?min=<float> overrides the per-head suggest thresholds for INCLUSION. The
|
# ?min=<float> overrides the configured per-category thresholds so the typed
|
||||||
# rail sends min=0 in its single per-image fetch to get EVERY head (each row
|
# tag-input dropdown can surface EVERY stored prediction (min=0), including
|
||||||
# still carries above_threshold vs its natural cut), then derives the panel
|
# low-confidence actions/features, in canonical formatting. Omitted → the
|
||||||
# (above_threshold) and the typed dropdown (all, filtered by text) client-side
|
# curated above-threshold list the Suggestions panel uses.
|
||||||
# — no second request. Omitted → only above-threshold rows.
|
|
||||||
override = None
|
override = None
|
||||||
raw_min = request.args.get("min")
|
raw_min = request.args.get("min")
|
||||||
if raw_min is not None:
|
if raw_min is not None:
|
||||||
@@ -37,19 +36,16 @@ async def get_suggestions(image_id: int):
|
|||||||
"category": s.category,
|
"category": s.category,
|
||||||
"score": round(s.score, 4),
|
"score": round(s.score, 4),
|
||||||
"source": s.source,
|
"source": s.source,
|
||||||
# whether the score cleared the head's own suggest cut.
|
"creates_new_tag": s.creates_new_tag,
|
||||||
# The single min=0 fetch returns every head; the panel
|
# raw model key (alias is stored under this) + whether an
|
||||||
# shows above_threshold, the typed dropdown shows all and
|
# operator alias produced this suggestion — drive the
|
||||||
# annotates each match with its score.
|
# modal's "Treat as alias"/"Remove alias" affordances.
|
||||||
"above_threshold": s.above_threshold,
|
"raw_name": s.raw_name,
|
||||||
|
"via_alias": s.via_alias,
|
||||||
# operator dismissed this tag for this image — surfaced
|
# operator dismissed this tag for this image — surfaced
|
||||||
# (not dropped) so the rail can show it rejected + offer
|
# (not dropped) so the rail can show it rejected + offer
|
||||||
# one-click un-reject.
|
# one-click un-reject.
|
||||||
"rejected": s.rejected,
|
"rejected": s.rejected,
|
||||||
# the crop region that produced this tag (#1206) —
|
|
||||||
# {bbox,kind,detector} or null (whole-image won). Drives
|
|
||||||
# the hover→overlay highlight.
|
|
||||||
"grounding": s.grounding,
|
|
||||||
}
|
}
|
||||||
for s in items
|
for s in items
|
||||||
]
|
]
|
||||||
@@ -73,6 +69,26 @@ async def accept_suggestion(image_id: int):
|
|||||||
return jsonify({"accepted": True, "tag_id": tag_id})
|
return jsonify({"accepted": True, "tag_id": tag_id})
|
||||||
|
|
||||||
|
|
||||||
|
@suggestions_bp.route(
|
||||||
|
"/images/<int:image_id>/suggestions/alias", methods=["POST"]
|
||||||
|
)
|
||||||
|
async def alias_suggestion(image_id: int):
|
||||||
|
body = await request.get_json()
|
||||||
|
required = {"alias_string", "alias_category", "canonical_tag_id"}
|
||||||
|
if not body or not required.issubset(body):
|
||||||
|
return jsonify({"error": f"required: {sorted(required)}"}), 400
|
||||||
|
canonical_tag_id = body["canonical_tag_id"]
|
||||||
|
async with get_session() as session:
|
||||||
|
await AllowlistService(session).add_alias_and_accept(
|
||||||
|
image_id,
|
||||||
|
body["alias_string"],
|
||||||
|
body["alias_category"],
|
||||||
|
canonical_tag_id,
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify({"accepted": True, "tag_id": canonical_tag_id})
|
||||||
|
|
||||||
|
|
||||||
@suggestions_bp.route(
|
@suggestions_bp.route(
|
||||||
"/images/<int:image_id>/suggestions/dismiss", methods=["POST"]
|
"/images/<int:image_id>/suggestions/dismiss", methods=["POST"]
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -11,7 +11,6 @@ from ..models.tag import image_tag
|
|||||||
from ..models.tag_suggestion_rejection import TagSuggestionRejection
|
from ..models.tag_suggestion_rejection import TagSuggestionRejection
|
||||||
from ..services.bulk_tag_service import BulkTagService
|
from ..services.bulk_tag_service import BulkTagService
|
||||||
from ..services.ml.aliases import AliasService
|
from ..services.ml.aliases import AliasService
|
||||||
from ..services.ml.heads import ground_applied_tag
|
|
||||||
from ..services.series_match_service import SeriesMatchService
|
from ..services.series_match_service import SeriesMatchService
|
||||||
from ..services.series_service import SeriesError, SeriesService
|
from ..services.series_service import SeriesError, SeriesService
|
||||||
from ..services.tag_directory_service import TagDirectoryService
|
from ..services.tag_directory_service import TagDirectoryService
|
||||||
@@ -311,21 +310,6 @@ async def confirm_tag_on_image(image_id: int, tag_id: int):
|
|||||||
return "", 204
|
return "", 204
|
||||||
|
|
||||||
|
|
||||||
@tags_bp.route(
|
|
||||||
"/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
|
|
||||||
)
|
|
||||||
async def tag_grounding(image_id: int, tag_id: int):
|
|
||||||
"""Which crop region best explains an ALREADY-APPLIED tag on this image
|
|
||||||
(#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
|
|
||||||
mirroring the suggestion rail's live grounding. Computed on demand (applied
|
|
||||||
tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
|
|
||||||
has_head: bool}; has_head False means the tag has no head to localize with,
|
|
||||||
so the chip shows no overlay."""
|
|
||||||
async with get_session() as session:
|
|
||||||
grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
|
|
||||||
return jsonify({"grounding": grounding, "has_head": has_head})
|
|
||||||
|
|
||||||
|
|
||||||
@tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
|
@tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
|
||||||
async def get_tag(tag_id: int):
|
async def get_tag(tag_id: int):
|
||||||
"""Resolve a single tag (used by the gallery to label its active
|
"""Resolve a single tag (used by the gallery to label its active
|
||||||
|
|||||||
@@ -35,7 +35,6 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.backup",
|
"backend.app.tasks.backup",
|
||||||
"backend.app.tasks.admin",
|
"backend.app.tasks.admin",
|
||||||
"backend.app.tasks.library_audit",
|
"backend.app.tasks.library_audit",
|
||||||
"backend.app.tasks.translation",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
app.conf.update(
|
app.conf.update(
|
||||||
@@ -64,20 +63,9 @@ def make_celery() -> Celery:
|
|||||||
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||||
# Translation backfill hits the LLM (~1–6s/item) → the long lane so it
|
|
||||||
# never starves the quick self-healing sweeps (#143).
|
|
||||||
"backend.app.tasks.translation.*": {"queue": "maintenance_long"},
|
|
||||||
},
|
},
|
||||||
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
# Heavy ML tasks need fair dispatch — see ImageRepo's precedent.
|
||||||
task_acks_late=True,
|
task_acks_late=True,
|
||||||
# Deploy graceful-shutdown safety: with acks_late, a task killed because
|
|
||||||
# it outran the container's stop-grace window (SIGKILL) is re-queued
|
|
||||||
# rather than silently lost. Safe because our long tasks are idempotent +
|
|
||||||
# chunked (translation per-post commit, downloads terminal-status, audits
|
|
||||||
# chunk) and the 5-min recovery sweeps re-drive anything left non-terminal
|
|
||||||
# — a re-run resumes cleanly and never corrupts. No redeliver-loop risk:
|
|
||||||
# heavy GPU work is tombstoned via gpu_queue, not run inline in a worker.
|
|
||||||
task_reject_on_worker_lost=True,
|
|
||||||
worker_prefetch_multiplier=1,
|
worker_prefetch_multiplier=1,
|
||||||
# Broker resilience (2026-06-24): a swarm overlay-network blip after a
|
# Broker resilience (2026-06-24): a swarm overlay-network blip after a
|
||||||
# redeploy left Redis healthy but transiently unreachable, and a worker
|
# redeploy left Redis healthy but transiently unreachable, and a worker
|
||||||
@@ -115,24 +103,10 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
"task": "backend.app.tasks.maintenance.cleanup_old_tasks",
|
||||||
"schedule": 86400.0, # daily
|
"schedule": 86400.0, # daily
|
||||||
},
|
},
|
||||||
"cleanup-orphaned-temp-files": {
|
|
||||||
"task": "backend.app.tasks.maintenance.cleanup_orphaned_temp_files",
|
|
||||||
"schedule": 86400.0, # daily — sweep .part/.partial left by a
|
|
||||||
# download/import killed mid-write (graceful-shutdown fallout)
|
|
||||||
},
|
|
||||||
"train-heads-nightly": {
|
"train-heads-nightly": {
|
||||||
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
"task": "backend.app.tasks.ml.scheduled_train_heads",
|
||||||
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
"schedule": 86400.0, # passive cadence; manual retrain stays available
|
||||||
},
|
},
|
||||||
"refresh-character-prototypes": {
|
|
||||||
"task": "backend.app.tasks.ml.refresh_character_prototypes",
|
|
||||||
"schedule": 900.0, # ~15 min; cheap global-gate no-op when idle (#1317)
|
|
||||||
},
|
|
||||||
"reconcile-character-prototypes-nightly": {
|
|
||||||
"task": "backend.app.tasks.ml.refresh_character_prototypes",
|
|
||||||
"schedule": 86400.0, # nightly FULL reconcile (belt-and-suspenders)
|
|
||||||
"args": (True,), # full=True
|
|
||||||
},
|
|
||||||
"apply-head-tags-daily": {
|
"apply-head-tags-daily": {
|
||||||
"task": "backend.app.tasks.ml.scheduled_apply_head_tags",
|
"task": "backend.app.tasks.ml.scheduled_apply_head_tags",
|
||||||
"schedule": 86400.0, # no-op unless head_auto_apply_enabled
|
"schedule": 86400.0, # no-op unless head_auto_apply_enabled
|
||||||
@@ -164,38 +138,6 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
|
"task": "backend.app.tasks.ml.scheduled_ccip_auto_apply",
|
||||||
"schedule": 86400.0, # no-op unless ccip_auto_apply_enabled
|
"schedule": 86400.0, # no-op unless ccip_auto_apply_enabled
|
||||||
},
|
},
|
||||||
"retract-auto-tags-daily": {
|
|
||||||
"task": "backend.app.tasks.ml.scheduled_retract_auto_tags",
|
|
||||||
"schedule": 86400.0, # soft auto-apply: drop auto-tags now below
|
|
||||||
# their threshold (m139); no-op unless the auto-apply switch is on
|
|
||||||
},
|
|
||||||
"presentation-auto-apply-daily": {
|
|
||||||
"task": "backend.app.tasks.ml.scheduled_presentation_auto_apply",
|
|
||||||
"schedule": 86400.0, # auto-hide banner chrome (#141);
|
|
||||||
# no-op unless presentation_auto_apply_enabled
|
|
||||||
},
|
|
||||||
"process-auto-apply-daily": {
|
|
||||||
"task": "backend.app.tasks.ml.scheduled_process_auto_apply",
|
|
||||||
"schedule": 86400.0, # auto-tag wip/editor process art (#1464);
|
|
||||||
# no-op unless process_auto_apply_enabled (opt-in)
|
|
||||||
},
|
|
||||||
"soft-wip-conflict-audit-daily": {
|
|
||||||
"task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
|
||||||
"schedule": 86400.0, # flag ring-loud soft-WIP (sketch/doodle) tags
|
|
||||||
# for review (#1474); no-op with no content heads
|
|
||||||
},
|
|
||||||
"prune-presentation-reviews-daily": {
|
|
||||||
"task": "backend.app.tasks.ml.prune_presentation_reviews",
|
|
||||||
"schedule": 86400.0, # retention: drop resolved review flags >30d
|
|
||||||
},
|
|
||||||
"translate-posts-8h": {
|
|
||||||
"task": "backend.app.tasks.translation.translate_posts",
|
|
||||||
"schedule": 28800.0, # every 8h: steady-state cadence for the
|
|
||||||
# trickle of newly-imported posts (no-op unless translation
|
|
||||||
# configured + healthy). One bounded 300-chunk per fire — the
|
|
||||||
# one-time backlog drains via the "Translate now" button
|
|
||||||
# (drain=True, run-until-done), not this sweep.
|
|
||||||
},
|
|
||||||
"snapshot-head-metrics-daily": {
|
"snapshot-head-metrics-daily": {
|
||||||
"task": "backend.app.tasks.maintenance.snapshot_head_metrics",
|
"task": "backend.app.tasks.maintenance.snapshot_head_metrics",
|
||||||
"schedule": 86400.0,
|
"schedule": 86400.0,
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ from .artist import Artist
|
|||||||
from .artist_visit import ArtistVisit
|
from .artist_visit import ArtistVisit
|
||||||
from .backup_run import BackupRun
|
from .backup_run import BackupRun
|
||||||
from .base import Base
|
from .base import Base
|
||||||
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
|
||||||
from .credential import Credential
|
from .credential import Credential
|
||||||
from .download_event import DownloadEvent
|
from .download_event import DownloadEvent
|
||||||
from .external_link import ExternalLink
|
from .external_link import ExternalLink
|
||||||
@@ -28,7 +27,6 @@ from .pixiv_failed_media import PixivFailedMedia
|
|||||||
from .pixiv_seen_media import PixivSeenMedia
|
from .pixiv_seen_media import PixivSeenMedia
|
||||||
from .post import Post
|
from .post import Post
|
||||||
from .post_attachment import PostAttachment
|
from .post_attachment import PostAttachment
|
||||||
from .presentation_review import PresentationReview
|
|
||||||
from .series_chapter import SeriesChapter
|
from .series_chapter import SeriesChapter
|
||||||
from .series_page import SeriesPage
|
from .series_page import SeriesPage
|
||||||
from .series_suggestion import SeriesSuggestion
|
from .series_suggestion import SeriesSuggestion
|
||||||
@@ -58,7 +56,6 @@ __all__ = [
|
|||||||
"SubscribeStarSeenMedia",
|
"SubscribeStarSeenMedia",
|
||||||
"Post",
|
"Post",
|
||||||
"PostAttachment",
|
"PostAttachment",
|
||||||
"PresentationReview",
|
|
||||||
"SeriesChapter",
|
"SeriesChapter",
|
||||||
"SeriesPage",
|
"SeriesPage",
|
||||||
"SeriesSuggestion",
|
"SeriesSuggestion",
|
||||||
@@ -82,8 +79,6 @@ __all__ = [
|
|||||||
"HeadTrainingRun",
|
"HeadTrainingRun",
|
||||||
"TagAlias",
|
"TagAlias",
|
||||||
"TagHead",
|
"TagHead",
|
||||||
"CharacterPrototype",
|
|
||||||
"CcipPrototypeState",
|
|
||||||
"TagPositiveConfirmation",
|
"TagPositiveConfirmation",
|
||||||
"TagSuggestionRejection",
|
"TagSuggestionRejection",
|
||||||
"TaskRun",
|
"TaskRun",
|
||||||
|
|||||||
@@ -1,62 +0,0 @@
|
|||||||
"""Precomputed CCIP character prototypes (#1317, milestone 138).
|
|
||||||
|
|
||||||
The live matcher (ccip.match_image) needs each character's reference figure
|
|
||||||
vectors. Building that on the request path reloaded EVERY figure CCIP vector in
|
|
||||||
the library on any change (~4s, invalidated by every character accept). These
|
|
||||||
tables make the references a PRECOMPUTED, INCREMENTAL artifact refreshed off the
|
|
||||||
request path (services.ml.character_prototypes):
|
|
||||||
|
|
||||||
- CharacterPrototype: a character's reference vectors, capped to
|
|
||||||
MLSettings.ccip_prototype_cap so MATCH cost doesn't grow with a character's
|
|
||||||
popularity. The async matcher only READS these.
|
|
||||||
- CcipPrototypeState: a per-character fingerprint (reference count + max region
|
|
||||||
id) so a refresh rebuilds ONLY the characters whose references changed, and
|
|
||||||
its updated_at lets the matcher's cache reload just the advanced characters.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from pgvector.sqlalchemy import Vector
|
|
||||||
from sqlalchemy import DateTime, ForeignKey, Integer, String, func
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from .base import Base
|
|
||||||
from .image_region import CCIP_DIM
|
|
||||||
|
|
||||||
|
|
||||||
class CharacterPrototype(Base):
|
|
||||||
__tablename__ = "character_prototype"
|
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
|
||||||
# The character tag these vectors identify. CASCADE: deleting the tag drops
|
|
||||||
# its prototypes.
|
|
||||||
tag_id: Mapped[int] = mapped_column(
|
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
|
||||||
)
|
|
||||||
# A reference figure/face CCIP vector (same space as
|
|
||||||
# ImageRegion.ccip_embedding).
|
|
||||||
ccip_embedding: Mapped[list[float]] = mapped_column(
|
|
||||||
Vector(CCIP_DIM), nullable=False
|
|
||||||
)
|
|
||||||
# Provenance: the region this vector was copied from. SET NULL so pruning a
|
|
||||||
# region doesn't delete the prototype mid-cycle (the next refresh reconciles).
|
|
||||||
region_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class CcipPrototypeState(Base):
|
|
||||||
__tablename__ = "ccip_prototype_state"
|
|
||||||
|
|
||||||
# One row per character that currently has prototypes.
|
|
||||||
tag_id: Mapped[int] = mapped_column(
|
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
|
||||||
)
|
|
||||||
# count(reference regions) + max(region id) at last build — the cheap
|
|
||||||
# per-character change detector that drives incremental rebuilds.
|
|
||||||
fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
# Bumped when this character's prototypes are rebuilt; the matcher cache
|
|
||||||
# reloads only characters whose updated_at advanced.
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
||||||
)
|
|
||||||
@@ -92,48 +92,6 @@ class ImportSettings(Base):
|
|||||||
Boolean, nullable=False, default=True, server_default="true",
|
Boolean, nullable=False, default=True, server_default="true",
|
||||||
)
|
)
|
||||||
|
|
||||||
# -- Post-text translation via the Interpreter LAN service (milestone 143).
|
|
||||||
# Off by default with NO default host — it needs a reachable Interpreter
|
|
||||||
# service (the operator's, behind a reverse proxy), which not every install
|
|
||||||
# has; the operator sets the URL and flips it on. Empty base_url OR disabled
|
|
||||||
# → the translate sweep no-ops.
|
|
||||||
translation_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=False, server_default="false",
|
|
||||||
)
|
|
||||||
interpreter_base_url: Mapped[str] = mapped_column(
|
|
||||||
Text, nullable=False, default="", server_default="",
|
|
||||||
)
|
|
||||||
translation_target_lang: Mapped[str] = mapped_column(
|
|
||||||
Text, nullable=False, default="en", server_default="en",
|
|
||||||
)
|
|
||||||
# The latin-script acceptance floor for the translation gate: a translation
|
|
||||||
# whose Interpreter-reported confidence is below this is kept as the original
|
|
||||||
# (operator-tunable, milestone 155). Default 0.9 — stricter than the old
|
|
||||||
# hardcoded 0.8, because Interpreter confidently mis-detects short ASCII
|
|
||||||
# English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
|
|
||||||
# trusted regardless (script-detected). Per-post overrides handle the misses.
|
|
||||||
translation_min_confidence: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.9, server_default="0.9",
|
|
||||||
)
|
|
||||||
|
|
||||||
# Title-based WIP auto-tagging (task #1458). When a freshly-imported post's
|
|
||||||
# TITLE explicitly declares work-in-progress ("WIP" / "work in progress"),
|
|
||||||
# the importer applies the `wip` system tag to its images — the artist's own
|
|
||||||
# label, used to keep unfinished pieces out of the Explore/gallery browse. ON
|
|
||||||
# by default (rule 26 — the feature works out of the box). Gates only the
|
|
||||||
# LIVE import hook; the existing catalogue is caught by the operator-triggered
|
|
||||||
# "Scan existing posts" backfill (which runs regardless of this flag).
|
|
||||||
wip_title_tagging_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=True, server_default="true",
|
|
||||||
)
|
|
||||||
# Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
|
|
||||||
# a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
|
|
||||||
# are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
|
|
||||||
# precision tier is opt-in (the ring-loud audit surfaces false positives).
|
|
||||||
wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=False, server_default="false",
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def load(cls, session) -> ImportSettings:
|
async def load(cls, session) -> ImportSettings:
|
||||||
"""The singleton settings row (id=1), via an async session."""
|
"""The singleton settings row (id=1), via an async session."""
|
||||||
|
|||||||
@@ -10,7 +10,6 @@ from sqlalchemy import (
|
|||||||
Integer,
|
Integer,
|
||||||
String,
|
String,
|
||||||
func,
|
func,
|
||||||
select,
|
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -64,9 +63,7 @@ class MLSettings(Base):
|
|||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True
|
||||||
)
|
)
|
||||||
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
||||||
# Support floor raised 30→50 (operator-asked 2026-07-06): a head needs
|
Integer, nullable=False, default=30
|
||||||
# more human labels before it may fire without a human.
|
|
||||||
Integer, nullable=False, default=50
|
|
||||||
)
|
)
|
||||||
# CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
|
# CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
|
||||||
# over-fired (high-reference characters matched a scatter of images); 0.85
|
# over-fired (high-reference characters matched a scatter of images); 0.85
|
||||||
@@ -81,47 +78,7 @@ class MLSettings(Base):
|
|||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True
|
||||||
)
|
)
|
||||||
ccip_auto_apply_threshold: Mapped[float] = mapped_column(
|
ccip_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||||
# Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident
|
Float, nullable=False, default=0.92
|
||||||
# character matches auto-tag.
|
|
||||||
Float, nullable=False, default=0.95
|
|
||||||
)
|
|
||||||
# -- Presentation chrome auto-hide (#141) -------------------------------
|
|
||||||
# `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
|
|
||||||
# with its OWN flat threshold (decoupled from content-head graduation) and is
|
|
||||||
# HIDDEN from the gallery. Hiding is consequential so it runs HIGH. When an
|
|
||||||
# image would be auto-hidden but ALSO scores >= presentation_conflict_threshold
|
|
||||||
# on a content head, it's still hidden but flagged for review
|
|
||||||
# (PresentationReview, mode='chrome') instead of buried silently. ON by default
|
|
||||||
# (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
|
|
||||||
# screenshot` are no longer chrome — they went to the PROCESS path below.
|
|
||||||
presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=True
|
|
||||||
)
|
|
||||||
presentation_auto_apply_threshold: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.90
|
|
||||||
)
|
|
||||||
presentation_conflict_threshold: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.50
|
|
||||||
)
|
|
||||||
# -- Process auto-apply (#1464) ----------------------------------------
|
|
||||||
# `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
|
|
||||||
# screenshots that must stay OUT of head/CCIP training but, unlike chrome,
|
|
||||||
# remain VISIBLE in the gallery (operator 2026-07-12). They auto-apply on the
|
|
||||||
# sweep with their OWN flat threshold and a PROVISIONAL source (`process_auto`,
|
|
||||||
# in training_data._AUTO_SOURCES) so the head NEVER trains on its own output —
|
|
||||||
# it learns only from title (`wip_title`) + manual labels, which breaks the
|
|
||||||
# runaway loop. When a process tag would be applied but the image ALSO scores
|
|
||||||
# >= process_conflict_threshold on a content head, it's flagged for review
|
|
||||||
# (PresentationReview, mode='process') rather than silently marked. OFF by
|
|
||||||
# default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
|
|
||||||
process_auto_apply_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=False
|
|
||||||
)
|
|
||||||
process_auto_apply_threshold: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.90
|
|
||||||
)
|
|
||||||
process_conflict_threshold: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.50
|
|
||||||
)
|
)
|
||||||
# Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
|
# Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
|
||||||
# existing libraries keep their stored value until the operator re-embeds.
|
# existing libraries keep their stored value until the operator re-embeds.
|
||||||
@@ -134,93 +91,6 @@ class MLSettings(Base):
|
|||||||
embedder_model_name: Mapped[str] = mapped_column(
|
embedder_model_name: Mapped[str] = mapped_column(
|
||||||
String(128), nullable=False, default="google/siglip2-so400m-patch16-512"
|
String(128), nullable=False, default="google/siglip2-so400m-patch16-512"
|
||||||
)
|
)
|
||||||
# -- Crop proposers / detectors (#1202, #134) --------------------------
|
|
||||||
# WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config
|
|
||||||
# lives HERE (DB) and is announced to the GPU agent in the lease — same as
|
|
||||||
# the embedder model — so it is UI-tunable with NO restart, and the agent's
|
|
||||||
# env is bootstrap-only. Each weights spec is an ultralytics builtin name,
|
|
||||||
# an http(s) URL, or "hf_repo::file" (agent's _resolve). enabled off (or an
|
|
||||||
# empty weights) skips that proposer. All ON by default (operator 2026-07-05)
|
|
||||||
# so a fresh install crops out-of-the-box.
|
|
||||||
# person: general COCO figure detector for Western/realistic art the anime
|
|
||||||
# person-detector misses → NMS-merged with imgutils → CCIP + concept.
|
|
||||||
detector_person_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=True
|
|
||||||
)
|
|
||||||
detector_person_weights: Mapped[str] = mapped_column(
|
|
||||||
String(512), nullable=False, default="yolo11n.pt"
|
|
||||||
)
|
|
||||||
detector_person_conf: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.35
|
|
||||||
)
|
|
||||||
# anatomy: booru_yolo anime/furry/NSFW torso components → concept crops.
|
|
||||||
# Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the
|
|
||||||
# upstream repo so the URL resolves. License UNSTATED — fine for a private
|
|
||||||
# homelab (operator accepted #1202).
|
|
||||||
detector_anatomy_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=True
|
|
||||||
)
|
|
||||||
detector_anatomy_weights: Mapped[str] = mapped_column(
|
|
||||||
String(512), nullable=False,
|
|
||||||
default=(
|
|
||||||
"https://github.com/aperveyev/booru_yolo/raw/main/models/"
|
|
||||||
"yolov11m_aa22.pt"
|
|
||||||
),
|
|
||||||
)
|
|
||||||
detector_anatomy_conf: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.30
|
|
||||||
)
|
|
||||||
# panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x).
|
|
||||||
detector_panel_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=True
|
|
||||||
)
|
|
||||||
detector_panel_weights: Mapped[str] = mapped_column(
|
|
||||||
String(512), nullable=False,
|
|
||||||
default="mosesb/best-comic-panel-detection::best.pt",
|
|
||||||
)
|
|
||||||
detector_panel_conf: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.30
|
|
||||||
)
|
|
||||||
# Per-frame caps bound the crop→embed explosion; max_regions is the hard
|
|
||||||
# per-job backstop; dedupe_iou drops near-duplicate crops before the embed.
|
|
||||||
detector_max_figures: Mapped[int] = mapped_column(
|
|
||||||
Integer, nullable=False, default=8
|
|
||||||
)
|
|
||||||
detector_max_components: Mapped[int] = mapped_column(
|
|
||||||
Integer, nullable=False, default=8
|
|
||||||
)
|
|
||||||
detector_max_panels: Mapped[int] = mapped_column(
|
|
||||||
Integer, nullable=False, default=8
|
|
||||||
)
|
|
||||||
detector_max_regions: Mapped[int] = mapped_column(
|
|
||||||
Integer, nullable=False, default=128
|
|
||||||
)
|
|
||||||
detector_dedupe_iou: Mapped[float] = mapped_column(
|
|
||||||
Float, nullable=False, default=0.85
|
|
||||||
)
|
|
||||||
# -- CCIP character prototypes (#1317) ---------------------------------
|
|
||||||
# The per-character reference set is precomputed + refreshed INCREMENTALLY
|
|
||||||
# (services.ml.character_prototypes) instead of rebuilt on the request path.
|
|
||||||
# ccip_ref_signature is the cheap GLOBAL gate — when it's unchanged the
|
|
||||||
# refresh no-ops; ccip_prototype_cap bounds the reference vectors kept per
|
|
||||||
# character so MATCH cost doesn't grow with a character's popularity.
|
|
||||||
ccip_ref_signature: Mapped[str | None] = mapped_column(
|
|
||||||
String(128), nullable=True
|
|
||||||
)
|
|
||||||
ccip_prototype_cap: Mapped[int] = mapped_column(
|
|
||||||
Integer, nullable=False, default=64
|
|
||||||
)
|
|
||||||
updated_at: Mapped[datetime] = mapped_column(
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
async def load(cls, session) -> MLSettings:
|
|
||||||
"""The singleton settings row (id=1), via an async session. Mirrors
|
|
||||||
ImportSettings.load — the shared singleton-loader pattern."""
|
|
||||||
return (await session.execute(select(cls).where(cls.id == 1))).scalar_one()
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def load_sync(cls, session) -> MLSettings:
|
|
||||||
"""The singleton settings row (id=1), via a sync session."""
|
|
||||||
return session.execute(select(cls).where(cls.id == 1)).scalar_one()
|
|
||||||
|
|||||||
@@ -8,17 +8,7 @@ artist-filter queries don't depend on the Source detour).
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import (
|
from sqlalchemy import JSON, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||||
JSON,
|
|
||||||
CheckConstraint,
|
|
||||||
DateTime,
|
|
||||||
ForeignKey,
|
|
||||||
Integer,
|
|
||||||
String,
|
|
||||||
Text,
|
|
||||||
UniqueConstraint,
|
|
||||||
func,
|
|
||||||
)
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -33,10 +23,6 @@ class Post(Base):
|
|||||||
# (created in alembic 0030) covers that case via
|
# (created in alembic 0030) covers that case via
|
||||||
# (artist_id, external_post_id).
|
# (artist_id, external_post_id).
|
||||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||||
CheckConstraint(
|
|
||||||
"translation_override IN ('auto', 'force', 'original')",
|
|
||||||
name="ck_post_translation_override",
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
@@ -61,34 +47,6 @@ class Post(Base):
|
|||||||
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
description: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
attachment_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
attachment_count: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
|
||||||
# -- Post-text translation (milestone 143). Filled by the translate_posts
|
|
||||||
# sweep via the Interpreter LAN service so viewing is instant.
|
|
||||||
# translated_source_lang is the DETECTED original language; "en" (or a
|
|
||||||
# passthrough) means nothing to translate and the *_translated columns stay
|
|
||||||
# NULL. engine_version keys the Interpreter cache — re-runs are ~1ms and a
|
|
||||||
# model upgrade re-translates instead of serving stale.
|
|
||||||
post_title_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
||||||
description_translated: Mapped[str | None] = mapped_column(Text, nullable=True)
|
|
||||||
translated_source_lang: Mapped[str | None] = mapped_column(
|
|
||||||
String(8), nullable=True
|
|
||||||
)
|
|
||||||
translation_engine_version: Mapped[str | None] = mapped_column(
|
|
||||||
String(128), nullable=True
|
|
||||||
)
|
|
||||||
translated_at: Mapped[datetime | None] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=True
|
|
||||||
)
|
|
||||||
# Sticky per-post override of the translation decision (milestone 155):
|
|
||||||
# 'auto' = the acceptance gate decides; 'force' = always store Interpreter's
|
|
||||||
# translation even below the confidence floor (rescue a skipped legit-foreign
|
|
||||||
# title); 'original' = never translate, keep the original (kill a confidently
|
|
||||||
# mis-flagged one the floor can't catch). The sweep reads this on every run,
|
|
||||||
# and re-translate leaves 'original' posts alone, so the choice survives a
|
|
||||||
# Re-translate-all.
|
|
||||||
translation_override: Mapped[str] = mapped_column(
|
|
||||||
String(16), nullable=False, default="auto", server_default="auto",
|
|
||||||
)
|
|
||||||
|
|
||||||
downloaded_at: Mapped[datetime] = mapped_column(
|
downloaded_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,48 +0,0 @@
|
|||||||
"""PresentationReview — a system-tag the auto-apply sweep applied that ALSO looked
|
|
||||||
like real content, flagged for operator review (milestone 141 + #1464).
|
|
||||||
|
|
||||||
When a sweep applies a system tag but the image ALSO scores highly on a content
|
|
||||||
head, it still applies the tag but records this row so a review strip can surface
|
|
||||||
it ("⚠ also looks like <conflict tag>"). Two modes (#1464): 'chrome' (banner —
|
|
||||||
image is HIDDEN, review is keep-hidden / un-hide) and 'process' (wip / editor
|
|
||||||
screenshot — image stays VISIBLE, review is confirm / remove-tag). Resolved rows
|
|
||||||
are pruned by retention.
|
|
||||||
"""
|
|
||||||
|
|
||||||
from datetime import datetime
|
|
||||||
|
|
||||||
from sqlalchemy import DateTime, Float, ForeignKey, String, func
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from .base import Base
|
|
||||||
|
|
||||||
|
|
||||||
class PresentationReview(Base):
|
|
||||||
__tablename__ = "presentation_review"
|
|
||||||
|
|
||||||
image_record_id: Mapped[int] = mapped_column(
|
|
||||||
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
|
||||||
)
|
|
||||||
# The presentation tag that was auto-applied (banner / editor screenshot).
|
|
||||||
tag_id: Mapped[int] = mapped_column(
|
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
|
||||||
)
|
|
||||||
# The content tag the image ALSO scored high on — the "concerning" signal.
|
|
||||||
# SET NULL (not CASCADE): losing the conflict tag shouldn't erase the flag.
|
|
||||||
conflict_tag_id: Mapped[int | None] = mapped_column(
|
|
||||||
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True
|
|
||||||
)
|
|
||||||
conflict_score: Mapped[float] = mapped_column(Float, nullable=False)
|
|
||||||
# Which sweep flagged this (#1464): 'chrome' (banner, hidden) or 'process'
|
|
||||||
# (wip / editor screenshot, shown). Drives which review strip surfaces it and
|
|
||||||
# what "resolve" means (un-hide vs remove-tag). Existing rows backfill 'chrome'.
|
|
||||||
mode: Mapped[str] = mapped_column(
|
|
||||||
String(16), nullable=False, default="chrome", server_default="chrome"
|
|
||||||
)
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
|
||||||
)
|
|
||||||
# Set when the operator keeps-hidden or un-hides; retention prunes resolved.
|
|
||||||
resolved_at: Mapped[datetime | None] = mapped_column(
|
|
||||||
DateTime(timezone=True), nullable=True
|
|
||||||
)
|
|
||||||
@@ -43,20 +43,11 @@ class TagKind(StrEnum):
|
|||||||
# to keep historic tag rows queryable.
|
# to keep historic tag rows queryable.
|
||||||
|
|
||||||
|
|
||||||
# The seeded system tags (migration 0075). Two behavior groups (#1464):
|
# The seeded system tags (migration 0075). PRESENTATION tags additionally
|
||||||
# CHROME (banner): clusters on UI chrome, not content → HIDDEN from the default
|
# hide from whole-image similarity results — they cluster on UI chrome, not
|
||||||
# gallery + from similarity; auto-applied via the sweep's chrome mode.
|
# content. `wip` is real art: only the training pipelines exclude it.
|
||||||
# PROCESS (wip, editor screenshot): real-but-unfinished art / program screenshots
|
|
||||||
# → SHOWN in the gallery (operator 2026-07-12), but excluded from the Explore
|
|
||||||
# rabbit-hole; auto-applied via the sweep's process mode (provisional source,
|
|
||||||
# ring-loud review guard).
|
|
||||||
# ALL three are excluded from OTHER concepts' head/CCIP training (training-hygiene,
|
|
||||||
# keyed on is_system); a system tag's OWN head trains on them — that's what makes
|
|
||||||
# auto-flagging work.
|
|
||||||
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
|
SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot")
|
||||||
CHROME_SYSTEM_TAGS = ("banner",)
|
PRESENTATION_SYSTEM_TAGS = ("banner", "editor screenshot")
|
||||||
PROCESS_SYSTEM_TAGS = ("wip", "editor screenshot")
|
|
||||||
WIP_SYSTEM_TAG = "wip"
|
|
||||||
|
|
||||||
image_tag = Table(
|
image_tag = Table(
|
||||||
"image_tag",
|
"image_tag",
|
||||||
|
|||||||
@@ -73,12 +73,5 @@ class TagHead(Base):
|
|||||||
trained_at: Mapped[datetime] = mapped_column(
|
trained_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
# Training-data fingerprint (positives + rejections) at last fit — the
|
|
||||||
# incremental-retrain change detector (#1317 p2). A manual Retrain refits only
|
|
||||||
# heads whose fingerprint moved; the nightly run ignores it (full reconcile).
|
|
||||||
# NULL forces a refit (pre-fingerprint heads).
|
|
||||||
train_fingerprint: Mapped[str | None] = mapped_column(
|
|
||||||
String(128), nullable=True
|
|
||||||
)
|
|
||||||
# Extra detail (auto-apply operating point, F1, etc.) — non-load-bearing.
|
# Extra detail (auto-apply operating point, F1, etc.) — non-load-bearing.
|
||||||
metrics: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)
|
metrics: Mapped[dict[str, Any] | None] = mapped_column(JSONB, nullable=True)
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ import asyncio
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from .gallery_dl import DownloadResult, ErrorType
|
from .gallery_dl import DownloadResult, ErrorType
|
||||||
|
from .native_ingest_common import NativeIngestError
|
||||||
from .patreon_ingester import PatreonIngester
|
from .patreon_ingester import PatreonIngester
|
||||||
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
from .patreon_resolver import extract_vanity, resolve_campaign_id_for_source
|
||||||
from .pixiv_client import user_id_from_url
|
from .pixiv_client import user_id_from_url
|
||||||
@@ -203,6 +204,49 @@ async def _run_native_ingester(
|
|||||||
return dl_result, resolved_campaign_id
|
return dl_result, resolved_campaign_id
|
||||||
|
|
||||||
|
|
||||||
|
async def preview_source(
|
||||||
|
*,
|
||||||
|
platform: str,
|
||||||
|
url: str,
|
||||||
|
source_id: int,
|
||||||
|
config_overrides: dict | None,
|
||||||
|
cookies_path: str | None,
|
||||||
|
images_root: Path,
|
||||||
|
sync_session_factory,
|
||||||
|
auth_token: str | None = None,
|
||||||
|
page_limit: int = 3,
|
||||||
|
) -> dict:
|
||||||
|
"""Dry-run preview for a native platform (plan #708 B4): resolve the campaign
|
||||||
|
id, then walk a few pages counting media not already seen/dead — no download.
|
||||||
|
|
||||||
|
Returns the preview dict (total_new / posts_scanned / pages_scanned /
|
||||||
|
has_more / sample), or `{"error": msg}` on a resolve / auth / drift failure.
|
||||||
|
Native-only — the caller gates on `uses_native_ingester`.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
campaign_id, _ = await _resolve_native_campaign_id(
|
||||||
|
platform, url, cookies_path, config_overrides or {}
|
||||||
|
)
|
||||||
|
if not campaign_id:
|
||||||
|
return {"error": _campaign_resolution_error(platform, url)}
|
||||||
|
ingester = _native_ingester_cls(platform)(
|
||||||
|
images_root=images_root,
|
||||||
|
cookies_path=cookies_path,
|
||||||
|
session_factory=sync_session_factory,
|
||||||
|
auth_token=auth_token,
|
||||||
|
)
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
try:
|
||||||
|
result = await loop.run_in_executor(
|
||||||
|
None,
|
||||||
|
lambda: ingester.preview(source_id, campaign_id, page_limit=page_limit),
|
||||||
|
)
|
||||||
|
except NativeIngestError as exc:
|
||||||
|
return {"error": f"Couldn't preview: {exc}"}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
async def verify_source_credential(
|
async def verify_source_credential(
|
||||||
*,
|
*,
|
||||||
platform: str,
|
platform: str,
|
||||||
|
|||||||
@@ -35,14 +35,9 @@ class InvalidUrlError(Exception):
|
|||||||
# reviewers catch drift.
|
# reviewers catch drift.
|
||||||
_PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
_PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||||
("patreon", re.compile(
|
("patreon", re.compile(
|
||||||
# Three creator URL shapes — bare (patreon.com/Atole), `c/`, and `cw/`
|
|
||||||
# (the "creator workspace" URL served once subscribed, see
|
|
||||||
# patreon_resolver._VANITY_RE). A trailing sub-path is allowed so a
|
|
||||||
# creator's inner page still derives the slug. Nav pages stay excluded.
|
|
||||||
r"^https?://(?:www\.)?patreon\.com/"
|
r"^https?://(?:www\.)?patreon\.com/"
|
||||||
r"(?:cw/|c/)?"
|
r"(?!home$|search\b|messages\b|notifications\b|library\b|settings\b|posts\b|c/)"
|
||||||
r"(?!(?:home|search|messages|notifications|library|settings|posts)(?:[/?#]|$))"
|
r"(?P<slug>[^/?#]+)/?$",
|
||||||
r"(?P<slug>[^/?#]+)",
|
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)),
|
)),
|
||||||
("subscribestar", re.compile(
|
("subscribestar", re.compile(
|
||||||
|
|||||||
@@ -22,16 +22,8 @@ from sqlalchemy import Select, and_, distinct, exists, func, or_, select
|
|||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.orm import aliased
|
from sqlalchemy.orm import aliased
|
||||||
|
|
||||||
from ..models import (
|
from ..models import Artist, ImageProvenance, ImageRecord, Post, Source, Tag
|
||||||
Artist,
|
from ..models.tag import PRESENTATION_SYSTEM_TAGS, image_tag
|
||||||
ImageProvenance,
|
|
||||||
ImageRecord,
|
|
||||||
Post,
|
|
||||||
Source,
|
|
||||||
Tag,
|
|
||||||
TagPositiveConfirmation,
|
|
||||||
)
|
|
||||||
from ..models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
|
|
||||||
from .pagination import decode_cursor, encode_cursor
|
from .pagination import decode_cursor, encode_cursor
|
||||||
from .tag_query import (
|
from .tag_query import (
|
||||||
fandom_join_alias,
|
fandom_join_alias,
|
||||||
@@ -187,7 +179,7 @@ def _apply_scope(
|
|||||||
stmt, *, tag_ids, post_id, artist_id, media_type,
|
stmt, *, tag_ids, post_id, artist_id, media_type,
|
||||||
tag_or_groups=None, tag_exclude=None,
|
tag_or_groups=None, tag_exclude=None,
|
||||||
platform=None, untagged=False, no_artist=False,
|
platform=None, untagged=False, no_artist=False,
|
||||||
date_from=None, date_to=None, hidden_tag_ids=None,
|
date_from=None, date_to=None,
|
||||||
):
|
):
|
||||||
"""Apply the composable gallery filters to a statement.
|
"""Apply the composable gallery filters to a statement.
|
||||||
|
|
||||||
@@ -224,12 +216,6 @@ def _apply_scope(
|
|||||||
stmt = stmt.where(image_in_any_tag_scope(group))
|
stmt = stmt.where(image_in_any_tag_scope(group))
|
||||||
if tag_exclude:
|
if tag_exclude:
|
||||||
stmt = stmt.where(~image_in_any_tag_scope(tag_exclude))
|
stmt = stmt.where(~image_in_any_tag_scope(tag_exclude))
|
||||||
# Presentation chrome (banner / editor screenshot) is hidden from the default
|
|
||||||
# gallery — an implicit exclude the caller supplies unless the operator asked
|
|
||||||
# to include hidden or is explicitly filtering for a presentation tag
|
|
||||||
# (milestone 141). `wip` is NOT hidden. Resolved to ids by _hidden_tag_ids.
|
|
||||||
if hidden_tag_ids:
|
|
||||||
stmt = stmt.where(~image_in_any_tag_scope(hidden_tag_ids))
|
|
||||||
prov = _provenance_clause(post_id, artist_id)
|
prov = _provenance_clause(post_id, artist_id)
|
||||||
if prov is not None:
|
if prov is not None:
|
||||||
stmt = stmt.where(prov)
|
stmt = stmt.where(prov)
|
||||||
@@ -396,25 +382,6 @@ def _diversify_similar(src, rows, limit, *, dup_threshold=8, lam=0.40):
|
|||||||
return [kept[i] for i in order]
|
return [kept[i] for i in order]
|
||||||
|
|
||||||
|
|
||||||
def _reach_sample(rows, limit, reach):
|
|
||||||
"""From a distance-sorted candidate pool (nearest first), pick a spread of ranks
|
|
||||||
that MIXES near (tag the current cluster) and mid-far (escape it) BEFORE dedup +
|
|
||||||
MMR — the Explore "reach" dial (#1476).
|
|
||||||
|
|
||||||
reach in (0, 1]: the sampled span grows outward from the anchor (0.25→1.0 of the
|
|
||||||
pool), evenly strided from rank 0 so the nearest are still represented. In a
|
|
||||||
dense signature the nearest ranks are near-identical, so reaching farther is the
|
|
||||||
only way to hand MMR genuinely different content — MMR alone can't escape a pool
|
|
||||||
that's already all-near. reach<=0 or a small pool passes through unchanged."""
|
|
||||||
n = len(rows)
|
|
||||||
want = max(limit * 8, 100)
|
|
||||||
if reach <= 0 or n <= want:
|
|
||||||
return rows
|
|
||||||
span = int(min(1.0, 0.25 + 0.75 * reach) * n)
|
|
||||||
idx = sorted({min(int(i * span / want), n - 1) for i in range(want)})
|
|
||||||
return [rows[i] for i in idx]
|
|
||||||
|
|
||||||
|
|
||||||
async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
|
async def _artists_for(session, image_ids: list[int]) -> dict[int, dict]:
|
||||||
"""Map image_id -> {"name","slug"} via the canonical
|
"""Map image_id -> {"name","slug"} via the canonical
|
||||||
image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
|
image_record.artist_id (FC-2d-vii-c). Bounded by page size."""
|
||||||
@@ -435,32 +402,6 @@ class GalleryService:
|
|||||||
def __init__(self, session: AsyncSession):
|
def __init__(self, session: AsyncSession):
|
||||||
self.session = session
|
self.session = session
|
||||||
|
|
||||||
async def _hidden_tag_ids(
|
|
||||||
self, include_hidden, tag_ids, tag_or_groups,
|
|
||||||
) -> list[int] | None:
|
|
||||||
"""Chrome (banner) tag ids to implicitly exclude from a gallery query, or
|
|
||||||
None. None when the caller asked to include hidden, when the operator is
|
|
||||||
explicitly filtering FOR a chrome tag (they clearly want to see it), or when
|
|
||||||
no chrome tags exist. (milestone 141; #1464: editor screenshot is now PROCESS
|
|
||||||
— shown — so only `banner` hides here.)"""
|
|
||||||
if include_hidden:
|
|
||||||
return None
|
|
||||||
rows = await self.session.execute(
|
|
||||||
select(Tag.id).where(
|
|
||||||
Tag.is_system.is_(True),
|
|
||||||
Tag.name.in_(CHROME_SYSTEM_TAGS),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
pres = [r[0] for r in rows]
|
|
||||||
if not pres:
|
|
||||||
return None
|
|
||||||
explicit = set(tag_ids or [])
|
|
||||||
for group in tag_or_groups or []:
|
|
||||||
explicit.update(group)
|
|
||||||
if explicit & set(pres):
|
|
||||||
return None
|
|
||||||
return pres
|
|
||||||
|
|
||||||
async def scroll(
|
async def scroll(
|
||||||
self,
|
self,
|
||||||
cursor: str | None,
|
cursor: str | None,
|
||||||
@@ -477,16 +418,12 @@ class GalleryService:
|
|||||||
no_artist: bool = False,
|
no_artist: bool = False,
|
||||||
date_from: datetime | None = None,
|
date_from: datetime | None = None,
|
||||||
date_to: datetime | None = None,
|
date_to: datetime | None = None,
|
||||||
include_hidden: bool = False,
|
|
||||||
) -> GalleryPage:
|
) -> GalleryPage:
|
||||||
if limit < 1 or limit > 200:
|
if limit < 1 or limit > 200:
|
||||||
raise ValueError("limit must be between 1 and 200")
|
raise ValueError("limit must be between 1 and 200")
|
||||||
_require_single_filter(
|
_require_single_filter(
|
||||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||||
)
|
)
|
||||||
hidden = await self._hidden_tag_ids(
|
|
||||||
include_hidden, tag_ids, tag_or_groups,
|
|
||||||
)
|
|
||||||
|
|
||||||
# eff is the ACTIVE sort column (effective_date or earliest_post_date);
|
# eff is the ACTIVE sort column (effective_date or earliest_post_date);
|
||||||
# the cursor, ordering and year/month grouping all key off it, so the
|
# the cursor, ordering and year/month grouping all key off it, so the
|
||||||
@@ -499,7 +436,7 @@ class GalleryService:
|
|||||||
artist_id=artist_id, media_type=media_type,
|
artist_id=artist_id, media_type=media_type,
|
||||||
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
||||||
platform=platform, untagged=untagged, no_artist=no_artist,
|
platform=platform, untagged=untagged, no_artist=no_artist,
|
||||||
date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
|
date_from=date_from, date_to=date_to,
|
||||||
)
|
)
|
||||||
|
|
||||||
descending = sort not in _ASCENDING_SORTS
|
descending = sort not in _ASCENDING_SORTS
|
||||||
@@ -552,7 +489,6 @@ class GalleryService:
|
|||||||
no_artist: bool = False,
|
no_artist: bool = False,
|
||||||
date_from: datetime | None = None,
|
date_from: datetime | None = None,
|
||||||
date_to: datetime | None = None,
|
date_to: datetime | None = None,
|
||||||
include_hidden: bool = False,
|
|
||||||
) -> list[TimelineBucket]:
|
) -> list[TimelineBucket]:
|
||||||
eff = _effective_date_col()
|
eff = _effective_date_col()
|
||||||
year_col = func.date_part("year", eff).label("yr")
|
year_col = func.date_part("year", eff).label("yr")
|
||||||
@@ -564,15 +500,12 @@ class GalleryService:
|
|||||||
_require_single_filter(
|
_require_single_filter(
|
||||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||||
)
|
)
|
||||||
hidden = await self._hidden_tag_ids(
|
|
||||||
include_hidden, tag_ids, tag_or_groups,
|
|
||||||
)
|
|
||||||
stmt = _apply_scope(
|
stmt = _apply_scope(
|
||||||
stmt, tag_ids=tag_ids, post_id=post_id,
|
stmt, tag_ids=tag_ids, post_id=post_id,
|
||||||
artist_id=artist_id, media_type=media_type,
|
artist_id=artist_id, media_type=media_type,
|
||||||
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
||||||
platform=platform, untagged=untagged, no_artist=no_artist,
|
platform=platform, untagged=untagged, no_artist=no_artist,
|
||||||
date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
|
date_from=date_from, date_to=date_to,
|
||||||
)
|
)
|
||||||
stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
|
stmt = stmt.group_by(year_col, month_col).order_by(year_col.desc(), month_col.desc())
|
||||||
rows = (await self.session.execute(stmt)).all()
|
rows = (await self.session.execute(stmt)).all()
|
||||||
@@ -586,7 +519,7 @@ class GalleryService:
|
|||||||
tag_exclude: list[int] | None = None,
|
tag_exclude: list[int] | None = None,
|
||||||
platform: str | None = None, untagged: bool = False,
|
platform: str | None = None, untagged: bool = False,
|
||||||
no_artist: bool = False, date_from: datetime | None = None,
|
no_artist: bool = False, date_from: datetime | None = None,
|
||||||
date_to: datetime | None = None, include_hidden: bool = False,
|
date_to: datetime | None = None,
|
||||||
) -> str | None:
|
) -> str | None:
|
||||||
"""Returns a cursor that, when passed to scroll() with the same sort,
|
"""Returns a cursor that, when passed to scroll() with the same sort,
|
||||||
positions at the first image of the given year-month. None if the
|
positions at the first image of the given year-month. None if the
|
||||||
@@ -603,15 +536,12 @@ class GalleryService:
|
|||||||
_require_single_filter(
|
_require_single_filter(
|
||||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||||
)
|
)
|
||||||
hidden = await self._hidden_tag_ids(
|
|
||||||
include_hidden, tag_ids, tag_or_groups,
|
|
||||||
)
|
|
||||||
stmt = _apply_scope(
|
stmt = _apply_scope(
|
||||||
stmt, tag_ids=tag_ids, post_id=post_id,
|
stmt, tag_ids=tag_ids, post_id=post_id,
|
||||||
artist_id=artist_id, media_type=media_type,
|
artist_id=artist_id, media_type=media_type,
|
||||||
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
tag_or_groups=tag_or_groups, tag_exclude=tag_exclude,
|
||||||
platform=platform, untagged=untagged, no_artist=no_artist,
|
platform=platform, untagged=untagged, no_artist=no_artist,
|
||||||
date_from=date_from, date_to=date_to, hidden_tag_ids=hidden,
|
date_from=date_from, date_to=date_to,
|
||||||
)
|
)
|
||||||
descending = sort != "oldest"
|
descending = sort != "oldest"
|
||||||
if descending:
|
if descending:
|
||||||
@@ -636,7 +566,6 @@ class GalleryService:
|
|||||||
platform: str | None = None,
|
platform: str | None = None,
|
||||||
untagged: bool = False, no_artist: bool = False,
|
untagged: bool = False, no_artist: bool = False,
|
||||||
date_from: datetime | None = None, date_to: datetime | None = None,
|
date_from: datetime | None = None, date_to: datetime | None = None,
|
||||||
include_hidden: bool = False,
|
|
||||||
) -> GalleryFacets:
|
) -> GalleryFacets:
|
||||||
"""Live facet counts scoped to the current filter. Each facet GROUP is
|
"""Live facet counts scoped to the current filter. Each facet GROUP is
|
||||||
computed with all OTHER active filters applied but its OWN selection
|
computed with all OTHER active filters applied but its OWN selection
|
||||||
@@ -647,14 +576,10 @@ class GalleryService:
|
|||||||
_require_single_filter(
|
_require_single_filter(
|
||||||
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
tag_ids, post_id, artist_id, tag_or_groups, tag_exclude,
|
||||||
)
|
)
|
||||||
hidden = await self._hidden_tag_ids(
|
|
||||||
include_hidden, tag_ids, tag_or_groups,
|
|
||||||
)
|
|
||||||
common = {
|
common = {
|
||||||
"tag_ids": tag_ids, "post_id": post_id,
|
"tag_ids": tag_ids, "post_id": post_id,
|
||||||
"artist_id": artist_id, "media_type": media_type,
|
"artist_id": artist_id, "media_type": media_type,
|
||||||
"tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
|
"tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
|
||||||
"hidden_tag_ids": hidden,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
# total — the full active filter (the headline result count).
|
# total — the full active filter (the headline result count).
|
||||||
@@ -735,8 +660,6 @@ class GalleryService:
|
|||||||
platform: str | None = None,
|
platform: str | None = None,
|
||||||
untagged: bool = False, no_artist: bool = False,
|
untagged: bool = False, no_artist: bool = False,
|
||||||
date_from: datetime | None = None, date_to: datetime | None = None,
|
date_from: datetime | None = None, date_to: datetime | None = None,
|
||||||
exclude_wip: bool = False,
|
|
||||||
reach: float = 0.0, exclude_ids: list[int] | None = None,
|
|
||||||
) -> list[GalleryImage] | None:
|
) -> list[GalleryImage] | None:
|
||||||
"""Visual "more like this": images near `image_id`'s SigLIP embedding
|
"""Visual "more like this": images near `image_id`'s SigLIP embedding
|
||||||
(pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
|
(pgvector, HNSW-indexed — alembic 0036), then DIVERSIFIED so the result
|
||||||
@@ -765,33 +688,22 @@ class GalleryService:
|
|||||||
# wide pool there's nothing but the near-dupes to choose from. Widened
|
# wide pool there's nothing but the near-dupes to choose from. Widened
|
||||||
# (5×→8×, cap 200→400) so the stronger MMR has genuinely distinct
|
# (5×→8×, cap 200→400) so the stronger MMR has genuinely distinct
|
||||||
# neighbourhoods to reach into for more variance (operator, 2026-07-01).
|
# neighbourhoods to reach into for more variance (operator, 2026-07-01).
|
||||||
# Explore's reach>0 (#1476) widens it a LOT more: in a dense signature the
|
pool_n = min(400, max(limit * 8, 100))
|
||||||
# nearest few hundred are all near-identical, so far-enough candidates only
|
|
||||||
# exist deeper in the ranked pool. _reach_sample then strides across them.
|
|
||||||
if reach > 0:
|
|
||||||
pool_n = min(1000, max(limit * 25, 100))
|
|
||||||
else:
|
|
||||||
pool_n = min(400, max(limit * 8, 100))
|
|
||||||
distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding)
|
distance = ImageRecord.siglip_embedding.cosine_distance(src.siglip_embedding)
|
||||||
eff = _effective_date_col()
|
eff = _effective_date_col()
|
||||||
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
|
stmt = select(ImageRecord, Post.post_date, eff.label("eff"))
|
||||||
stmt = _outer_join_primary_post(stmt)
|
stmt = _outer_join_primary_post(stmt)
|
||||||
# Chrome (banner, #128) clusters on UI rather than content, so near any one
|
# Presentation images (banner / editor-screenshot system tags, #128)
|
||||||
# of them they'd fill the grid → excluded from CANDIDATES always (the anchor
|
# cluster on UI chrome rather than content, so near any one of them
|
||||||
# itself may be a banner). PROCESS art (wip / editor screenshot) stays
|
# they'd fill the grid. Excluded from CANDIDATES only — the anchor
|
||||||
# surfaced here by default (real content; only the training pipelines exclude
|
# itself may be a banner, and `wip` stays surfaced (real art; only
|
||||||
# it), but the Explore rabbit-hole passes exclude_wip to also drop the whole
|
# the training pipelines exclude it).
|
||||||
# process group so a browse doesn't keep surfacing work-in-progress
|
|
||||||
# (operator, 2026-07-08; #1464 — editor now rides with wip here).
|
|
||||||
excluded_system_tags = CHROME_SYSTEM_TAGS
|
|
||||||
if exclude_wip:
|
|
||||||
excluded_system_tags = (*CHROME_SYSTEM_TAGS, *PROCESS_SYSTEM_TAGS)
|
|
||||||
presentation = (
|
presentation = (
|
||||||
select(image_tag.c.image_record_id)
|
select(image_tag.c.image_record_id)
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||||
.where(
|
.where(
|
||||||
Tag.is_system.is_(True),
|
Tag.is_system.is_(True),
|
||||||
Tag.name.in_(excluded_system_tags),
|
Tag.name.in_(PRESENTATION_SYSTEM_TAGS),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
stmt = stmt.where(
|
stmt = stmt.where(
|
||||||
@@ -799,10 +711,6 @@ class GalleryService:
|
|||||||
ImageRecord.id != image_id,
|
ImageRecord.id != image_id,
|
||||||
ImageRecord.id.not_in(presentation),
|
ImageRecord.id.not_in(presentation),
|
||||||
)
|
)
|
||||||
# Anti-revisit (#1476): the Explore walk passes its breadcrumb so already-
|
|
||||||
# walked images aren't re-served as neighbours — → can't loop you back in.
|
|
||||||
if exclude_ids:
|
|
||||||
stmt = stmt.where(ImageRecord.id.not_in(exclude_ids))
|
|
||||||
stmt = _apply_scope(
|
stmt = _apply_scope(
|
||||||
stmt, tag_ids=tag_ids, post_id=None,
|
stmt, tag_ids=tag_ids, post_id=None,
|
||||||
artist_id=artist_id, media_type=media_type,
|
artist_id=artist_id, media_type=media_type,
|
||||||
@@ -812,10 +720,6 @@ class GalleryService:
|
|||||||
)
|
)
|
||||||
stmt = stmt.order_by(distance.asc()).limit(pool_n)
|
stmt = stmt.order_by(distance.asc()).limit(pool_n)
|
||||||
rows = (await self.session.execute(stmt)).all()
|
rows = (await self.session.execute(stmt)).all()
|
||||||
# Explore reach: stride across an outward-growing distance span so the pool
|
|
||||||
# handed to MMR spans near→mid-far, not just the tight cluster (#1476).
|
|
||||||
if reach > 0:
|
|
||||||
rows = _reach_sample(rows, limit, reach)
|
|
||||||
rows = _diversify_similar(src, rows, limit)
|
rows = _diversify_similar(src, rows, limit)
|
||||||
artists = await _artists_for(self.session, [r[0].id for r in rows])
|
artists = await _artists_for(self.session, [r[0].id for r in rows])
|
||||||
return _gallery_images(rows, artists)
|
return _gallery_images(rows, artists)
|
||||||
@@ -827,14 +731,8 @@ class GalleryService:
|
|||||||
# Self-join Tag to resolve a character's fandom NAME (not just id) so the
|
# Self-join Tag to resolve a character's fandom NAME (not just id) so the
|
||||||
# modal chip can label it without an N+1 (shared tag_query helpers).
|
# modal chip can label it without an N+1 (shared tag_query helpers).
|
||||||
fandom_alias = fandom_join_alias()
|
fandom_alias = fandom_join_alias()
|
||||||
# source drives the auto-applied badge; confirmed = operator affirmed the
|
|
||||||
# tag (positive + retraction-shielded, milestone 139).
|
|
||||||
confirmed = exists().where(
|
|
||||||
TagPositiveConfirmation.image_record_id == image_id,
|
|
||||||
TagPositiveConfirmation.tag_id == Tag.id,
|
|
||||||
).label("confirmed")
|
|
||||||
tag_stmt = (
|
tag_stmt = (
|
||||||
select(*tag_columns(fandom_alias), image_tag.c.source, confirmed)
|
select(*tag_columns(fandom_alias))
|
||||||
.select_from(
|
.select_from(
|
||||||
Tag.__table__
|
Tag.__table__
|
||||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||||
|
|||||||
@@ -47,21 +47,9 @@ from .attachment_store import AttachmentStore
|
|||||||
from .audits import single_color
|
from .audits import single_color
|
||||||
from .link_extract import extract_external_links
|
from .link_extract import extract_external_links
|
||||||
from .thumbnailer import Thumbnailer
|
from .thumbnailer import Thumbnailer
|
||||||
from .wip_title import (
|
|
||||||
WIP_TITLE_SOFT_SOURCE,
|
|
||||||
WIP_TITLE_SOURCE,
|
|
||||||
apply_wip_image_tags,
|
|
||||||
matches_soft_wip_title,
|
|
||||||
matches_wip_title,
|
|
||||||
resolve_wip_tag_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
# Sentinel for the lazily-resolved wip tag id (distinguishes "not resolved yet"
|
|
||||||
# from a genuine None = tag absent, so absence is cached and not re-queried).
|
|
||||||
_UNSET = object()
|
|
||||||
|
|
||||||
|
|
||||||
class SkipReason(StrEnum):
|
class SkipReason(StrEnum):
|
||||||
too_small = "too_small"
|
too_small = "too_small"
|
||||||
@@ -195,10 +183,6 @@ class Importer:
|
|||||||
# invalidated mid-Importer (Importer instances are per-task /
|
# invalidated mid-Importer (Importer instances are per-task /
|
||||||
# per-archive-import so cross-instance staleness is harmless).
|
# per-archive-import so cross-instance staleness is harmless).
|
||||||
self._phash_candidates: list[tuple] | None = None
|
self._phash_candidates: list[tuple] | None = None
|
||||||
# Lazily-resolved `wip` system tag id for title-based WIP auto-tagging
|
|
||||||
# (task #1458). Sentinel _UNSET so a genuine None (tag absent) is cached
|
|
||||||
# and not re-queried per media. Importer is per-task, so this can't stale.
|
|
||||||
self._wip_tag_id: int | None = _UNSET
|
|
||||||
|
|
||||||
def _phash_candidates_cache(self) -> list[tuple]:
|
def _phash_candidates_cache(self) -> list[tuple]:
|
||||||
"""Cached `(phash, width, height, id)` rows from image_record.
|
"""Cached `(phash, width, height, id)` rows from image_record.
|
||||||
@@ -949,10 +933,6 @@ class Importer:
|
|||||||
# Thumbnail is queued separately by the calling task; the importer
|
# Thumbnail is queued separately by the calling task; the importer
|
||||||
# does not generate thumbnails inline so the import queue stays moving.
|
# does not generate thumbnails inline so the import queue stays moving.
|
||||||
|
|
||||||
# Title-based WIP auto-tag (task #1458): fresh import only, after the
|
|
||||||
# sidecar has linked the post so record.primary_post_id / its title exist.
|
|
||||||
self._maybe_apply_wip_title(record)
|
|
||||||
|
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
return ImportResult(status="imported", image_id=record.id)
|
return ImportResult(status="imported", image_id=record.id)
|
||||||
|
|
||||||
@@ -996,47 +976,6 @@ class Importer:
|
|||||||
self.session.commit()
|
self.session.commit()
|
||||||
return ImportResult(status="refreshed", image_id=existing.id)
|
return ImportResult(status="refreshed", image_id=existing.id)
|
||||||
|
|
||||||
def _maybe_apply_wip_title(self, record: ImageRecord) -> None:
|
|
||||||
"""Auto-apply the `wip` system tag to a FRESHLY-imported image when its
|
|
||||||
primary post's TITLE explicitly declares work-in-progress (task #1458 —
|
|
||||||
the artist's own "WIP" / "work in progress" label).
|
|
||||||
|
|
||||||
Called ONLY from the two new-record paths (never deep-scan / supersede),
|
|
||||||
so a manually-removed WIP tag is never re-applied by a routine re-scan —
|
|
||||||
removal sticks. The existing catalogue is covered separately by the
|
|
||||||
operator-triggered backfill sweep. Gated by the settings toggle, and
|
|
||||||
best-effort: any failure is logged, never allowed to fail the import."""
|
|
||||||
hard_on = self.settings.wip_title_tagging_enabled
|
|
||||||
soft_on = self.settings.wip_soft_title_tagging_enabled
|
|
||||||
if not (hard_on or soft_on):
|
|
||||||
return
|
|
||||||
if record.primary_post_id is None:
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
title = self.session.execute(
|
|
||||||
select(Post.post_title).where(Post.id == record.primary_post_id)
|
|
||||||
).scalar_one_or_none()
|
|
||||||
# HARD tier ("WIP"/"work in progress") wins — higher precision, and it
|
|
||||||
# trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
|
|
||||||
# that never trains (source wip_title_soft).
|
|
||||||
if hard_on and matches_wip_title(title):
|
|
||||||
source = WIP_TITLE_SOURCE
|
|
||||||
elif soft_on and matches_soft_wip_title(title):
|
|
||||||
source = WIP_TITLE_SOFT_SOURCE
|
|
||||||
else:
|
|
||||||
return
|
|
||||||
if self._wip_tag_id is _UNSET:
|
|
||||||
self._wip_tag_id = resolve_wip_tag_id(self.session)
|
|
||||||
if self._wip_tag_id is None:
|
|
||||||
return
|
|
||||||
apply_wip_image_tags(
|
|
||||||
self.session, [record.id], self._wip_tag_id, source=source
|
|
||||||
)
|
|
||||||
except Exception as exc: # noqa: BLE001 — a tag must never fail an import
|
|
||||||
log.warning(
|
|
||||||
"wip-title auto-tag failed for image %s: %s", record.id, exc
|
|
||||||
)
|
|
||||||
|
|
||||||
def _apply_post_fields(self, post: Post, sd) -> None:
|
def _apply_post_fields(self, post: Post, sd) -> None:
|
||||||
"""Write a parsed sidecar's post-level fields onto a Post — the SINGLE
|
"""Write a parsed sidecar's post-level fields onto a Post — the SINGLE
|
||||||
predicate shared by BOTH ingest paths: the per-media path (_apply_sidecar)
|
predicate shared by BOTH ingest paths: the per-media path (_apply_sidecar)
|
||||||
@@ -1314,10 +1253,6 @@ class Importer:
|
|||||||
# per-post Source row.
|
# per-post Source row.
|
||||||
self._apply_sidecar(record, path, artist, explicit_source=source)
|
self._apply_sidecar(record, path, artist, explicit_source=source)
|
||||||
|
|
||||||
# Title-based WIP auto-tag (task #1458): fresh import only, see the
|
|
||||||
# matching call in _import_media.
|
|
||||||
self._maybe_apply_wip_title(record)
|
|
||||||
|
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
return ImportResult(status="imported", image_id=record.id)
|
return ImportResult(status="imported", image_id=record.id)
|
||||||
|
|
||||||
|
|||||||
@@ -577,6 +577,73 @@ class Ingester:
|
|||||||
error_type=None, error_message=None,
|
error_type=None, error_message=None,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# -- preview (dry-run) -------------------------------------------------
|
||||||
|
|
||||||
|
def preview(
|
||||||
|
self,
|
||||||
|
source_id: int,
|
||||||
|
campaign_id: str,
|
||||||
|
*,
|
||||||
|
page_limit: int = 3,
|
||||||
|
sample_size: int = 10,
|
||||||
|
) -> dict:
|
||||||
|
"""Dry-run (plan #708 B4): walk up to `page_limit` pages and count media
|
||||||
|
NOT already in the seen/dead ledgers, WITHOUT downloading anything.
|
||||||
|
|
||||||
|
Read-only — only the seen/dead SELECTs touch the DB (short sessions). Lets
|
||||||
|
an operator gauge "is this source worth a backfill?" cheaply. Returns:
|
||||||
|
{total_new, posts_scanned, pages_scanned, has_more,
|
||||||
|
sample: [{title, date, new}, ...]} # sample = posts with new media
|
||||||
|
A client-level failure (auth/drift) propagates to the caller.
|
||||||
|
"""
|
||||||
|
total_new = 0
|
||||||
|
posts_scanned = 0
|
||||||
|
pages_scanned = 0
|
||||||
|
has_more = False
|
||||||
|
sample: list[dict] = []
|
||||||
|
unset = object()
|
||||||
|
last_page: object = unset
|
||||||
|
# #874: same gated-post gate as run() — the preview must not count
|
||||||
|
# blurred locked-preview media as "new", or it would overstate a gated
|
||||||
|
# source's backlog (preview/apply parity, rule 93).
|
||||||
|
post_is_gated = getattr(self.client, "post_is_gated", None)
|
||||||
|
for post, included, page_cursor in self.client.iter_posts(
|
||||||
|
campaign_id, cursor=None
|
||||||
|
):
|
||||||
|
if page_cursor != last_page:
|
||||||
|
last_page = page_cursor
|
||||||
|
pages_scanned += 1
|
||||||
|
if pages_scanned > page_limit:
|
||||||
|
has_more = True
|
||||||
|
pages_scanned = page_limit
|
||||||
|
break
|
||||||
|
posts_scanned += 1
|
||||||
|
if post_is_gated and post_is_gated(post):
|
||||||
|
continue
|
||||||
|
media = self.client.extract_media(post, included)
|
||||||
|
if not media:
|
||||||
|
continue
|
||||||
|
keys = [self._ledger_key(m) for m in media]
|
||||||
|
skip = self._seen_keys(source_id, keys) | self._dead_keys(source_id, keys)
|
||||||
|
new_count = sum(1 for m in media if self._ledger_key(m) not in skip)
|
||||||
|
total_new += new_count
|
||||||
|
if new_count > 0 and len(sample) < sample_size:
|
||||||
|
meta = self.client.post_meta(post)
|
||||||
|
sample.append(
|
||||||
|
{
|
||||||
|
"title": meta.get("title") or "(untitled)",
|
||||||
|
"date": meta.get("date"),
|
||||||
|
"new": new_count,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"total_new": total_new,
|
||||||
|
"posts_scanned": posts_scanned,
|
||||||
|
"pages_scanned": pages_scanned,
|
||||||
|
"has_more": has_more,
|
||||||
|
"sample": sample,
|
||||||
|
}
|
||||||
|
|
||||||
# -- failure mapping (adapter overrides) -------------------------------
|
# -- failure mapping (adapter overrides) -------------------------------
|
||||||
|
|
||||||
def _failure_result(self, exc: Exception, _result) -> DownloadResult:
|
def _failure_result(self, exc: Exception, _result) -> DownloadResult:
|
||||||
|
|||||||
@@ -1,152 +0,0 @@
|
|||||||
"""Interpreter translation client (milestone 143) — a thin SYNC wrapper over the
|
|
||||||
self-hosted Interpreter LAN service (LibreTranslate-compatible `/v1/translate`).
|
|
||||||
|
|
||||||
Sync (requests) because the only caller is the sync celery translate sweep, and
|
|
||||||
it mirrors FC's other platform clients. The service knows nothing about Curator —
|
|
||||||
all Curator logic stays here. base_url is operator-configured (empty until set;
|
|
||||||
behind a reverse proxy). Full API contract: Scribe note #1347.
|
|
||||||
"""
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
from email.utils import parsedate_to_datetime
|
|
||||||
|
|
||||||
import requests
|
|
||||||
from requests.adapters import HTTPAdapter
|
|
||||||
from urllib3.util.retry import Retry
|
|
||||||
|
|
||||||
|
|
||||||
class InterpreterUnavailable(Exception):
|
|
||||||
"""The translation engine is down / unreachable / draining — a connection
|
|
||||||
error, HTTP 429, or a 5xx (commonly 502/503/504 through a reverse proxy while
|
|
||||||
the service restarts). Retry later, don't drop the item. ``retry_after`` holds
|
|
||||||
the server's Retry-After hint in seconds when it sent one, so the caller can
|
|
||||||
back off exactly as long as it's asked to."""
|
|
||||||
|
|
||||||
def __init__(self, message: str, *, retry_after: float | None = None):
|
|
||||||
super().__init__(message)
|
|
||||||
self.retry_after = retry_after
|
|
||||||
|
|
||||||
|
|
||||||
class InterpreterBadRequest(Exception):
|
|
||||||
"""Bad request params (HTTP 400)."""
|
|
||||||
|
|
||||||
|
|
||||||
def _url(base_url: str, path: str) -> str:
|
|
||||||
return f"{base_url.rstrip('/')}{path}"
|
|
||||||
|
|
||||||
|
|
||||||
def _parse_retry_after(resp) -> float | None:
|
|
||||||
"""Parse a Retry-After header (RFC 7231: delta-seconds or an HTTP-date) into
|
|
||||||
non-negative seconds, or None if absent/unparseable — so a gracefully-
|
|
||||||
draining Interpreter can tell Curator exactly how long to wait before it
|
|
||||||
tries again."""
|
|
||||||
raw = (resp.headers.get("Retry-After") or "").strip()
|
|
||||||
if not raw:
|
|
||||||
return None
|
|
||||||
try:
|
|
||||||
return max(0.0, float(int(raw))) # delta-seconds form
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
try: # HTTP-date form
|
|
||||||
when = parsedate_to_datetime(raw)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return None
|
|
||||||
if when is None:
|
|
||||||
return None
|
|
||||||
if when.tzinfo is None:
|
|
||||||
when = when.replace(tzinfo=UTC)
|
|
||||||
return max(0.0, (when - datetime.now(UTC)).total_seconds())
|
|
||||||
|
|
||||||
|
|
||||||
# A shared session pools the keep-alive connection across the per-post sweep
|
|
||||||
# calls, and retries CONNECT failures only (connect=2, short backoff) — smoothing
|
|
||||||
# the instant a reverse proxy reloads. Status codes are deliberately NOT retried
|
|
||||||
# (status=0, raise_on_status=False): translate() maps 429/5xx → InterpreterUnavailable
|
|
||||||
# itself, and letting urllib3 retry a draining 503 would defeat the Retry-After
|
|
||||||
# backoff we honour upstream.
|
|
||||||
_retry = Retry(
|
|
||||||
total=None, connect=2, read=0, redirect=0, status=0,
|
|
||||||
backoff_factor=0.3, raise_on_status=False,
|
|
||||||
)
|
|
||||||
session = requests.Session()
|
|
||||||
_adapter = HTTPAdapter(max_retries=_retry)
|
|
||||||
session.mount("http://", _adapter)
|
|
||||||
session.mount("https://", _adapter)
|
|
||||||
|
|
||||||
|
|
||||||
def health(base_url: str, *, timeout: float = 5.0) -> bool:
|
|
||||||
"""True iff the Interpreter LLM engine is up. Any error (unset URL, network,
|
|
||||||
non-200, engine down) → False, so the sweep just no-ops rather than raising."""
|
|
||||||
if not base_url:
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
r = session.get(_url(base_url, "/v1/health"), timeout=timeout)
|
|
||||||
except requests.RequestException:
|
|
||||||
return False
|
|
||||||
if r.status_code != 200:
|
|
||||||
return False
|
|
||||||
engines = (r.json() or {}).get("engines") or {}
|
|
||||||
return bool(engines.get("llm"))
|
|
||||||
|
|
||||||
|
|
||||||
def translate(
|
|
||||||
texts: list[str], *, base_url: str, target: str = "en",
|
|
||||||
source: str = "auto", timeout: float = 120.0,
|
|
||||||
) -> dict:
|
|
||||||
"""Translate a batch. Returns::
|
|
||||||
|
|
||||||
{"translations": [str, ...], # SAME order & length as `texts`
|
|
||||||
"detected_lang": str | None, # aggregate: describes the FIRST item
|
|
||||||
"detected_confidence": float | None, # detector's confidence, if given
|
|
||||||
"engine": str | None,
|
|
||||||
"engine_version": str | None}
|
|
||||||
|
|
||||||
Interpreter batch metadata is aggregate (first item only) — fine for one
|
|
||||||
post's ``[title, description]`` batch since they share a language. Passthrough
|
|
||||||
items (already target-language / emoji-only) come back UNCHANGED in their
|
|
||||||
slot. Raises InterpreterUnavailable on a connection error / 429 / 5xx (retry
|
|
||||||
later, honouring Retry-After), InterpreterBadRequest on 400. (Scribe note
|
|
||||||
#1347.)
|
|
||||||
"""
|
|
||||||
if not texts:
|
|
||||||
return {"translations": [], "detected_lang": None,
|
|
||||||
"detected_confidence": None,
|
|
||||||
"engine": None, "engine_version": None}
|
|
||||||
try:
|
|
||||||
r = session.post(
|
|
||||||
_url(base_url, "/v1/translate"),
|
|
||||||
json={"q": list(texts), "source": source,
|
|
||||||
"target": target, "engine": "auto"},
|
|
||||||
timeout=timeout,
|
|
||||||
)
|
|
||||||
except requests.RequestException as e:
|
|
||||||
raise InterpreterUnavailable(str(e)) from e
|
|
||||||
if r.status_code == 400:
|
|
||||||
raise InterpreterBadRequest((r.json() or {}).get("error", "bad request"))
|
|
||||||
# A gracefully-draining service — often behind a reverse proxy — returns 429
|
|
||||||
# or a 5xx gateway error (502/503/504), not always a clean 503. Treat every
|
|
||||||
# one as "unavailable, retry later" and honour any Retry-After it sends, so a
|
|
||||||
# rolling Interpreter restart interrupts the sweep cleanly (instead of raising
|
|
||||||
# an opaque HTTPError) and resumes exactly when the service says it's ready.
|
|
||||||
if r.status_code == 429 or r.status_code >= 500:
|
|
||||||
raise InterpreterUnavailable(
|
|
||||||
f"interpreter unavailable (HTTP {r.status_code})",
|
|
||||||
retry_after=_parse_retry_after(r),
|
|
||||||
)
|
|
||||||
r.raise_for_status()
|
|
||||||
body = r.json() or {}
|
|
||||||
out = body.get("translatedText")
|
|
||||||
# q was an array → translatedText is an array (same order & length). Guard a
|
|
||||||
# scalar reply just in case (shouldn't happen for an array q).
|
|
||||||
if not isinstance(out, list):
|
|
||||||
out = [out]
|
|
||||||
interp = body.get("interpreter") or {}
|
|
||||||
detected = body.get("detectedLanguage") or {}
|
|
||||||
return {
|
|
||||||
"translations": out,
|
|
||||||
"detected_lang": detected.get("language"),
|
|
||||||
"detected_confidence": detected.get("confidence"),
|
|
||||||
"engine": interp.get("engine"),
|
|
||||||
"engine_version": interp.get("engine_version"),
|
|
||||||
}
|
|
||||||
@@ -12,11 +12,13 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
|
|
||||||
from ...models import TagSuggestionRejection
|
from ...models import TagSuggestionRejection
|
||||||
from ...models.tag import image_tag
|
from ...models.tag import image_tag
|
||||||
|
from .aliases import AliasService
|
||||||
|
|
||||||
|
|
||||||
class AllowlistService:
|
class AllowlistService:
|
||||||
def __init__(self, session: AsyncSession):
|
def __init__(self, session: AsyncSession):
|
||||||
self.session = session
|
self.session = session
|
||||||
|
self.aliases = AliasService(session)
|
||||||
|
|
||||||
async def _apply_image_tag(self, image_id: int, tag_id: int, source: str):
|
async def _apply_image_tag(self, image_id: int, tag_id: int, source: str):
|
||||||
stmt = insert(image_tag).values(
|
stmt = insert(image_tag).values(
|
||||||
@@ -39,6 +41,18 @@ class AllowlistService:
|
|||||||
await self._apply_image_tag(image_id, tag_id, source="ml_accepted")
|
await self._apply_image_tag(image_id, tag_id, source="ml_accepted")
|
||||||
await self._clear_rejection(image_id, tag_id)
|
await self._clear_rejection(image_id, tag_id)
|
||||||
|
|
||||||
|
async def add_alias_and_accept(
|
||||||
|
self,
|
||||||
|
image_id: int,
|
||||||
|
alias_string: str,
|
||||||
|
alias_category: str,
|
||||||
|
canonical_tag_id: int,
|
||||||
|
) -> None:
|
||||||
|
await self.aliases.create(
|
||||||
|
alias_string, alias_category, canonical_tag_id
|
||||||
|
)
|
||||||
|
await self.accept(image_id, canonical_tag_id)
|
||||||
|
|
||||||
async def dismiss(self, image_id: int, tag_id: int) -> None:
|
async def dismiss(self, image_id: int, tag_id: int) -> None:
|
||||||
stmt = insert(TagSuggestionRejection).values(
|
stmt = insert(TagSuggestionRejection).values(
|
||||||
image_record_id=image_id, tag_id=tag_id
|
image_record_id=image_id, tag_id=tag_id
|
||||||
|
|||||||
@@ -13,20 +13,11 @@ exact CCIP difference metric/threshold gets validated against the model during
|
|||||||
the hands-on eval. numpy is imported lazily (API worker has it via pgvector).
|
the hands-on eval. numpy is imported lazily (API worker has it via pgvector).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy import exists, func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ...models import (
|
from ...models import ImageRegion, MLSettings, Tag, TagKind
|
||||||
CcipPrototypeState,
|
|
||||||
CharacterPrototype,
|
|
||||||
ImageRegion,
|
|
||||||
MLSettings,
|
|
||||||
Tag,
|
|
||||||
TagKind,
|
|
||||||
TagPositiveConfirmation,
|
|
||||||
)
|
|
||||||
from ...models.tag import image_tag
|
from ...models.tag import image_tag
|
||||||
from .training_data import _AUTO_SOURCES
|
|
||||||
|
|
||||||
# Cosine-similarity floor to call a figure the same character. The live setting
|
# Cosine-similarity floor to call a figure the same character. The live setting
|
||||||
# (ml_settings.ccip_match_threshold) drives it; this is only the fallback when no
|
# (ml_settings.ccip_match_threshold) drives it; this is only the fallback when no
|
||||||
@@ -113,17 +104,6 @@ async def _ref_signature(session: AsyncSession) -> tuple:
|
|||||||
return (n_tags, n_regs, max_id, n_hygiene)
|
return (n_tags, n_regs, max_id, n_hygiene)
|
||||||
|
|
||||||
|
|
||||||
def _positive_char_tag():
|
|
||||||
"""Condition on the joined character image_tag: HUMAN-applied or operator-
|
|
||||||
confirmed — NOT an unconfirmed auto-apply. Keeps an auto-tagged character from
|
|
||||||
self-seeding CCIP references, so a ccip_auto misfire can't reinforce itself
|
|
||||||
(milestone 139) — mirrors the head-training positive exclusion."""
|
|
||||||
return image_tag.c.source.not_in(_AUTO_SOURCES) | exists().where(
|
|
||||||
TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
|
|
||||||
TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
async def character_references(session: AsyncSession) -> dict[int, list]:
|
async def character_references(session: AsyncSession) -> dict[int, list]:
|
||||||
"""Per character-tag CCIP reference vectors: figure/face-region CCIP
|
"""Per character-tag CCIP reference vectors: figure/face-region CCIP
|
||||||
embeddings on UNAMBIGUOUS (single-character) images carrying that tag.
|
embeddings on UNAMBIGUOUS (single-character) images carrying that tag.
|
||||||
@@ -141,7 +121,6 @@ async def character_references(session: AsyncSession) -> dict[int, list]:
|
|||||||
)
|
)
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||||
.where(Tag.kind == TagKind.character)
|
.where(Tag.kind == TagKind.character)
|
||||||
.where(_positive_char_tag())
|
|
||||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
||||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||||
.where(ImageRegion.image_record_id.in_(_single_character_images()))
|
.where(ImageRegion.image_record_id.in_(_single_character_images()))
|
||||||
@@ -169,60 +148,6 @@ async def _tag_names(session: AsyncSession, tag_ids: list[int]) -> dict[int, str
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# Per-character normalized prototype matrices, cached per process and refreshed
|
|
||||||
# INCREMENTALLY: only characters whose ccip_prototype_state.updated_at advanced
|
|
||||||
# are reloaded. This replaces the request-path rebuild of the ENTIRE reference
|
|
||||||
# blob (the ~4s stall, #1317) — the prototypes are precomputed off the request
|
|
||||||
# path by services.ml.character_prototypes (a beat + after each retrain).
|
|
||||||
_PROTO_CACHE: dict = {"mats": {}, "ver": {}}
|
|
||||||
|
|
||||||
|
|
||||||
async def _load_prototypes(session: AsyncSession) -> dict:
|
|
||||||
"""{tag_id: (P, D) L2-normalized prototype matrix} from character_prototype,
|
|
||||||
served from the in-process cache and reloading ONLY the characters whose
|
|
||||||
updated_at changed. Empty dict when the store isn't populated yet (cold start
|
|
||||||
→ match_image falls back to the legacy on-the-fly reference build)."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
versions = dict(
|
|
||||||
(
|
|
||||||
await session.execute(
|
|
||||||
select(CcipPrototypeState.tag_id, CcipPrototypeState.updated_at)
|
|
||||||
)
|
|
||||||
).all()
|
|
||||||
)
|
|
||||||
mats = _PROTO_CACHE["mats"]
|
|
||||||
ver = _PROTO_CACHE["ver"]
|
|
||||||
# Forget characters that no longer have prototypes.
|
|
||||||
for tag_id in [t for t in mats if t not in versions]:
|
|
||||||
mats.pop(tag_id, None)
|
|
||||||
ver.pop(tag_id, None)
|
|
||||||
# Reload only the characters whose prototypes changed since we cached them.
|
|
||||||
stale = [t for t, u in versions.items() if ver.get(t) != u]
|
|
||||||
if stale:
|
|
||||||
rows = (
|
|
||||||
await session.execute(
|
|
||||||
select(
|
|
||||||
CharacterPrototype.tag_id, CharacterPrototype.ccip_embedding
|
|
||||||
).where(CharacterPrototype.tag_id.in_(stale))
|
|
||||||
)
|
|
||||||
).all()
|
|
||||||
by_tag: dict[int, list] = {}
|
|
||||||
for tag_id, vec in rows:
|
|
||||||
by_tag.setdefault(tag_id, []).append(
|
|
||||||
np.asarray(vec, dtype=np.float32)
|
|
||||||
)
|
|
||||||
for tag_id in stale:
|
|
||||||
vecs = by_tag.get(tag_id)
|
|
||||||
if vecs:
|
|
||||||
mats[tag_id] = _l2norm(np.vstack(vecs), np)
|
|
||||||
ver[tag_id] = versions[tag_id]
|
|
||||||
else:
|
|
||||||
mats.pop(tag_id, None)
|
|
||||||
ver.pop(tag_id, None)
|
|
||||||
return mats
|
|
||||||
|
|
||||||
|
|
||||||
async def match_image(
|
async def match_image(
|
||||||
session: AsyncSession, image_id: int, threshold: float | None = None
|
session: AsyncSession, image_id: int, threshold: float | None = None
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
@@ -236,30 +161,18 @@ async def match_image(
|
|||||||
if threshold is None:
|
if threshold is None:
|
||||||
threshold = await _settings_threshold(session)
|
threshold = await _settings_threshold(session)
|
||||||
|
|
||||||
# Keep each figure region's bbox alongside its vector so a match can point at
|
qvecs = (
|
||||||
# the figure that matched (#1206 grounding), not just the score.
|
|
||||||
fig_rows = (
|
|
||||||
await session.execute(
|
await session.execute(
|
||||||
select(
|
select(ImageRegion.ccip_embedding).where(
|
||||||
ImageRegion.ccip_embedding,
|
|
||||||
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
|
|
||||||
ImageRegion.kind, ImageRegion.detector_version,
|
|
||||||
).where(
|
|
||||||
ImageRegion.image_record_id == image_id,
|
ImageRegion.image_record_id == image_id,
|
||||||
ImageRegion.kind.in_(_FIGURE_KINDS),
|
ImageRegion.kind.in_(_FIGURE_KINDS),
|
||||||
ImageRegion.ccip_embedding.is_not(None),
|
ImageRegion.ccip_embedding.is_not(None),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
).all()
|
).scalars().all()
|
||||||
if not fig_rows:
|
if not qvecs:
|
||||||
return []
|
return []
|
||||||
# Prefer the precomputed prototype store (fast, incremental). On a cold start
|
refs = await character_references(session)
|
||||||
# (store not yet populated post-deploy) fall back to the legacy on-the-fly
|
|
||||||
# reference build so character suggestions work immediately — the background
|
|
||||||
# refresh populates the store within ~15 min, after which this path is used
|
|
||||||
# and the per-accept ~4s rebuild is gone (#1317).
|
|
||||||
protos = await _load_prototypes(session)
|
|
||||||
refs = protos if protos else await character_references(session)
|
|
||||||
if not refs:
|
if not refs:
|
||||||
return []
|
return []
|
||||||
applied = set(
|
applied = set(
|
||||||
@@ -273,25 +186,13 @@ async def match_image(
|
|||||||
)
|
)
|
||||||
names = await _tag_names(session, [t for t in refs if t not in applied])
|
names = await _tag_names(session, [t for t in refs if t not in applied])
|
||||||
|
|
||||||
qvecs = [r[0] for r in fig_rows]
|
|
||||||
fig_meta = [
|
|
||||||
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
|
|
||||||
for _v, rx, ry, rw, rh, kind, detector in fig_rows
|
|
||||||
]
|
|
||||||
Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np)
|
Q = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np)
|
||||||
out = []
|
out = []
|
||||||
for tag_id, vecs in refs.items():
|
for tag_id, vecs in refs.items():
|
||||||
if tag_id in applied:
|
if tag_id in applied:
|
||||||
continue
|
continue
|
||||||
# Prototype matrices are already L2-normalized; legacy refs are raw
|
R = _l2norm(np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np)
|
||||||
# vector lists that still need stacking + normalizing.
|
best = float((Q @ R.T).max()) # best (query figure, reference) cosine
|
||||||
R = vecs if protos else _l2norm(
|
|
||||||
np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
|
|
||||||
)
|
|
||||||
sims = Q @ R.T # (n_query_figures, n_references)
|
|
||||||
per_figure = sims.max(axis=1) # best reference cosine per figure
|
|
||||||
best_figure = int(per_figure.argmax())
|
|
||||||
best = float(per_figure[best_figure])
|
|
||||||
if best >= threshold:
|
if best >= threshold:
|
||||||
out.append({
|
out.append({
|
||||||
"tag_id": tag_id,
|
"tag_id": tag_id,
|
||||||
@@ -299,8 +200,6 @@ async def match_image(
|
|||||||
"category": "character",
|
"category": "character",
|
||||||
"score": round(best, 4),
|
"score": round(best, 4),
|
||||||
"source": "ccip",
|
"source": "ccip",
|
||||||
# the figure region that matched → grounds the character tag.
|
|
||||||
"grounding": fig_meta[best_figure],
|
|
||||||
})
|
})
|
||||||
out.sort(key=lambda d: d["score"], reverse=True)
|
out.sort(key=lambda d: d["score"], reverse=True)
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -1,262 +0,0 @@
|
|||||||
"""Precomputed CCIP character prototypes — incremental builder (#1317, m138).
|
|
||||||
|
|
||||||
Turns the per-character CCIP reference set into a precomputed artifact so the
|
|
||||||
matcher never rebuilds it on the request path. Sync (runs in the celery ml
|
|
||||||
worker via tasks.ml.refresh_character_prototypes); the async matcher only READS
|
|
||||||
the character_prototype table.
|
|
||||||
|
|
||||||
`refresh_character_prototypes`:
|
|
||||||
1. Cheap GLOBAL gate — a few COUNTs (`_global_signature`). Unchanged + not
|
|
||||||
`full` → no-op (nothing that affects references changed since last refresh).
|
|
||||||
2. Per-character fingerprint diff (one GROUP BY) → rebuild ONLY the characters
|
|
||||||
whose references changed (or are new); drop characters that lost all refs.
|
|
||||||
Each rebuild loads just ONE character's reference vectors, caps them to
|
|
||||||
MLSettings.ccip_prototype_cap, and replaces that character's prototype rows — so
|
|
||||||
cost scales with WHAT changed, not with the library size.
|
|
||||||
|
|
||||||
The reference PREDICATE (single-character, non-hygiene, figure CCIP) is imported
|
|
||||||
from ccip so the prototypes match exactly what the legacy matcher selected.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import random
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
|
|
||||||
from sqlalchemy import delete, func, select
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from ...models import (
|
|
||||||
CcipPrototypeState,
|
|
||||||
CharacterPrototype,
|
|
||||||
ImageRegion,
|
|
||||||
MLSettings,
|
|
||||||
Tag,
|
|
||||||
TagKind,
|
|
||||||
TagPositiveConfirmation,
|
|
||||||
)
|
|
||||||
from ...models.tag import image_tag
|
|
||||||
from .ccip import (
|
|
||||||
_FIGURE_KINDS,
|
|
||||||
_hygiene_tagged_images,
|
|
||||||
_l2norm,
|
|
||||||
_positive_char_tag,
|
|
||||||
_single_character_images,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Deterministic per-tag capping so a rebuild of an UNCHANGED reference set
|
|
||||||
# resamples identically (stable prototypes, no churn between refreshes).
|
|
||||||
_SAMPLE_SEED = 1317
|
|
||||||
|
|
||||||
|
|
||||||
def _global_signature(session: Session) -> str:
|
|
||||||
"""Cheap 'could any references have changed' gate: character-tag count,
|
|
||||||
figure-CCIP region count + max id, hygiene-tag count. A few COUNTs — the same
|
|
||||||
quantities the legacy per-request signature used, now computed once per
|
|
||||||
refresh instead of on every /suggestions call."""
|
|
||||||
n_tags = session.execute(
|
|
||||||
select(func.count())
|
|
||||||
.select_from(image_tag)
|
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
|
||||||
.where(Tag.kind == TagKind.character)
|
|
||||||
).scalar_one()
|
|
||||||
n_regs, max_id = session.execute(
|
|
||||||
select(func.count(), func.max(ImageRegion.id)).where(
|
|
||||||
ImageRegion.kind.in_(_FIGURE_KINDS),
|
|
||||||
ImageRegion.ccip_embedding.is_not(None),
|
|
||||||
)
|
|
||||||
).one()
|
|
||||||
n_hygiene = session.execute(
|
|
||||||
select(func.count())
|
|
||||||
.select_from(image_tag)
|
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
|
||||||
.where(Tag.is_system.is_(True))
|
|
||||||
).scalar_one()
|
|
||||||
# Character confirmations affect the reference set now that auto-tags only
|
|
||||||
# seed references once confirmed (milestone 139) — so a confirm must trip the
|
|
||||||
# gate, or the per-character diff (which reflects it) never runs.
|
|
||||||
n_conf = session.execute(
|
|
||||||
select(func.count())
|
|
||||||
.select_from(TagPositiveConfirmation)
|
|
||||||
.join(Tag, Tag.id == TagPositiveConfirmation.tag_id)
|
|
||||||
.where(Tag.kind == TagKind.character)
|
|
||||||
).scalar_one()
|
|
||||||
return f"{n_tags}:{n_regs}:{max_id or 0}:{n_hygiene}:{n_conf}"
|
|
||||||
|
|
||||||
|
|
||||||
def _current_fingerprints(session: Session) -> dict[int, str]:
|
|
||||||
"""Per-character (reference count, max reference region id) over the SAME
|
|
||||||
predicate the matcher's references use. One GROUP BY → the change detector:
|
|
||||||
a character whose fingerprint moved (gained/lost a reference) needs a
|
|
||||||
rebuild; everyone else is left untouched."""
|
|
||||||
rows = session.execute(
|
|
||||||
select(
|
|
||||||
image_tag.c.tag_id,
|
|
||||||
func.count(ImageRegion.id),
|
|
||||||
func.max(ImageRegion.id),
|
|
||||||
)
|
|
||||||
.select_from(ImageRegion)
|
|
||||||
.join(
|
|
||||||
image_tag,
|
|
||||||
image_tag.c.image_record_id == ImageRegion.image_record_id,
|
|
||||||
)
|
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
|
||||||
.where(Tag.kind == TagKind.character)
|
|
||||||
.where(_positive_char_tag())
|
|
||||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
|
||||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
|
||||||
.where(ImageRegion.image_record_id.in_(_single_character_images()))
|
|
||||||
.where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
|
|
||||||
.group_by(image_tag.c.tag_id)
|
|
||||||
).all()
|
|
||||||
return {tag_id: f"{cnt}:{mx}" for tag_id, cnt, mx in rows}
|
|
||||||
|
|
||||||
|
|
||||||
def _rebuild_one(session: Session, tag_id: int, cap: int) -> int:
|
|
||||||
"""Replace ONE character's prototype rows from its current references, capped
|
|
||||||
to `cap`. Loads only this character's vectors (bounded by its popularity)."""
|
|
||||||
rows = session.execute(
|
|
||||||
select(ImageRegion.id, ImageRegion.ccip_embedding)
|
|
||||||
.select_from(ImageRegion)
|
|
||||||
.join(
|
|
||||||
image_tag,
|
|
||||||
image_tag.c.image_record_id == ImageRegion.image_record_id,
|
|
||||||
)
|
|
||||||
.where(image_tag.c.tag_id == tag_id)
|
|
||||||
.where(_positive_char_tag())
|
|
||||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
|
||||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
|
||||||
.where(ImageRegion.image_record_id.in_(_single_character_images()))
|
|
||||||
.where(ImageRegion.image_record_id.not_in(_hygiene_tagged_images()))
|
|
||||||
).all()
|
|
||||||
# Cap for bounded MATCH cost. A random sample (not most-recent) keeps the
|
|
||||||
# prototypes representative of the whole reference set; a fixed per-tag seed
|
|
||||||
# makes an unchanged set resample identically.
|
|
||||||
if cap > 0 and len(rows) > cap:
|
|
||||||
rows = random.Random(f"{_SAMPLE_SEED}:{tag_id}").sample(rows, cap)
|
|
||||||
session.execute(
|
|
||||||
delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
|
|
||||||
)
|
|
||||||
for region_id, vec in rows:
|
|
||||||
session.add(
|
|
||||||
CharacterPrototype(
|
|
||||||
tag_id=tag_id, region_id=region_id, ccip_embedding=vec
|
|
||||||
)
|
|
||||||
)
|
|
||||||
return len(rows)
|
|
||||||
|
|
||||||
|
|
||||||
def refresh_character_prototypes(
|
|
||||||
session: Session, *, full: bool = False
|
|
||||||
) -> dict[str, int | bool]:
|
|
||||||
"""Incrementally refresh the prototype store. `full=True` rebuilds every
|
|
||||||
character regardless of the gate/fingerprints (nightly reconcile). Returns
|
|
||||||
{skipped, rebuilt, removed}; commits."""
|
|
||||||
settings = MLSettings.load_sync(session)
|
|
||||||
sig = _global_signature(session)
|
|
||||||
if not full and settings.ccip_ref_signature == sig:
|
|
||||||
return {"skipped": True, "rebuilt": 0, "removed": 0}
|
|
||||||
|
|
||||||
cap = settings.ccip_prototype_cap
|
|
||||||
current = _current_fingerprints(session)
|
|
||||||
stored = dict(
|
|
||||||
session.execute(
|
|
||||||
select(CcipPrototypeState.tag_id, CcipPrototypeState.fingerprint)
|
|
||||||
).all()
|
|
||||||
)
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
rebuilt = 0
|
|
||||||
for tag_id, fp in current.items():
|
|
||||||
if full or stored.get(tag_id) != fp:
|
|
||||||
_rebuild_one(session, tag_id, cap)
|
|
||||||
state = session.get(CcipPrototypeState, tag_id)
|
|
||||||
if state is None:
|
|
||||||
state = CcipPrototypeState(tag_id=tag_id)
|
|
||||||
session.add(state)
|
|
||||||
state.fingerprint = fp
|
|
||||||
state.updated_at = now
|
|
||||||
rebuilt += 1
|
|
||||||
# Characters that lost every reference (refs removed / re-kinded / image now
|
|
||||||
# multi-character) → drop their prototypes + state so they stop matching.
|
|
||||||
removed = 0
|
|
||||||
for tag_id in set(stored) - set(current):
|
|
||||||
session.execute(
|
|
||||||
delete(CharacterPrototype).where(CharacterPrototype.tag_id == tag_id)
|
|
||||||
)
|
|
||||||
session.execute(
|
|
||||||
delete(CcipPrototypeState).where(CcipPrototypeState.tag_id == tag_id)
|
|
||||||
)
|
|
||||||
removed += 1
|
|
||||||
|
|
||||||
settings.ccip_ref_signature = sig
|
|
||||||
session.commit()
|
|
||||||
return {"skipped": False, "rebuilt": rebuilt, "removed": removed}
|
|
||||||
|
|
||||||
|
|
||||||
def retract_auto_applied_ccip(session: Session) -> int:
|
|
||||||
"""Soft auto-apply for CCIP character tags (milestone 139): re-score every
|
|
||||||
standing source='ccip_auto' character tag against that character's prototypes
|
|
||||||
and REMOVE the ones whose best figure match is now BELOW
|
|
||||||
ccip_auto_apply_threshold. Skips operator-confirmed tags. SILENT — a low score
|
|
||||||
isn't proof the tag was wrong (that's reserved for an operator removal). No-op
|
|
||||||
unless ccip_auto_apply_enabled. A character with no prototypes yet, or an image
|
|
||||||
with no figure vectors, is left alone (can't judge → keep). Returns
|
|
||||||
n_retracted."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
settings = MLSettings.load_sync(session)
|
|
||||||
if not settings.ccip_auto_apply_enabled:
|
|
||||||
return 0
|
|
||||||
thr = float(settings.ccip_auto_apply_threshold)
|
|
||||||
pairs = session.execute(
|
|
||||||
select(image_tag.c.image_record_id, image_tag.c.tag_id)
|
|
||||||
.where(image_tag.c.source == "ccip_auto")
|
|
||||||
).all()
|
|
||||||
if not pairs:
|
|
||||||
return 0
|
|
||||||
confirmed = {
|
|
||||||
(iid, tid) for iid, tid in session.execute(
|
|
||||||
select(
|
|
||||||
TagPositiveConfirmation.image_record_id,
|
|
||||||
TagPositiveConfirmation.tag_id,
|
|
||||||
)
|
|
||||||
).all()
|
|
||||||
}
|
|
||||||
# Each involved character's normalized prototype matrix, loaded once.
|
|
||||||
proto: dict[int, object] = {}
|
|
||||||
for tid in {tid for _iid, tid in pairs}:
|
|
||||||
vecs = [
|
|
||||||
v for (v,) in session.execute(
|
|
||||||
select(CharacterPrototype.ccip_embedding)
|
|
||||||
.where(CharacterPrototype.tag_id == tid)
|
|
||||||
)
|
|
||||||
]
|
|
||||||
if vecs:
|
|
||||||
proto[tid] = _l2norm(
|
|
||||||
np.vstack([np.asarray(v, dtype=np.float32) for v in vecs]), np
|
|
||||||
)
|
|
||||||
retracted = 0
|
|
||||||
for iid, tid in pairs:
|
|
||||||
if (iid, tid) in confirmed or tid not in proto:
|
|
||||||
continue # confirmed / no prototypes
|
|
||||||
qvecs = [
|
|
||||||
v for (v,) in session.execute(
|
|
||||||
select(ImageRegion.ccip_embedding)
|
|
||||||
.where(ImageRegion.image_record_id == iid)
|
|
||||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
|
||||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
|
||||||
)
|
|
||||||
]
|
|
||||||
if not qvecs:
|
|
||||||
continue # no figure vectors → keep
|
|
||||||
Q = _l2norm(
|
|
||||||
np.vstack([np.asarray(v, dtype=np.float32) for v in qvecs]), np
|
|
||||||
)
|
|
||||||
if float((Q @ proto[tid].T).max()) < thr:
|
|
||||||
session.execute(
|
|
||||||
image_tag.delete()
|
|
||||||
.where(image_tag.c.image_record_id == iid)
|
|
||||||
.where(image_tag.c.tag_id == tid)
|
|
||||||
.where(image_tag.c.source == "ccip_auto")
|
|
||||||
)
|
|
||||||
retracted += 1
|
|
||||||
session.commit()
|
|
||||||
return retracted
|
|
||||||
@@ -22,8 +22,7 @@ import logging
|
|||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import delete, exists, func, select
|
from sqlalchemy import delete, func, select
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
@@ -33,17 +32,13 @@ from ...models import (
|
|||||||
ImageRecord,
|
ImageRecord,
|
||||||
ImageRegion,
|
ImageRegion,
|
||||||
MLSettings,
|
MLSettings,
|
||||||
PresentationReview,
|
|
||||||
Tag,
|
Tag,
|
||||||
TagHead,
|
TagHead,
|
||||||
TagKind,
|
TagKind,
|
||||||
TagPositiveConfirmation,
|
|
||||||
TagSuggestionRejection,
|
TagSuggestionRejection,
|
||||||
)
|
)
|
||||||
from ...models.tag import CHROME_SYSTEM_TAGS, PROCESS_SYSTEM_TAGS, image_tag
|
from ...models.tag import image_tag
|
||||||
from .training_data import (
|
from .training_data import (
|
||||||
_AUTO_SOURCES,
|
|
||||||
_applied_or_rejected,
|
|
||||||
_auto_apply_point,
|
_auto_apply_point,
|
||||||
_hygiene_excluded_ids,
|
_hygiene_excluded_ids,
|
||||||
_ids_with_tag,
|
_ids_with_tag,
|
||||||
@@ -63,14 +58,6 @@ MIN_POSITIVES_FLOOR = 8 # hard floor; settings.head_min_positives can raise
|
|||||||
_UNLABELED_POOL = 4000
|
_UNLABELED_POOL = 4000
|
||||||
_EXAMPLES_MIN = 8 # need at least this many embedded +/- to fit a head
|
_EXAMPLES_MIN = 8 # need at least this many embedded +/- to fit a head
|
||||||
|
|
||||||
# Auto-apply / match confidence operating range. Every graduated auto-apply or
|
|
||||||
# CCIP-match threshold the operator can set lives in this band, and the head
|
|
||||||
# precision target is clamped to it: below 0.5 "auto-apply" is meaningless, and
|
|
||||||
# 1.0 is unachievable so 0.999 is the ceiling. One source shared by the service
|
|
||||||
# clamp (_normalize_params) and the API validator (ml_admin._validate).
|
|
||||||
AUTO_APPLY_THRESHOLD_MIN = 0.5
|
|
||||||
AUTO_APPLY_THRESHOLD_MAX = 0.999
|
|
||||||
|
|
||||||
# Only these tag kinds get heads (the surfaced suggestion categories).
|
# Only these tag kinds get heads (the surfaced suggestion categories).
|
||||||
_HEAD_KINDS = (TagKind.general, TagKind.character)
|
_HEAD_KINDS = (TagKind.general, TagKind.character)
|
||||||
# tag.kind -> the suggestion category the rail groups under.
|
# tag.kind -> the suggestion category the rail groups under.
|
||||||
@@ -88,38 +75,6 @@ _CATEGORY = {TagKind.general: "general", TagKind.character: "character"}
|
|||||||
_SYSTEM_TAG_SUGGEST_FLOOR = 0.65
|
_SYSTEM_TAG_SUGGEST_FLOOR = 0.65
|
||||||
|
|
||||||
|
|
||||||
def _sigmoid(z, np):
|
|
||||||
"""Logistic sigmoid 1/(1+e^-z): the head score→probability transform. One home
|
|
||||||
for what was inlined at every scoring site (suggest, both sweeps, retract)."""
|
|
||||||
return 1.0 / (1.0 + np.exp(-z))
|
|
||||||
|
|
||||||
|
|
||||||
def _conflict_scores(Xn, Wc, bc, np):
|
|
||||||
"""The presentation conflict signal (#141): per row, the MAX content-head
|
|
||||||
probability and WHICH head produced it. Shared by the system-tag sweep's guard-2
|
|
||||||
and the soft-wip audit — both ask "does this ALSO look like real content?"."""
|
|
||||||
cprobs = _sigmoid(Xn @ Wc.T + bc, np)
|
|
||||||
return cprobs.max(axis=1), cprobs.argmax(axis=1)
|
|
||||||
|
|
||||||
|
|
||||||
def _insert_presentation_review(
|
|
||||||
session, *, image_record_id, tag_id, conflict_tag_id, conflict_score, mode,
|
|
||||||
):
|
|
||||||
"""Single-source the ring-loud PresentationReview row shape so the two writers
|
|
||||||
(system-tag sweep guard-2 + soft-wip audit) can't drift on columns or `mode` —
|
|
||||||
they share the (image_record_id, tag_id) composite PK, so a divergent `mode`
|
|
||||||
would be a silent first-writer-wins bug."""
|
|
||||||
session.execute(
|
|
||||||
pg_insert(PresentationReview)
|
|
||||||
.values(
|
|
||||||
image_record_id=image_record_id, tag_id=tag_id,
|
|
||||||
conflict_tag_id=conflict_tag_id, conflict_score=conflict_score,
|
|
||||||
mode=mode,
|
|
||||||
)
|
|
||||||
.on_conflict_do_nothing()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class HeadTrainingAlreadyRunning(Exception):
|
class HeadTrainingAlreadyRunning(Exception):
|
||||||
"""Raised by start_head_training_run when a run is already in flight."""
|
"""Raised by start_head_training_run when a run is already in flight."""
|
||||||
|
|
||||||
@@ -145,7 +100,9 @@ def start_head_training_run(session: Session, params: dict[str, Any]) -> int:
|
|||||||
|
|
||||||
|
|
||||||
def _settings(session: Session) -> MLSettings:
|
def _settings(session: Session) -> MLSettings:
|
||||||
return MLSettings.load_sync(session)
|
return session.execute(
|
||||||
|
select(MLSettings).where(MLSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
def _normalize_params(session: Session, params: dict[str, Any] | None) -> dict[str, Any]:
|
def _normalize_params(session: Session, params: dict[str, Any] | None) -> dict[str, Any]:
|
||||||
@@ -164,7 +121,7 @@ def _normalize_params(session: Session, params: dict[str, Any] | None) -> dict[s
|
|||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
cv_folds = DEFAULT_CV_FOLDS
|
cv_folds = DEFAULT_CV_FOLDS
|
||||||
try:
|
try:
|
||||||
precision_target = min(max(float(params.get("precision_target", s.head_auto_apply_precision)), AUTO_APPLY_THRESHOLD_MIN), AUTO_APPLY_THRESHOLD_MAX)
|
precision_target = min(max(float(params.get("precision_target", s.head_auto_apply_precision)), 0.5), 0.999)
|
||||||
except (TypeError, ValueError):
|
except (TypeError, ValueError):
|
||||||
precision_target = s.head_auto_apply_precision
|
precision_target = s.head_auto_apply_precision
|
||||||
return {
|
return {
|
||||||
@@ -180,132 +137,37 @@ def _embedder_version(session: Session) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def _eligible_tag_ids(session: Session, min_pos: int) -> list[int]:
|
def _eligible_tag_ids(session: Session, min_pos: int) -> list[int]:
|
||||||
"""Concept tags (general/character) with >= min_pos POSITIVE images — the set
|
"""Concept tags (general/character) with >= min_pos labelled images — the
|
||||||
that gets a head. Counts human-applied + operator-confirmed tags only;
|
set that gets a head. Counts all sources; source-aware filtering (#1133) is
|
||||||
unconfirmed auto-applied predictions do NOT count toward eligibility (they
|
a separate, optional refinement."""
|
||||||
don't train the head — milestone 139), so a concept can't graduate on its own
|
|
||||||
guesses."""
|
|
||||||
confirmed = exists().where(
|
|
||||||
TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
|
|
||||||
TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
|
|
||||||
)
|
|
||||||
rows = session.execute(
|
rows = session.execute(
|
||||||
select(Tag.id)
|
select(Tag.id)
|
||||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||||
.where(Tag.kind.in_(_HEAD_KINDS))
|
.where(Tag.kind.in_(_HEAD_KINDS))
|
||||||
.where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed)
|
|
||||||
.group_by(Tag.id)
|
.group_by(Tag.id)
|
||||||
.having(func.count(image_tag.c.image_record_id) >= min_pos)
|
.having(func.count(image_tag.c.image_record_id) >= min_pos)
|
||||||
).all()
|
).all()
|
||||||
return [r[0] for r in rows]
|
return [r[0] for r in rows]
|
||||||
|
|
||||||
|
|
||||||
def _head_fingerprints(session: Session, tag_ids: list[int]) -> dict[int, str]:
|
|
||||||
"""Per-tag training-data fingerprint: (positive count, latest positive
|
|
||||||
created_at) + (rejection count, latest rejected_at). It moves whenever a tag
|
|
||||||
gains/loses a positive or a rejection — the incremental-retrain change
|
|
||||||
detector (#1317 p2). A newly-added positive/rejection always has the latest
|
|
||||||
timestamp, so even a remove-one-add-one (unchanged count) is caught. The
|
|
||||||
sampled-unlabeled negative pool + the hygiene set drift GLOBALLY and are
|
|
||||||
reconciled by the nightly full run, not captured here."""
|
|
||||||
if not tag_ids:
|
|
||||||
return {}
|
|
||||||
pos = session.execute(
|
|
||||||
select(
|
|
||||||
image_tag.c.tag_id,
|
|
||||||
func.count(image_tag.c.image_record_id),
|
|
||||||
func.max(image_tag.c.created_at),
|
|
||||||
)
|
|
||||||
.where(image_tag.c.tag_id.in_(tag_ids))
|
|
||||||
.group_by(image_tag.c.tag_id)
|
|
||||||
).all()
|
|
||||||
pos_map = {t: (c, m) for t, c, m in pos}
|
|
||||||
rej = session.execute(
|
|
||||||
select(
|
|
||||||
TagSuggestionRejection.tag_id,
|
|
||||||
func.count(),
|
|
||||||
func.max(TagSuggestionRejection.rejected_at),
|
|
||||||
)
|
|
||||||
.where(TagSuggestionRejection.tag_id.in_(tag_ids))
|
|
||||||
.group_by(TagSuggestionRejection.tag_id)
|
|
||||||
).all()
|
|
||||||
rej_map = {t: (c, m) for t, c, m in rej}
|
|
||||||
# Confirmations promote an auto-applied tag to a positive (milestone 139), so
|
|
||||||
# a confirm must move the fingerprint too — else a manual Retrain right after
|
|
||||||
# confirming wouldn't fold the tag in (the nightly full run would).
|
|
||||||
conf = session.execute(
|
|
||||||
select(TagPositiveConfirmation.tag_id, func.count())
|
|
||||||
.where(TagPositiveConfirmation.tag_id.in_(tag_ids))
|
|
||||||
.group_by(TagPositiveConfirmation.tag_id)
|
|
||||||
).all()
|
|
||||||
conf_map = dict(conf)
|
|
||||||
out = {}
|
|
||||||
for t in tag_ids:
|
|
||||||
pc, pm = pos_map.get(t, (0, None))
|
|
||||||
rc, rm = rej_map.get(t, (0, None))
|
|
||||||
out[t] = f"{pc}:{pm}:{rc}:{rm}:{conf_map.get(t, 0)}"
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _heads_needing_retrain(
|
|
||||||
session: Session, eligible: list[int], embedding_version: str,
|
|
||||||
fps: dict[int, str], full: bool,
|
|
||||||
) -> list[int]:
|
|
||||||
"""The eligible tag_ids to (re)fit: no head yet, a head trained in a DIFFERENT
|
|
||||||
embedding space (a model swap), or a changed training-data fingerprint.
|
|
||||||
full=True forces every eligible tag. sklearn-free (train_head itself needs
|
|
||||||
scikit-learn) so the incremental decision is unit-testable on its own."""
|
|
||||||
if full:
|
|
||||||
return list(eligible)
|
|
||||||
existing = {
|
|
||||||
tag_id: (fp, ev)
|
|
||||||
for tag_id, fp, ev in session.execute(
|
|
||||||
select(
|
|
||||||
TagHead.tag_id, TagHead.train_fingerprint,
|
|
||||||
TagHead.embedding_version,
|
|
||||||
)
|
|
||||||
).all()
|
|
||||||
}
|
|
||||||
out = []
|
|
||||||
for tag_id in eligible:
|
|
||||||
prev = existing.get(tag_id)
|
|
||||||
if (
|
|
||||||
prev is None
|
|
||||||
or prev[1] != embedding_version
|
|
||||||
or prev[0] != fps.get(tag_id)
|
|
||||||
):
|
|
||||||
out.append(tag_id)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def train_all_heads(
|
def train_all_heads(
|
||||||
session: Session, params: dict[str, Any], run: HeadTrainingRun | None = None
|
session: Session, params: dict[str, Any], run: HeadTrainingRun | None = None
|
||||||
) -> dict[str, int]:
|
) -> dict[str, int]:
|
||||||
"""(Re)train eligible concept heads, INCREMENTALLY by default (#1317 p2):
|
"""(Re)train a head for every eligible concept; prune heads whose tag is no
|
||||||
refit only the tags whose training data changed since last fit, so a manual
|
longer eligible. Commits per head so a SIGKILL leaves trained heads durable
|
||||||
Retrain click is fast. `params["full"]=True` (the nightly run) refits every
|
(training is idempotent). Returns {n_trained, n_skipped}."""
|
||||||
head to reconcile sampled-negative + hygiene drift. Prunes heads whose tag is
|
|
||||||
no longer eligible. Commits per head so a SIGKILL leaves trained heads durable.
|
|
||||||
Returns {n_trained, n_skipped} (n_skipped = unchanged + too-few-examples)."""
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
cfg = _normalize_params(session, params)
|
cfg = _normalize_params(session, params)
|
||||||
embedding_version = _embedder_version(session)
|
embedding_version = _embedder_version(session)
|
||||||
full = bool((params or {}).get("full"))
|
|
||||||
eligible = _eligible_tag_ids(session, cfg["min_positives"])
|
eligible = _eligible_tag_ids(session, cfg["min_positives"])
|
||||||
eligible_set = set(eligible)
|
eligible_set = set(eligible)
|
||||||
# Computed once per run, not per head — the hygiene set is identical for
|
# Computed once per run, not per head — the hygiene set is identical for
|
||||||
# every non-system concept.
|
# every non-system concept.
|
||||||
hygiene = _hygiene_excluded_ids(session)
|
hygiene = _hygiene_excluded_ids(session)
|
||||||
fps = _head_fingerprints(session, eligible)
|
|
||||||
to_train = set(
|
|
||||||
_heads_needing_retrain(session, eligible, embedding_version, fps, full)
|
|
||||||
)
|
|
||||||
trained = 0
|
trained = 0
|
||||||
failed = 0
|
skipped = 0
|
||||||
for i, tag_id in enumerate(eligible):
|
for i, tag_id in enumerate(eligible):
|
||||||
if tag_id not in to_train:
|
|
||||||
continue
|
|
||||||
try:
|
try:
|
||||||
ok = train_head(
|
ok = train_head(
|
||||||
session, tag_id, embedding_version, cfg, np, hygiene=hygiene
|
session, tag_id, embedding_version, cfg, np, hygiene=hygiene
|
||||||
@@ -313,15 +175,9 @@ def train_all_heads(
|
|||||||
except Exception:
|
except Exception:
|
||||||
log.exception("train_head failed for tag %d", tag_id)
|
log.exception("train_head failed for tag %d", tag_id)
|
||||||
ok = False
|
ok = False
|
||||||
if ok:
|
|
||||||
# Stamp the fingerprint we trained against so an unchanged tag is
|
|
||||||
# skipped on the next incremental run.
|
|
||||||
head = session.get(TagHead, tag_id)
|
|
||||||
if head is not None:
|
|
||||||
head.train_fingerprint = fps.get(tag_id)
|
|
||||||
session.commit()
|
session.commit()
|
||||||
trained += int(ok)
|
trained += int(ok)
|
||||||
failed += int(not ok)
|
skipped += int(not ok)
|
||||||
if run is not None and i % 10 == 0:
|
if run is not None and i % 10 == 0:
|
||||||
run.last_progress_at = datetime.now(UTC)
|
run.last_progress_at = datetime.now(UTC)
|
||||||
session.commit()
|
session.commit()
|
||||||
@@ -332,11 +188,7 @@ def train_all_heads(
|
|||||||
else:
|
else:
|
||||||
session.execute(delete(TagHead))
|
session.execute(delete(TagHead))
|
||||||
session.commit()
|
session.commit()
|
||||||
# n_skipped = unchanged (not attempted) + failed-to-fit (too few examples).
|
return {"n_trained": trained, "n_skipped": skipped}
|
||||||
return {
|
|
||||||
"n_trained": trained,
|
|
||||||
"n_skipped": (len(eligible) - len(to_train)) + failed,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def head_training_ids(
|
def head_training_ids(
|
||||||
@@ -489,70 +341,17 @@ async def _current_heads(session: AsyncSession, embedding_version: str):
|
|||||||
return loaded
|
return loaded
|
||||||
|
|
||||||
|
|
||||||
async def _image_bag(
|
|
||||||
session: AsyncSession, image_id: int, cur_version: str,
|
|
||||||
) -> tuple[list, list[dict | None]]:
|
|
||||||
"""The max-over-bag inputs for one image: the whole-image SigLIP vector (when
|
|
||||||
it's in the current model's space) PLUS every concept-region crop embedded in
|
|
||||||
that space. Returns (bag, bag_meta) as PARALLEL lists — bag_meta[i] is None for
|
|
||||||
the whole-image row, else the region's {bbox, kind, detector} so a surfaced tag
|
|
||||||
can point back at the crop that produced it (#1206 grounding).
|
|
||||||
|
|
||||||
Only current-version embeddings enter the bag: mid model-swap (#1190) an image
|
|
||||||
still carrying an OLD-version whole-image vector is skipped rather than scored
|
|
||||||
by heads trained in a different space; a legacy NULL version is treated as
|
|
||||||
current (those predate per-row stamping). Shared by live scoring (score_image)
|
|
||||||
and on-demand applied-tag grounding (ground_applied_tag, #1206 Step 4)."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
img = await session.get(ImageRecord, image_id)
|
|
||||||
bag: list = []
|
|
||||||
bag_meta: list[dict | None] = []
|
|
||||||
if img is None:
|
|
||||||
return bag, bag_meta
|
|
||||||
if img.siglip_embedding is not None and img.siglip_model_version in (
|
|
||||||
cur_version, None,
|
|
||||||
):
|
|
||||||
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
|
|
||||||
bag_meta.append(None)
|
|
||||||
region_rows = (
|
|
||||||
await session.execute(
|
|
||||||
select(
|
|
||||||
ImageRegion.siglip_embedding,
|
|
||||||
ImageRegion.rx, ImageRegion.ry, ImageRegion.rw, ImageRegion.rh,
|
|
||||||
ImageRegion.kind, ImageRegion.detector_version,
|
|
||||||
)
|
|
||||||
.where(ImageRegion.image_record_id == image_id)
|
|
||||||
.where(ImageRegion.siglip_embedding.is_not(None))
|
|
||||||
.where(ImageRegion.embedding_version == cur_version)
|
|
||||||
)
|
|
||||||
).all()
|
|
||||||
for vec, rx, ry, rw, rh, kind, detector in region_rows:
|
|
||||||
if vec is not None:
|
|
||||||
bag.append(np.asarray(vec, dtype=np.float32))
|
|
||||||
bag_meta.append(
|
|
||||||
{"bbox": [rx, ry, rw, rh], "kind": kind, "detector": detector}
|
|
||||||
)
|
|
||||||
return bag, bag_meta
|
|
||||||
|
|
||||||
|
|
||||||
async def score_image(
|
async def score_image(
|
||||||
session: AsyncSession, image_id: int, threshold_override: float | None = None,
|
session: AsyncSession, image_id: int, threshold_override: float | None = None,
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
"""Suggestions for one image from the trained heads: [{tag_id, name,
|
"""Suggestions for one image from the trained heads: [{tag_id, name,
|
||||||
category, score, above_threshold, grounding}], ranked. A concept is INCLUDED
|
category, score}], ranked. A concept surfaces when its score clears the
|
||||||
when its score clears the head's own suggest_threshold — or, when
|
head's own suggest_threshold — or, when threshold_override is given (the
|
||||||
threshold_override is given (the typed-dropdown "show everything" mode), that
|
typed-dropdown "show everything" mode), that flat floor instead (0 → every
|
||||||
flat floor instead (0 → every head). System-tag heads (wip/banner/editor)
|
head). System-tag heads (wip/banner/editor) instead use a flat
|
||||||
instead use a flat _SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface
|
_SYSTEM_TAG_SUGGEST_FLOOR so their false positives surface for rejection
|
||||||
for rejection (still overridden by threshold_override). Empty if the image has
|
(still overridden by threshold_override). Empty if the image has no
|
||||||
no embedding or no heads exist yet.
|
embedding or no heads exist yet.
|
||||||
|
|
||||||
``above_threshold`` is reported SEPARATELY from inclusion: it's always whether
|
|
||||||
the score cleared the head's NATURAL cut (suggest_threshold, or the system
|
|
||||||
floor), regardless of any override. So the single min=0 fetch returns every
|
|
||||||
head, and the caller can split panel (above_threshold) from dropdown (all)
|
|
||||||
without a second request.
|
|
||||||
|
|
||||||
MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
|
MAX-OVER-BAG: the image is scored as a BAG of embeddings — the whole-image
|
||||||
vector PLUS every concept-region crop the agent embedded (same model
|
vector PLUS every concept-region crop the agent embedded (same model
|
||||||
@@ -562,12 +361,35 @@ async def score_image(
|
|||||||
always in the bag, so this can never score lower than whole-image alone."""
|
always in the bag, so this can never score lower than whole-image alone."""
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
|
img = await session.get(ImageRecord, image_id)
|
||||||
|
if img is None:
|
||||||
|
return []
|
||||||
settings = await _settings_async(session)
|
settings = await _settings_async(session)
|
||||||
cur_version = settings.embedder_model_version
|
cur_version = settings.embedder_model_version
|
||||||
heads = await _current_heads(session, cur_version)
|
heads = await _current_heads(session, cur_version)
|
||||||
if heads["W"] is None:
|
if heads["W"] is None:
|
||||||
return []
|
return []
|
||||||
bag, bag_meta = await _image_bag(session, image_id, cur_version)
|
|
||||||
|
# Only embeddings in the CURRENT model's space enter the bag. Mid model-swap
|
||||||
|
# (#1190), an image still carrying the OLD-version whole-image vector is
|
||||||
|
# skipped rather than scored by heads trained in a different space; a legacy
|
||||||
|
# NULL version is treated as current (those predate per-row stamping).
|
||||||
|
bag = []
|
||||||
|
if img.siglip_embedding is not None and img.siglip_model_version in (
|
||||||
|
cur_version, None,
|
||||||
|
):
|
||||||
|
bag.append(np.asarray(img.siglip_embedding, dtype=np.float32))
|
||||||
|
region_vecs = (
|
||||||
|
await session.execute(
|
||||||
|
select(ImageRegion.siglip_embedding)
|
||||||
|
.where(ImageRegion.image_record_id == image_id)
|
||||||
|
.where(ImageRegion.siglip_embedding.is_not(None))
|
||||||
|
.where(ImageRegion.embedding_version == cur_version)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
for (vec,) in region_vecs:
|
||||||
|
if vec is not None:
|
||||||
|
bag.append(np.asarray(vec, dtype=np.float32))
|
||||||
if not bag:
|
if not bag:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
@@ -576,85 +398,33 @@ async def score_image(
|
|||||||
norms[norms == 0] = 1.0
|
norms[norms == 0] = 1.0
|
||||||
Xn = X / norms
|
Xn = X / norms
|
||||||
Z = Xn @ heads["W"].T + heads["b"] # (B, H)
|
Z = Xn @ heads["W"].T + heads["b"] # (B, H)
|
||||||
probs_bag = _sigmoid(Z, np) # (B, H)
|
probs = (1.0 / (1.0 + np.exp(-Z))).max(axis=0) # (H,) best over the bag
|
||||||
probs = probs_bag.max(axis=0) # (H,) best over the bag
|
|
||||||
# ARGMAX beside the max: WHICH bag row won each head → the region that grounds
|
|
||||||
# the tag (bag_meta[win]); None when the whole-image vector won (#1206).
|
|
||||||
winners = probs_bag.argmax(axis=0) # (H,)
|
|
||||||
out = []
|
out = []
|
||||||
for i, p in enumerate(probs):
|
for i, p in enumerate(probs):
|
||||||
m = heads["meta"][i]
|
if threshold_override is not None:
|
||||||
# The head's NATURAL suggest cut — system tags use the flat floor (see
|
cut = threshold_override
|
||||||
# _SYSTEM_TAG_SUGGEST_FLOOR) so their false positives show up for the
|
elif heads["meta"][i]["is_system"]:
|
||||||
# operator to reject; content heads use their own precision-tuned
|
# System tags surface at the flat floor (see _SYSTEM_TAG_SUGGEST_FLOOR)
|
||||||
# threshold. This is what "above threshold" means (drives the panel).
|
# so their false positives show up for the operator to reject.
|
||||||
natural = (
|
cut = _SYSTEM_TAG_SUGGEST_FLOOR
|
||||||
_SYSTEM_TAG_SUGGEST_FLOOR if m["is_system"] else float(heads["thr"][i])
|
else:
|
||||||
)
|
cut = heads["thr"][i]
|
||||||
# INCLUSION is looser under threshold_override (dropdown show-all,
|
|
||||||
# override=0): every head comes back so a low-confidence concept can still
|
|
||||||
# be typed + picked, each carrying its own above_threshold flag.
|
|
||||||
cut = threshold_override if threshold_override is not None else natural
|
|
||||||
if p >= cut:
|
if p >= cut:
|
||||||
|
m = heads["meta"][i]
|
||||||
out.append({
|
out.append({
|
||||||
"tag_id": m["tag_id"],
|
"tag_id": m["tag_id"],
|
||||||
"name": m["name"],
|
"name": m["name"],
|
||||||
"category": m["category"],
|
"category": m["category"],
|
||||||
"score": float(p),
|
"score": float(p),
|
||||||
"above_threshold": bool(p >= natural),
|
|
||||||
"grounding": bag_meta[int(winners[i])],
|
|
||||||
})
|
})
|
||||||
out.sort(key=lambda d: d["score"], reverse=True)
|
out.sort(key=lambda d: d["score"], reverse=True)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def ground_applied_tag(
|
|
||||||
session: AsyncSession, image_id: int, tag_id: int,
|
|
||||||
) -> tuple[dict | None, bool]:
|
|
||||||
"""On-demand grounding for an ALREADY-APPLIED tag (#1206 Step 4). Applied tags
|
|
||||||
aren't scored live, so recompute the max-over-bag argmax for just this tag's
|
|
||||||
head — which crop region best explains the tag on this image — mirroring what
|
|
||||||
score_image records for live suggestions. Returns (grounding, has_head):
|
|
||||||
|
|
||||||
- has_head False → the tag has no head in the current embedding space (manual/
|
|
||||||
artist/meta tags, or a concept below the head floor). Nothing to localize
|
|
||||||
with, so the UI shows no overlay (distinct from "the whole image won").
|
|
||||||
- grounding None (has_head True) → the whole-image vector best explains it,
|
|
||||||
not any crop; the UI shows the subtle whole-image frame.
|
|
||||||
- grounding {bbox, kind, detector} → the winning region.
|
|
||||||
|
|
||||||
Character heads are covered too (character is a head kind); this deliberately
|
|
||||||
reuses the SigLIP head bag rather than the CCIP figure path so every applied
|
|
||||||
concept grounds through one consistent mechanism."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
cur_version = (await _settings_async(session)).embedder_model_version
|
|
||||||
row = (
|
|
||||||
await session.execute(
|
|
||||||
select(TagHead.weights, TagHead.bias).where(
|
|
||||||
TagHead.tag_id == tag_id,
|
|
||||||
TagHead.embedding_version == cur_version,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
).one_or_none()
|
|
||||||
if row is None:
|
|
||||||
return None, False
|
|
||||||
bag, bag_meta = await _image_bag(session, image_id, cur_version)
|
|
||||||
if not bag:
|
|
||||||
return None, True
|
|
||||||
|
|
||||||
X = np.vstack(bag)
|
|
||||||
norms = np.linalg.norm(X, axis=1, keepdims=True)
|
|
||||||
norms[norms == 0] = 1.0
|
|
||||||
Xn = X / norms
|
|
||||||
# The sigmoid is monotonic in the logit, so the highest-probability bag row is
|
|
||||||
# just argmax of the raw score — no need to exponentiate to pick the winner.
|
|
||||||
z = Xn @ np.asarray(row.weights, dtype=np.float32) + float(row.bias) # (B,)
|
|
||||||
return bag_meta[int(z.argmax())], True
|
|
||||||
|
|
||||||
|
|
||||||
async def _settings_async(session: AsyncSession) -> MLSettings:
|
async def _settings_async(session: AsyncSession) -> MLSettings:
|
||||||
return await MLSettings.load(session)
|
return (
|
||||||
|
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
# --- Earned auto-apply (sync, ml worker) ---------------------------------
|
# --- Earned auto-apply (sync, ml worker) ---------------------------------
|
||||||
@@ -698,11 +468,8 @@ def start_head_auto_apply_run(session: Session, params: dict[str, Any]) -> int:
|
|||||||
|
|
||||||
|
|
||||||
def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
|
def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
|
||||||
"""Eligible CONTENT heads to fire: graduated (auto_apply_threshold set),
|
"""Eligible heads to fire: graduated (auto_apply_threshold set), enough
|
||||||
enough support, current embedding, NON-system. System tags never auto-apply
|
support, current embedding. Returns the row list (tag_id/name/weights/...)."""
|
||||||
via this path — `wip` never auto-applies at all, and banner/editor screenshot
|
|
||||||
go through the presentation path at their own flat threshold (#141). Returns
|
|
||||||
the row list (tag_id/name/weights/...)."""
|
|
||||||
return session.execute(
|
return session.execute(
|
||||||
select(
|
select(
|
||||||
TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias,
|
TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias,
|
||||||
@@ -712,7 +479,6 @@ def _auto_apply_heads(session: Session, embedding_version: str, min_pos: int):
|
|||||||
.where(TagHead.embedding_version == embedding_version)
|
.where(TagHead.embedding_version == embedding_version)
|
||||||
.where(TagHead.auto_apply_threshold.is_not(None))
|
.where(TagHead.auto_apply_threshold.is_not(None))
|
||||||
.where(TagHead.n_pos >= min_pos)
|
.where(TagHead.n_pos >= min_pos)
|
||||||
.where(~Tag.is_system)
|
|
||||||
).all()
|
).all()
|
||||||
|
|
||||||
|
|
||||||
@@ -725,6 +491,7 @@ def auto_apply_sweep(
|
|||||||
embeddings in chunks; commits per chunk on a real run. Returns
|
embeddings in chunks; commits per chunk on a real run. Returns
|
||||||
{n_applied, concepts:[{tag_id,name,applied,scanned,threshold}]}."""
|
{n_applied, concepts:[{tag_id,name,applied,scanned,threshold}]}."""
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
|
||||||
settings = _settings(session)
|
settings = _settings(session)
|
||||||
rows = _auto_apply_heads(
|
rows = _auto_apply_heads(
|
||||||
@@ -741,7 +508,18 @@ def auto_apply_sweep(
|
|||||||
names = [r.name for r in rows]
|
names = [r.name for r in rows]
|
||||||
|
|
||||||
# Skip images that already carry, or have rejected, each tag.
|
# Skip images that already carry, or have rejected, each tag.
|
||||||
skip = _applied_or_rejected(session, tag_ids)
|
skip = {tid: set() for tid in tag_ids}
|
||||||
|
for tid in tag_ids:
|
||||||
|
for (iid,) in session.execute(
|
||||||
|
select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tid)
|
||||||
|
):
|
||||||
|
skip[tid].add(iid)
|
||||||
|
for (iid,) in session.execute(
|
||||||
|
select(TagSuggestionRejection.image_record_id).where(
|
||||||
|
TagSuggestionRejection.tag_id == tid
|
||||||
|
)
|
||||||
|
):
|
||||||
|
skip[tid].add(iid)
|
||||||
|
|
||||||
applied = [0] * len(rows)
|
applied = [0] * len(rows)
|
||||||
scanned = 0
|
scanned = 0
|
||||||
@@ -755,7 +533,7 @@ def auto_apply_sweep(
|
|||||||
if not cids:
|
if not cids:
|
||||||
continue
|
continue
|
||||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
||||||
probs = _sigmoid(Xn @ W.T + b, np) # (N, H)
|
probs = 1.0 / (1.0 + np.exp(-(Xn @ W.T + b))) # (N, H)
|
||||||
scanned += len(cids)
|
scanned += len(cids)
|
||||||
for h in range(len(rows)):
|
for h in range(len(rows)):
|
||||||
tid = tag_ids[h]
|
tid = tag_ids[h]
|
||||||
@@ -782,298 +560,3 @@ def auto_apply_sweep(
|
|||||||
for h in range(len(rows))
|
for h in range(len(rows))
|
||||||
]
|
]
|
||||||
return {"n_applied": sum(applied), "concepts": concepts}
|
return {"n_applied": sum(applied), "concepts": concepts}
|
||||||
|
|
||||||
|
|
||||||
_PRESENTATION_SOURCE = "presentation_auto"
|
|
||||||
_PROCESS_SOURCE = "process_auto"
|
|
||||||
|
|
||||||
# System-tag auto-apply modes (#1464). Both modes run the identical sweep — apply
|
|
||||||
# a system tag at a flat threshold with a PROVISIONAL source + a ring-loud review
|
|
||||||
# guard — and differ ONLY in which tags, which settings knobs, and which
|
|
||||||
# source/review-mode. 'chrome' (banner) is HIDDEN from the gallery; 'process'
|
|
||||||
# (wip / editor screenshot) stays VISIBLE (the hide is a gallery-query effect of
|
|
||||||
# the tag's group membership, not of this sweep).
|
|
||||||
_SWEEP_MODES = {
|
|
||||||
"chrome": {
|
|
||||||
"names": CHROME_SYSTEM_TAGS,
|
|
||||||
"enabled": "presentation_auto_apply_enabled",
|
|
||||||
"threshold": "presentation_auto_apply_threshold",
|
|
||||||
"conflict": "presentation_conflict_threshold",
|
|
||||||
"source": _PRESENTATION_SOURCE,
|
|
||||||
},
|
|
||||||
"process": {
|
|
||||||
"names": PROCESS_SYSTEM_TAGS,
|
|
||||||
"enabled": "process_auto_apply_enabled",
|
|
||||||
"threshold": "process_auto_apply_threshold",
|
|
||||||
"conflict": "process_conflict_threshold",
|
|
||||||
"source": _PROCESS_SOURCE,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def _system_tag_heads(session: Session, embedding_version: str, names):
|
|
||||||
"""Trained heads for a system-tag group (chrome banner / process wip+editor).
|
|
||||||
They fire at the group's FLAT threshold regardless of graduation — a head
|
|
||||||
exists once the operator has labelled enough (head_min_positives)."""
|
|
||||||
return session.execute(
|
|
||||||
select(TagHead.tag_id, Tag.name, TagHead.weights, TagHead.bias)
|
|
||||||
.join(Tag, Tag.id == TagHead.tag_id)
|
|
||||||
.where(TagHead.embedding_version == embedding_version)
|
|
||||||
.where(Tag.is_system.is_(True))
|
|
||||||
.where(Tag.name.in_(names))
|
|
||||||
).all()
|
|
||||||
|
|
||||||
|
|
||||||
def _conflict_heads(session: Session, embedding_version: str):
|
|
||||||
"""ALL content (non-system) heads — the "does this ALSO look like real
|
|
||||||
content" signal for the presentation conflict guard (#141)."""
|
|
||||||
return session.execute(
|
|
||||||
select(TagHead.tag_id, TagHead.weights, TagHead.bias)
|
|
||||||
.join(Tag, Tag.id == TagHead.tag_id)
|
|
||||||
.where(TagHead.embedding_version == embedding_version)
|
|
||||||
.where(~Tag.is_system)
|
|
||||||
).all()
|
|
||||||
|
|
||||||
|
|
||||||
def _valued_image_ids(session: Session) -> set[int]:
|
|
||||||
"""Images the operator has shown they value: carrying a HUMAN or CONFIRMED
|
|
||||||
content (non-system) tag. The presentation sweep never auto-hides these
|
|
||||||
(guard 1) — you tagged it, so the model doesn't get to bury it (#141)."""
|
|
||||||
confirmed = exists().where(
|
|
||||||
TagPositiveConfirmation.image_record_id == image_tag.c.image_record_id,
|
|
||||||
TagPositiveConfirmation.tag_id == image_tag.c.tag_id,
|
|
||||||
)
|
|
||||||
rows = session.execute(
|
|
||||||
select(image_tag.c.image_record_id)
|
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
|
||||||
.where(~Tag.is_system)
|
|
||||||
.where(image_tag.c.source.not_in(_AUTO_SOURCES) | confirmed)
|
|
||||||
).all()
|
|
||||||
return {r[0] for r in rows}
|
|
||||||
|
|
||||||
|
|
||||||
def system_tag_auto_apply_sweep(
|
|
||||||
session: Session, *, mode: str, dry_run: bool = False
|
|
||||||
) -> dict:
|
|
||||||
"""Auto-apply a system-tag group at its FLAT threshold. mode='chrome' (banner,
|
|
||||||
#141) hides the image; mode='process' (wip / editor screenshot, #1464) keeps it
|
|
||||||
VISIBLE — the ONLY difference is the tag group's gallery membership, not this
|
|
||||||
sweep. Two guards keep it safe: (1) never touch an image carrying a
|
|
||||||
human/confirmed content tag; (2) if the image ALSO scores >= the conflict
|
|
||||||
threshold on a content head, still apply but flag it (PresentationReview,
|
|
||||||
mode=<mode>) so the review strip surfaces "also looks like <X>". The source is
|
|
||||||
PROVISIONAL so the head never trains on its own output. No-op unless the mode's
|
|
||||||
enabled flag is set. numpy-only (no sklearn). Returns {n_applied, n_flagged,
|
|
||||||
concepts}."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
cfg = _SWEEP_MODES[mode]
|
|
||||||
settings = _settings(session)
|
|
||||||
if not dry_run and not getattr(settings, cfg["enabled"]):
|
|
||||||
return {"n_applied": 0, "n_flagged": 0, "concepts": []}
|
|
||||||
ver = settings.embedder_model_version
|
|
||||||
pres = _system_tag_heads(session, ver, cfg["names"])
|
|
||||||
if not pres:
|
|
||||||
return {"n_applied": 0, "n_flagged": 0, "concepts": []}
|
|
||||||
thr = float(getattr(settings, cfg["threshold"]))
|
|
||||||
conflict_thr = float(getattr(settings, cfg["conflict"]))
|
|
||||||
source = cfg["source"]
|
|
||||||
|
|
||||||
Wp = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in pres])
|
|
||||||
bp = np.asarray([r.bias for r in pres], dtype=np.float32)
|
|
||||||
pres_tag_ids = [r.tag_id for r in pres]
|
|
||||||
pres_names = [r.name for r in pres]
|
|
||||||
|
|
||||||
conf = _conflict_heads(session, ver)
|
|
||||||
Wc = bc = conf_tag_ids = None
|
|
||||||
if conf:
|
|
||||||
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
|
|
||||||
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
|
|
||||||
conf_tag_ids = [r.tag_id for r in conf]
|
|
||||||
|
|
||||||
valued = _valued_image_ids(session)
|
|
||||||
|
|
||||||
# Skip images that already carry, or have rejected, each presentation tag.
|
|
||||||
skip = _applied_or_rejected(session, pres_tag_ids)
|
|
||||||
|
|
||||||
applied = [0] * len(pres)
|
|
||||||
n_flagged = 0
|
|
||||||
scanned = 0
|
|
||||||
all_ids = list(session.execute(
|
|
||||||
select(ImageRecord.id).where(ImageRecord.siglip_embedding.is_not(None))
|
|
||||||
).scalars())
|
|
||||||
for start in range(0, len(all_ids), _AUTO_APPLY_CHUNK):
|
|
||||||
chunk = all_ids[start:start + _AUTO_APPLY_CHUNK]
|
|
||||||
emb = _load_embeddings(session, chunk)
|
|
||||||
cids = [i for i in chunk if i in emb]
|
|
||||||
if not cids:
|
|
||||||
continue
|
|
||||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
|
||||||
probs = _sigmoid(Xn @ Wp.T + bp, np) # (N, P)
|
|
||||||
if Wc is not None:
|
|
||||||
max_c, arg_c = _conflict_scores(Xn, Wc, bc, np) # (N,), (N,)
|
|
||||||
scanned += len(cids)
|
|
||||||
for p in range(len(pres)):
|
|
||||||
tid = pres_tag_ids[p]
|
|
||||||
for idx in np.where(probs[:, p] >= thr)[0]:
|
|
||||||
iid = cids[int(idx)]
|
|
||||||
if iid in skip[tid] or iid in valued:
|
|
||||||
continue
|
|
||||||
skip[tid].add(iid)
|
|
||||||
applied[p] += 1
|
|
||||||
if not dry_run:
|
|
||||||
session.execute(
|
|
||||||
pg_insert(image_tag)
|
|
||||||
.values(
|
|
||||||
image_record_id=iid, tag_id=tid,
|
|
||||||
source=source,
|
|
||||||
)
|
|
||||||
.on_conflict_do_nothing()
|
|
||||||
)
|
|
||||||
# Guard 2: also looks like real content → still apply, but flag it
|
|
||||||
# for the review strip instead of silently marking (chrome hides,
|
|
||||||
# process stays visible — either way the operator gets a heads-up).
|
|
||||||
if Wc is not None and float(max_c[idx]) >= conflict_thr:
|
|
||||||
n_flagged += 1
|
|
||||||
if not dry_run:
|
|
||||||
_insert_presentation_review(
|
|
||||||
session,
|
|
||||||
image_record_id=iid, tag_id=tid,
|
|
||||||
conflict_tag_id=conf_tag_ids[int(arg_c[idx])],
|
|
||||||
conflict_score=float(max_c[idx]),
|
|
||||||
mode=mode,
|
|
||||||
)
|
|
||||||
if not dry_run:
|
|
||||||
session.commit()
|
|
||||||
|
|
||||||
concepts = [
|
|
||||||
{"tag_id": pres_tag_ids[p], "name": pres_names[p],
|
|
||||||
"applied": applied[p], "scanned": scanned, "threshold": thr}
|
|
||||||
for p in range(len(pres))
|
|
||||||
]
|
|
||||||
return {
|
|
||||||
"n_applied": sum(applied), "n_flagged": n_flagged, "concepts": concepts,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def soft_wip_conflict_audit(session: Session, dry_run: bool = False) -> dict:
|
|
||||||
"""Ring-loud audit for the SOFT WIP-title cohort (#1474). Images auto-tagged
|
|
||||||
`wip` from a low-precision sketch/doodle title (source='wip_title_soft') that ALSO
|
|
||||||
score >= the process conflict threshold on a content head are probably FINISHED
|
|
||||||
art mis-tagged as process — flag them (PresentationReview, mode='process') so the
|
|
||||||
review strip surfaces them ("also looks like <X>", Keep tag / Remove tag). Does
|
|
||||||
NOT remove the tag; the operator decides. No-op when there are no content heads.
|
|
||||||
numpy-only. Returns {n_scanned, n_flagged}."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
from ..wip_title import WIP_TITLE_SOFT_SOURCE, resolve_wip_tag_id
|
|
||||||
|
|
||||||
settings = _settings(session)
|
|
||||||
ver = settings.embedder_model_version
|
|
||||||
conflict_thr = float(settings.process_conflict_threshold)
|
|
||||||
conf = _conflict_heads(session, ver)
|
|
||||||
wip_id = resolve_wip_tag_id(session)
|
|
||||||
if not conf or wip_id is None:
|
|
||||||
return {"n_scanned": 0, "n_flagged": 0}
|
|
||||||
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
|
|
||||||
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
|
|
||||||
conf_tag_ids = [r.tag_id for r in conf]
|
|
||||||
|
|
||||||
soft_ids = [iid for (iid,) in session.execute(
|
|
||||||
select(image_tag.c.image_record_id)
|
|
||||||
.where(image_tag.c.tag_id == wip_id)
|
|
||||||
.where(image_tag.c.source == WIP_TITLE_SOFT_SOURCE)
|
|
||||||
)]
|
|
||||||
# Skip images already flagged for this tag (idempotent re-runs).
|
|
||||||
flagged = {iid for (iid,) in session.execute(
|
|
||||||
select(PresentationReview.image_record_id)
|
|
||||||
.where(PresentationReview.tag_id == wip_id)
|
|
||||||
)}
|
|
||||||
soft_ids = [i for i in soft_ids if i not in flagged]
|
|
||||||
|
|
||||||
n_flagged = 0
|
|
||||||
scanned = 0
|
|
||||||
for start in range(0, len(soft_ids), _AUTO_APPLY_CHUNK):
|
|
||||||
chunk = soft_ids[start:start + _AUTO_APPLY_CHUNK]
|
|
||||||
emb = _load_embeddings(session, chunk)
|
|
||||||
cids = [i for i in chunk if i in emb]
|
|
||||||
if not cids:
|
|
||||||
continue
|
|
||||||
scanned += len(cids)
|
|
||||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
|
||||||
max_c, arg_c = _conflict_scores(Xn, Wc, bc, np)
|
|
||||||
for k in range(len(cids)):
|
|
||||||
if float(max_c[k]) >= conflict_thr:
|
|
||||||
n_flagged += 1
|
|
||||||
if not dry_run:
|
|
||||||
_insert_presentation_review(
|
|
||||||
session,
|
|
||||||
image_record_id=cids[k], tag_id=wip_id,
|
|
||||||
conflict_tag_id=conf_tag_ids[int(arg_c[k])],
|
|
||||||
conflict_score=float(max_c[k]),
|
|
||||||
mode="process",
|
|
||||||
)
|
|
||||||
if not dry_run:
|
|
||||||
session.commit()
|
|
||||||
return {"n_scanned": scanned, "n_flagged": n_flagged}
|
|
||||||
|
|
||||||
|
|
||||||
def retract_auto_applied_heads(session: Session) -> int:
|
|
||||||
"""Soft auto-apply (milestone 139): re-score every standing source='head_auto'
|
|
||||||
tag against its CURRENT head and REMOVE the ones now BELOW the head's
|
|
||||||
auto_apply_threshold — i.e. the head sharpened (or the operator raised the bar)
|
|
||||||
and no longer supports them. Skips operator-confirmed tags
|
|
||||||
(TagPositiveConfirmation). SILENT: a low score isn't proof the tag was wrong,
|
|
||||||
so no hard negative is recorded — that's reserved for an operator removal.
|
|
||||||
No-op unless head_auto_apply_enabled. Only re-scores the images that ALREADY
|
|
||||||
carry the auto-tag (bounded), never the whole library. Returns n_retracted."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
settings = _settings(session)
|
|
||||||
if not settings.head_auto_apply_enabled:
|
|
||||||
return 0
|
|
||||||
heads = session.execute(
|
|
||||||
select(
|
|
||||||
TagHead.tag_id, TagHead.weights, TagHead.bias,
|
|
||||||
TagHead.auto_apply_threshold,
|
|
||||||
)
|
|
||||||
.where(TagHead.embedding_version == settings.embedder_model_version)
|
|
||||||
.where(TagHead.auto_apply_threshold.is_not(None))
|
|
||||||
).all()
|
|
||||||
retracted = 0
|
|
||||||
for tag_id, weights, bias, thr in heads:
|
|
||||||
auto_ids = [
|
|
||||||
iid for (iid,) in session.execute(
|
|
||||||
select(image_tag.c.image_record_id)
|
|
||||||
.where(image_tag.c.tag_id == tag_id)
|
|
||||||
.where(image_tag.c.source == "head_auto")
|
|
||||||
)
|
|
||||||
]
|
|
||||||
if not auto_ids:
|
|
||||||
continue
|
|
||||||
confirmed = {
|
|
||||||
iid for (iid,) in session.execute(
|
|
||||||
select(TagPositiveConfirmation.image_record_id)
|
|
||||||
.where(TagPositiveConfirmation.tag_id == tag_id)
|
|
||||||
.where(TagPositiveConfirmation.image_record_id.in_(auto_ids))
|
|
||||||
)
|
|
||||||
}
|
|
||||||
candidates = [i for i in auto_ids if i not in confirmed]
|
|
||||||
emb = _load_embeddings(session, candidates)
|
|
||||||
cids = [i for i in candidates if i in emb]
|
|
||||||
if not cids:
|
|
||||||
continue
|
|
||||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
|
||||||
w = np.asarray(weights, dtype=np.float32)
|
|
||||||
probs = _sigmoid(Xn @ w + float(bias), np)
|
|
||||||
below = [cids[k] for k in np.where(probs < float(thr))[0]]
|
|
||||||
for iid in below:
|
|
||||||
session.execute(
|
|
||||||
image_tag.delete()
|
|
||||||
.where(image_tag.c.image_record_id == iid)
|
|
||||||
.where(image_tag.c.tag_id == tag_id)
|
|
||||||
.where(image_tag.c.source == "head_auto")
|
|
||||||
)
|
|
||||||
retracted += 1
|
|
||||||
session.commit()
|
|
||||||
return retracted
|
|
||||||
|
|||||||
@@ -22,30 +22,27 @@ from .heads import score_image
|
|||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class Suggestion:
|
class Suggestion:
|
||||||
# Every suggestion is a canonical Tag: heads/CCIP only score EXISTING concept
|
# canonical_tag_id is None when this is a raw Camie tag with no alias and
|
||||||
# tags (tagging-v2, #114). The old raw-model-key / creates-new / alias-remap
|
# no existing Tag row — accepting it will create the tag.
|
||||||
# cases are gone — a suggestion always maps to a real tag id.
|
canonical_tag_id: int | None
|
||||||
canonical_tag_id: int
|
|
||||||
display_name: str
|
display_name: str
|
||||||
category: str
|
category: str
|
||||||
score: float
|
score: float
|
||||||
source: str # 'head' | 'ccip' | 'both' (Camie tagger/centroid removed in v2)
|
source: str # 'head' | 'ccip' | 'both' (Camie tagger/centroid removed in v2)
|
||||||
# above_threshold = the score cleared the head's own suggest cut (or the
|
creates_new_tag: bool
|
||||||
# system floor). The Suggestions PANEL shows only these; the typed-tag
|
# raw_name = the booru model vocab key behind this suggestion. It's the key
|
||||||
# dropdown fetches ALL suggestions (every head, min=0) and just annotates each
|
# an alias MUST be stored under (resolution looks up the raw key), so the
|
||||||
# matching row with its score, so a low-confidence concept can still be typed
|
# modal needs it to author an alias correctly. None for centroid-only hits
|
||||||
# and picked. CCIP character matches are always above their match threshold.
|
# (no underlying prediction → nothing to alias).
|
||||||
above_threshold: bool
|
raw_name: str | None = None
|
||||||
|
# via_alias = this suggestion was surfaced because an operator alias remapped
|
||||||
|
# the raw prediction to this canonical tag. Lets the UI mark it + offer undo.
|
||||||
|
via_alias: bool = False
|
||||||
# rejected = the operator dismissed this tag for this image (a stored
|
# rejected = the operator dismissed this tag for this image (a stored
|
||||||
# TagSuggestionRejection). It stays in the list — flagged, not dropped — so
|
# TagSuggestionRejection). It stays in the list — flagged, not dropped — so
|
||||||
# the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery,
|
# the rejection is VISIBLE and REVERSIBLE in the rail (misclick recovery,
|
||||||
# operator-asked 2026-06-27) instead of silently vanishing or re-suggesting.
|
# operator-asked 2026-06-27) instead of silently vanishing or re-suggesting.
|
||||||
rejected: bool = False
|
rejected: bool = False
|
||||||
# grounding = the crop region that produced this suggestion (#1206):
|
|
||||||
# {bbox:[x,y,w,h] normalized, kind, detector}. None when the whole-image
|
|
||||||
# vector won (not localized) or for a CCIP-only hit (figure grounding TBD).
|
|
||||||
# Lets the rail highlight the exact region on hover.
|
|
||||||
grounding: dict | None = None
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -106,8 +103,6 @@ class SuggestionService:
|
|||||||
for h in hits:
|
for h in hits:
|
||||||
merged[(h["category"], h["tag_id"])] = {
|
merged[(h["category"], h["tag_id"])] = {
|
||||||
"name": h["name"], "score": h["score"], "source": "head",
|
"name": h["name"], "score": h["score"], "source": "head",
|
||||||
"above_threshold": h["above_threshold"],
|
|
||||||
"grounding": h.get("grounding"),
|
|
||||||
}
|
}
|
||||||
for c in ccip_hits:
|
for c in ccip_hits:
|
||||||
key = ("character", c["tag_id"])
|
key = ("character", c["tag_id"])
|
||||||
@@ -115,17 +110,9 @@ class SuggestionService:
|
|||||||
if ex is not None:
|
if ex is not None:
|
||||||
ex["source"] = "both"
|
ex["source"] = "both"
|
||||||
ex["score"] = max(ex["score"], c["score"])
|
ex["score"] = max(ex["score"], c["score"])
|
||||||
# CCIP only returns matches above its own threshold, so a CCIP
|
|
||||||
# corroboration always makes the merged suggestion above-threshold.
|
|
||||||
ex["above_threshold"] = True
|
|
||||||
# Keep the head's localized crop if it had one; else fall back to
|
|
||||||
# the CCIP figure so a corroborated character still grounds (#1206).
|
|
||||||
ex["grounding"] = ex.get("grounding") or c.get("grounding")
|
|
||||||
else:
|
else:
|
||||||
merged[key] = {
|
merged[key] = {
|
||||||
"name": c["name"], "score": c["score"], "source": "ccip",
|
"name": c["name"], "score": c["score"], "source": "ccip",
|
||||||
"above_threshold": True,
|
|
||||||
"grounding": c.get("grounding"),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
result = SuggestionList()
|
result = SuggestionList()
|
||||||
@@ -139,9 +126,8 @@ class SuggestionService:
|
|||||||
category=cat,
|
category=cat,
|
||||||
score=m["score"],
|
score=m["score"],
|
||||||
source=m["source"],
|
source=m["source"],
|
||||||
above_threshold=m["above_threshold"],
|
creates_new_tag=False,
|
||||||
rejected=tag_id in rejected,
|
rejected=tag_id in rejected,
|
||||||
grounding=m.get("grounding"),
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
for cat in result.by_category:
|
for cat in result.by_category:
|
||||||
@@ -160,7 +146,8 @@ class SuggestionService:
|
|||||||
was suggested for (or already applied to) >= threshold fraction of
|
was suggested for (or already applied to) >= threshold fraction of
|
||||||
the selection AND was acceptable on >= 1 image. Confidence is the
|
the selection AND was acceptable on >= 1 image. Confidence is the
|
||||||
mean over images where it was suggested. Aggregated by
|
mean over images where it was suggested. Aggregated by
|
||||||
canonical_tag_id (every suggestion is a canonical tag now)."""
|
canonical_tag_id; creates-new (no canonical id) suggestions are
|
||||||
|
skipped (bulk Accept applies by tag id)."""
|
||||||
if not image_ids:
|
if not image_ids:
|
||||||
return {}
|
return {}
|
||||||
threshold = min(1.0, max(0.0, threshold))
|
threshold = min(1.0, max(0.0, threshold))
|
||||||
@@ -171,6 +158,8 @@ class SuggestionService:
|
|||||||
sl = await self.for_image(image_id)
|
sl = await self.for_image(image_id)
|
||||||
for category, items in sl.by_category.items():
|
for category, items in sl.by_category.items():
|
||||||
for s in items:
|
for s in items:
|
||||||
|
if s.canonical_tag_id is None or s.creates_new_tag:
|
||||||
|
continue
|
||||||
# for_image keeps rejected tags (flagged) for the rail;
|
# for_image keeps rejected tags (flagged) for the rail;
|
||||||
# bulk consensus must still ignore them — a tag dismissed on
|
# bulk consensus must still ignore them — a tag dismissed on
|
||||||
# an image isn't a suggestion for that image.
|
# an image isn't a suggestion for that image.
|
||||||
|
|||||||
@@ -17,28 +17,9 @@ from typing import Any
|
|||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from ...models import (
|
from ...models import ImageRecord, Tag, TagSuggestionRejection
|
||||||
ImageRecord,
|
|
||||||
Tag,
|
|
||||||
TagPositiveConfirmation,
|
|
||||||
TagSuggestionRejection,
|
|
||||||
)
|
|
||||||
from ...models.tag import image_tag
|
from ...models.tag import image_tag
|
||||||
|
|
||||||
# Auto-apply sources whose tags are PROVISIONAL: they never train a head (or seed
|
|
||||||
# a CCIP reference) unless the operator confirms them (milestone 139). Keeping
|
|
||||||
# auto-applied predictions out of training is what makes them "soft" — a misfire
|
|
||||||
# can't reinforce itself, so the retraction sweep can actually drop it.
|
|
||||||
# `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
|
|
||||||
# ALSO provisional — the head must learn only from title (`wip_title`) + manual
|
|
||||||
# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
|
|
||||||
# `wip_title_soft` (#1474): the soft title tier (sketch/doodle) is LOW-precision, so
|
|
||||||
# it's provisional too — a finished piece titled "sketch" must not train the wip head.
|
|
||||||
_AUTO_SOURCES = (
|
|
||||||
"head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
|
|
||||||
"wip_title_soft",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _hygiene_excluded_ids(session: Session) -> set[int]:
|
def _hygiene_excluded_ids(session: Session) -> set[int]:
|
||||||
"""Ids of images carrying ANY system tag (wip / banner / editor
|
"""Ids of images carrying ANY system tag (wip / banner / editor
|
||||||
@@ -64,23 +45,9 @@ def _hygiene_excluded_ids(session: Session) -> set[int]:
|
|||||||
|
|
||||||
|
|
||||||
def _ids_with_tag(session: Session, tag_id: int) -> list[int]:
|
def _ids_with_tag(session: Session, tag_id: int) -> list[int]:
|
||||||
"""Image ids that count as POSITIVES for this tag's head: human-applied
|
|
||||||
(manual / accepted) tags PLUS any auto-applied tag the operator explicitly
|
|
||||||
confirmed (TagPositiveConfirmation). Unconfirmed auto-applied tags are
|
|
||||||
EXCLUDED — they are provisional and must not train the head that judges
|
|
||||||
them (milestone 139)."""
|
|
||||||
confirmed = (
|
|
||||||
select(TagPositiveConfirmation.image_record_id)
|
|
||||||
.where(TagPositiveConfirmation.tag_id == tag_id)
|
|
||||||
)
|
|
||||||
return [
|
return [
|
||||||
r[0] for r in session.execute(
|
r[0] for r in session.execute(
|
||||||
select(image_tag.c.image_record_id)
|
select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tag_id)
|
||||||
.where(image_tag.c.tag_id == tag_id)
|
|
||||||
.where(
|
|
||||||
image_tag.c.source.not_in(_AUTO_SOURCES)
|
|
||||||
| image_tag.c.image_record_id.in_(confirmed)
|
|
||||||
)
|
|
||||||
).all()
|
).all()
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -94,24 +61,6 @@ def _rejected_ids(session: Session, tag_id: int) -> list[int]:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _applied_or_rejected(session: Session, tag_ids) -> dict[int, set[int]]:
|
|
||||||
"""Per-tag skip set for the auto-apply sweeps: every image that ALREADY carries
|
|
||||||
the tag (ANY source — not just training positives) OR has rejected it. A sweep
|
|
||||||
never re-applies to these. Shared by auto_apply_sweep + system_tag_auto_apply_sweep
|
|
||||||
(heads.py) and scheduled_ccip_auto_apply (tasks/ml.py). Callers mutate the returned
|
|
||||||
sets in-place to also dedupe within a single run."""
|
|
||||||
skip: dict[int, set[int]] = {}
|
|
||||||
for tid in tag_ids:
|
|
||||||
ids = {
|
|
||||||
r[0] for r in session.execute(
|
|
||||||
select(image_tag.c.image_record_id).where(image_tag.c.tag_id == tid)
|
|
||||||
).all()
|
|
||||||
}
|
|
||||||
ids.update(_rejected_ids(session, tid))
|
|
||||||
skip[tid] = ids
|
|
||||||
return skip
|
|
||||||
|
|
||||||
|
|
||||||
def _sample_unlabeled(session: Session, exclude: set[int], limit: int) -> list[int]:
|
def _sample_unlabeled(session: Session, exclude: set[int], limit: int) -> list[int]:
|
||||||
"""Random image ids (with an embedding) NOT carrying the tag. Concepts are
|
"""Random image ids (with an embedding) NOT carrying the tag. Concepts are
|
||||||
sparse, so an untagged image is almost always a true negative."""
|
sparse, so an untagged image is almost always a true negative."""
|
||||||
|
|||||||
@@ -91,46 +91,48 @@ def _sync_lookup(vanity: str, cookies_path: str | None) -> str | None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _campaigns_api_first(vanity: str, cookies_path: str | None) -> dict | None:
|
def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
|
||||||
"""The first `data` object from Patreon's campaigns API filtered by vanity
|
|
||||||
(`?filter[vanity]=<vanity>&fields[campaign]=name`), or None on any failure
|
|
||||||
(network / non-200 / non-JSON / empty). The single request shape shared by
|
|
||||||
_lookup_via_api (plucks the campaign id) and resolve_display_name (plucks the
|
|
||||||
display name)."""
|
|
||||||
jar = _load_cookie_jar(cookies_path)
|
jar = _load_cookie_jar(cookies_path)
|
||||||
|
headers = {
|
||||||
|
"User-Agent": _USER_AGENT,
|
||||||
|
"Accept": "application/vnd.api+json",
|
||||||
|
}
|
||||||
|
params = {
|
||||||
|
"filter[vanity]": vanity,
|
||||||
|
"fields[campaign]": "name",
|
||||||
|
}
|
||||||
try:
|
try:
|
||||||
resp = requests.get(
|
resp = requests.get(
|
||||||
_CAMPAIGNS_URL,
|
_CAMPAIGNS_URL,
|
||||||
params={"filter[vanity]": vanity, "fields[campaign]": "name"},
|
params=params,
|
||||||
headers={"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"},
|
headers=headers,
|
||||||
cookies=jar,
|
cookies=jar,
|
||||||
timeout=_TIMEOUT_SECONDS,
|
timeout=_TIMEOUT_SECONDS,
|
||||||
)
|
)
|
||||||
except requests.RequestException as exc:
|
except requests.RequestException as exc:
|
||||||
log.warning("Patreon campaigns API request failed for vanity=%s: %s", vanity, exc)
|
log.warning("Patreon campaigns API request failed for vanity=%s: %s", vanity, exc)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if resp.status_code != 200:
|
if resp.status_code != 200:
|
||||||
log.warning(
|
log.warning(
|
||||||
"Patreon campaigns API returned HTTP %d for vanity=%s",
|
"Patreon campaigns API returned HTTP %d for vanity=%s",
|
||||||
resp.status_code, vanity,
|
resp.status_code, vanity,
|
||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
payload = resp.json()
|
payload = resp.json()
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
log.warning("Patreon campaigns API returned non-JSON for vanity=%s: %s", vanity, exc)
|
log.warning("Patreon campaigns API returned non-JSON for vanity=%s: %s", vanity, exc)
|
||||||
return None
|
return None
|
||||||
data = payload.get("data") if isinstance(payload, dict) else None
|
|
||||||
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
|
||||||
return None
|
|
||||||
return data[0]
|
|
||||||
|
|
||||||
|
if not isinstance(payload, dict):
|
||||||
def _lookup_via_api(vanity: str, cookies_path: str | None) -> str | None:
|
|
||||||
first = _campaigns_api_first(vanity, cookies_path)
|
|
||||||
if first is None:
|
|
||||||
return None
|
return None
|
||||||
campaign_id = first.get("id")
|
data = payload.get("data")
|
||||||
|
if not isinstance(data, list) or not data:
|
||||||
|
return None
|
||||||
|
first = data[0] if isinstance(data[0], dict) else None
|
||||||
|
campaign_id = first.get("id") if first else None
|
||||||
if not isinstance(campaign_id, str) or not campaign_id:
|
if not isinstance(campaign_id, str) or not campaign_id:
|
||||||
return None
|
return None
|
||||||
log.info("Resolved Patreon vanity=%s → campaign_id=%s", vanity, campaign_id)
|
log.info("Resolved Patreon vanity=%s → campaign_id=%s", vanity, campaign_id)
|
||||||
@@ -142,10 +144,24 @@ def resolve_display_name(vanity: str, cookies_path: str | None) -> str | None:
|
|||||||
(`fields[campaign]=name`), used to name the Artist at add-time (#130). None
|
(`fields[campaign]=name`), used to name the Artist at add-time (#130). None
|
||||||
on any failure — the caller falls back to the vanity handle. Sync: call from
|
on any failure — the caller falls back to the vanity handle. Sync: call from
|
||||||
an executor."""
|
an executor."""
|
||||||
first = _campaigns_api_first(vanity, cookies_path)
|
jar = _load_cookie_jar(cookies_path)
|
||||||
if first is None:
|
try:
|
||||||
|
resp = requests.get(
|
||||||
|
_CAMPAIGNS_URL,
|
||||||
|
params={"filter[vanity]": vanity, "fields[campaign]": "name"},
|
||||||
|
headers={"User-Agent": _USER_AGENT, "Accept": "application/vnd.api+json"},
|
||||||
|
cookies=jar,
|
||||||
|
timeout=_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
return None
|
||||||
|
data = resp.json().get("data")
|
||||||
|
except (requests.RequestException, ValueError) as exc:
|
||||||
|
log.warning("Patreon name lookup failed for vanity=%s: %s", vanity, exc)
|
||||||
return None
|
return None
|
||||||
name = (first.get("attributes") or {}).get("name")
|
if not isinstance(data, list) or not data or not isinstance(data[0], dict):
|
||||||
|
return None
|
||||||
|
name = (data[0].get("attributes") or {}).get("name")
|
||||||
return name.strip() if isinstance(name, str) and name.strip() else None
|
return name.strip() if isinstance(name, str) and name.strip() else None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -200,8 +200,6 @@ class PostFeedService:
|
|||||||
atts_map = await self._attachments_for([post.id])
|
atts_map = await self._attachments_for([post.id])
|
||||||
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
||||||
item["description_full"] = html_to_plain(post.description)
|
item["description_full"] = html_to_plain(post.description)
|
||||||
# Full (uncapped) translated description for the detail view (#143).
|
|
||||||
item["description_translated_full"] = post.description_translated
|
|
||||||
# Sanitized HTML body for faithful (semantic) rendering in the post view;
|
# Sanitized HTML body for faithful (semantic) rendering in the post view;
|
||||||
# detail-only (the feed list stays lightweight plain text). None when the
|
# detail-only (the feed list stays lightweight plain text). None when the
|
||||||
# post has no body. Inline `<img>` sources are remapped to locally-served
|
# post has no body. Inline `<img>` sources are remapped to locally-served
|
||||||
@@ -373,12 +371,6 @@ class PostFeedService:
|
|||||||
description_plain, truncated = None, False
|
description_plain, truncated = None, False
|
||||||
else:
|
else:
|
||||||
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
description_plain, truncated = truncate_at_word(plain_full, DESCRIPTION_LIMIT)
|
||||||
# Translation (#143): the stored translated description is already plain
|
|
||||||
# text; truncate it the same way for the card.
|
|
||||||
desc_trans = post.description_translated
|
|
||||||
desc_trans_short = (
|
|
||||||
truncate_at_word(desc_trans, DESCRIPTION_LIMIT)[0] if desc_trans else None
|
|
||||||
)
|
|
||||||
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
thumbs_entry = thumbs_map.get(post.id, {"thumbs": [], "more": 0})
|
||||||
# `source` is null for filesystem-imported posts with no live
|
# `source` is null for filesystem-imported posts with no live
|
||||||
# subscription (alembic 0030). Frontend renders that as a
|
# subscription (alembic 0030). Frontend renders that as a
|
||||||
@@ -392,13 +384,6 @@ class PostFeedService:
|
|||||||
"downloaded_at": post.downloaded_at.isoformat(),
|
"downloaded_at": post.downloaded_at.isoformat(),
|
||||||
"description_plain": description_plain,
|
"description_plain": description_plain,
|
||||||
"description_truncated": truncated,
|
"description_truncated": truncated,
|
||||||
# Translation-forward fields (#143): shown by default when present;
|
|
||||||
# UI toggles back to the originals above. Source lang labels the toggle.
|
|
||||||
"post_title_translated": post.post_title_translated,
|
|
||||||
"description_translated": desc_trans_short,
|
|
||||||
"translated_source_lang": post.translated_source_lang,
|
|
||||||
# Sticky per-post translation choice (auto/force/original, #155).
|
|
||||||
"translation_override": post.translation_override,
|
|
||||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||||
"source": (
|
"source": (
|
||||||
{"id": source.id, "platform": source.platform}
|
{"id": source.id, "platform": source.platform}
|
||||||
|
|||||||
@@ -29,13 +29,6 @@ def _post_dict(p: Post) -> dict:
|
|||||||
"date": p.post_date.isoformat() if p.post_date else None,
|
"date": p.post_date.isoformat() if p.post_date else None,
|
||||||
"description_html": sanitize_post_html(p.description),
|
"description_html": sanitize_post_html(p.description),
|
||||||
"attachment_count": p.attachment_count,
|
"attachment_count": p.attachment_count,
|
||||||
# Translation (#143): the English title/description shown by default when
|
|
||||||
# a translation exists; the UI toggles to the original. Source lang labels
|
|
||||||
# the original.
|
|
||||||
"title_translated": p.post_title_translated,
|
|
||||||
"description_translated": p.description_translated,
|
|
||||||
"translated_source_lang": p.translated_source_lang,
|
|
||||||
"translation_override": p.translation_override,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -91,11 +91,4 @@ def serialize_tag(row) -> dict:
|
|||||||
"fandom_id": row.fandom_id,
|
"fandom_id": row.fandom_id,
|
||||||
"fandom_name": row.fandom_name,
|
"fandom_name": row.fandom_name,
|
||||||
"is_system": bool(getattr(row, "is_system", False)),
|
"is_system": bool(getattr(row, "is_system", False)),
|
||||||
# Applied-tag context: only the image-scoped selects (list_for_image /
|
|
||||||
# get_image_with_tags) provide these; autocomplete / directory don't →
|
|
||||||
# default. `source` drives the auto-applied badge; `confirmed` = the
|
|
||||||
# operator affirmed the tag (a training positive, shielded from the
|
|
||||||
# retraction sweep — milestone 139).
|
|
||||||
"source": getattr(row, "source", None),
|
|
||||||
"confirmed": bool(getattr(row, "confirmed", False)),
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,14 +9,7 @@ from sqlalchemy import and_, case, exists, func, select, text, update
|
|||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from ..models import (
|
from ..models import HeadMetric, Tag, TagHead, TagKind, image_tag
|
||||||
HeadMetric,
|
|
||||||
Tag,
|
|
||||||
TagHead,
|
|
||||||
TagKind,
|
|
||||||
TagPositiveConfirmation,
|
|
||||||
image_tag,
|
|
||||||
)
|
|
||||||
from .db_helpers import get_or_create
|
from .db_helpers import get_or_create
|
||||||
from .tag_query import fandom_join_alias, tag_columns
|
from .tag_query import fandom_join_alias, tag_columns
|
||||||
|
|
||||||
@@ -295,14 +288,8 @@ class TagService:
|
|||||||
character chip with its fandom without an N+1 (mirrors the
|
character chip with its fandom without an N+1 (mirrors the
|
||||||
autocomplete/directory resolution)."""
|
autocomplete/directory resolution)."""
|
||||||
fandom_alias = fandom_join_alias()
|
fandom_alias = fandom_join_alias()
|
||||||
# source drives the auto-applied badge; confirmed = operator affirmed the
|
|
||||||
# tag (positive + retraction-shielded, milestone 139).
|
|
||||||
confirmed = exists().where(
|
|
||||||
TagPositiveConfirmation.image_record_id == image_id,
|
|
||||||
TagPositiveConfirmation.tag_id == Tag.id,
|
|
||||||
).label("confirmed")
|
|
||||||
stmt = (
|
stmt = (
|
||||||
select(*tag_columns(fandom_alias), image_tag.c.source, confirmed)
|
select(*tag_columns(fandom_alias))
|
||||||
.select_from(
|
.select_from(
|
||||||
Tag.__table__
|
Tag.__table__
|
||||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||||
|
|||||||
@@ -1,125 +0,0 @@
|
|||||||
"""Title-based WIP auto-tagging (task #1458).
|
|
||||||
|
|
||||||
Deterministic heuristic: when a post's TITLE explicitly declares work-in-progress
|
|
||||||
(the artist's own "WIP" / "work in progress" label), the ``wip`` system tag is
|
|
||||||
applied to that post's images — a cheap, high-precision complement to the
|
|
||||||
image-based ML ``wip`` head. WIP images are excluded from the Explore/gallery
|
|
||||||
browse (see gallery_service ``excluded_system_tags``), so honouring the artist's
|
|
||||||
own label keeps unfinished pieces out of the main browse right at import.
|
|
||||||
|
|
||||||
Precision over recall — a false WIP tag HIDES a finished post — so matching is
|
|
||||||
token-anchored: ``swipe`` / ``wiped`` / ``wiping`` never trip it (a letter on the
|
|
||||||
boundary blocks the match).
|
|
||||||
|
|
||||||
Sync-only: both consumers (the importer and the backfill Celery task) run on a
|
|
||||||
sync Session. Application is idempotent-additive (ON CONFLICT DO NOTHING) and
|
|
||||||
stamps a distinct ``image_tag.source`` so a later pass can tell where a wip tag
|
|
||||||
came from — the "manual" / "head_auto" / "ccip_auto" / "ml_accepted" provenance
|
|
||||||
family gains one member.
|
|
||||||
"""
|
|
||||||
import re
|
|
||||||
|
|
||||||
from sqlalchemy import select
|
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from ..models.tag import WIP_SYSTEM_TAG, Tag, image_tag
|
|
||||||
|
|
||||||
# image_tag.source stamped on title-heuristic WIP tags — distinct from the other
|
|
||||||
# apply sources so provenance stays legible and a future undo can target only these.
|
|
||||||
# HARD tier ("WIP"/"work in progress") is high-precision → trains the wip head.
|
|
||||||
WIP_TITLE_SOURCE = "wip_title"
|
|
||||||
# SOFT tier (sketch/doodle/scribble, #1474) is LOWER-precision — a finished "sketch"
|
|
||||||
# is often not WIP. This source is PROVISIONAL (in training_data._AUTO_SOURCES) so it
|
|
||||||
# NEVER trains the wip head; a soft-tagged image that also looks like real content is
|
|
||||||
# surfaced by the ring-loud audit for review.
|
|
||||||
WIP_TITLE_SOFT_SOURCE = "wip_title_soft"
|
|
||||||
|
|
||||||
# A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
|
|
||||||
# (space/underscore/hyphen separated). The letter-boundary lookarounds are what
|
|
||||||
# make this precision-first: `s|wip|e`, `|wip|ed`, `|wip|ing` all have a letter
|
|
||||||
# abutting the token, so they're rejected. A trailing digit is allowed so
|
|
||||||
# "WIP2" (= WIP part 2) still matches.
|
|
||||||
_WIP_RE = re.compile(
|
|
||||||
r"(?<![A-Za-z])(?:w\.?i\.?p\.?|work[\s_-]+in[\s_-]+progress)(?![A-Za-z])",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Soft tier: sketch / doodle / scribble (+ plurals), letter-boundary anchored so
|
|
||||||
# "sketchbook" / "kadoodle" don't trip it. Deliberately conservative — recall is
|
|
||||||
# secondary because the soft source doesn't train the head and the ring-loud audit
|
|
||||||
# catches false positives.
|
|
||||||
_SOFT_WIP_RE = re.compile(
|
|
||||||
r"(?<![A-Za-z])(?:sketch|sketches|doodle|doodles|scribble|scribbles)(?![A-Za-z])",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Coarse SQL prefilters for the backfill sweep — narrow the post scan to rows that
|
|
||||||
# COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
|
|
||||||
# Each MUST stay a SUPERSET of its regex or the sweep would silently miss posts.
|
|
||||||
WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
|
|
||||||
SOFT_WIP_TITLE_SQL_PREFILTER = ("%sketch%", "%doodle%", "%scribble%")
|
|
||||||
|
|
||||||
# Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
|
|
||||||
# ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
|
|
||||||
_INSERT_CHUNK = 5000
|
|
||||||
|
|
||||||
|
|
||||||
def matches_wip_title(title: str | None) -> bool:
|
|
||||||
"""True when a post title explicitly marks it work-in-progress (HARD tier)."""
|
|
||||||
if not title:
|
|
||||||
return False
|
|
||||||
return _WIP_RE.search(title) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def matches_soft_wip_title(title: str | None) -> bool:
|
|
||||||
"""True when a title carries a SOFT WIP cue (sketch/doodle/scribble, #1474)."""
|
|
||||||
if not title:
|
|
||||||
return False
|
|
||||||
return _SOFT_WIP_RE.search(title) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_wip_tag_id(session: Session) -> int | None:
|
|
||||||
"""The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
|
|
||||||
return session.execute(
|
|
||||||
select(Tag.id).where(Tag.name == WIP_SYSTEM_TAG, Tag.is_system.is_(True))
|
|
||||||
).scalar_one_or_none()
|
|
||||||
|
|
||||||
|
|
||||||
def apply_wip_image_tags(
|
|
||||||
session: Session, image_ids, tag_id: int, *, source: str = WIP_TITLE_SOURCE
|
|
||||||
) -> int:
|
|
||||||
"""Attach ``tag_id`` (stamped with ``source``) to each image id, idempotently —
|
|
||||||
never disturbs an existing tag or its source. Returns the number of image_tag
|
|
||||||
rows newly inserted. Does NOT commit.
|
|
||||||
|
|
||||||
The insert count is computed from a pre-SELECT of already-tagged ids rather
|
|
||||||
than the statement's ``rowcount``: psycopg reports -1 for a multi-row
|
|
||||||
ON CONFLICT DO NOTHING insert (it runs via an executemany path), so rowcount
|
|
||||||
is unusable here. The SELECT is accurate within this single transaction (no
|
|
||||||
concurrent writer touches these (image, wip) rows); ON CONFLICT DO NOTHING
|
|
||||||
stays as a race-safety belt so a rare concurrent insert can't error."""
|
|
||||||
ids = list({int(i) for i in image_ids})
|
|
||||||
if not ids:
|
|
||||||
return 0
|
|
||||||
inserted = 0
|
|
||||||
for start in range(0, len(ids), _INSERT_CHUNK):
|
|
||||||
chunk = ids[start:start + _INSERT_CHUNK]
|
|
||||||
already = set(session.execute(
|
|
||||||
select(image_tag.c.image_record_id)
|
|
||||||
.where(image_tag.c.tag_id == tag_id)
|
|
||||||
.where(image_tag.c.image_record_id.in_(chunk))
|
|
||||||
).scalars())
|
|
||||||
to_insert = [iid for iid in chunk if iid not in already]
|
|
||||||
if not to_insert:
|
|
||||||
continue
|
|
||||||
session.execute(
|
|
||||||
pg_insert(image_tag)
|
|
||||||
.values([
|
|
||||||
{"image_record_id": iid, "tag_id": tag_id, "source": source}
|
|
||||||
for iid in to_insert
|
|
||||||
])
|
|
||||||
.on_conflict_do_nothing(index_elements=["image_record_id", "tag_id"])
|
|
||||||
)
|
|
||||||
inserted += len(to_insert)
|
|
||||||
return inserted
|
|
||||||
@@ -76,19 +76,9 @@ DOWNLOAD_STALL_THRESHOLD_MINUTES = 30
|
|||||||
OLD_TASK_DAYS = 7
|
OLD_TASK_DAYS = 7
|
||||||
PHASH_PAGE = 500
|
PHASH_PAGE = 500
|
||||||
VERIFY_PAGE = 200
|
VERIFY_PAGE = 200
|
||||||
# Title-based WIP backfill (task #1458): posts scanned per keyset page.
|
|
||||||
WIP_BACKFILL_PAGE = 500
|
|
||||||
FFPROBE_TIMEOUT_SECONDS = 10
|
FFPROBE_TIMEOUT_SECONDS = 10
|
||||||
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
TASK_RUN_KEEP_OK_SECONDS = 24 * 3600 # 24 h
|
||||||
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
TASK_RUN_KEEP_FAILURE_SECONDS = 7 * 24 * 3600 # 7 days
|
||||||
# Orphaned staging files: downloads/imports stage into <name>.part / <name>.partial
|
|
||||||
# then os.replace() into place (importer / external_fetch / native_ingest_common /
|
|
||||||
# attachment_store), so a kill mid-write leaves a discardable temp, never a corrupt
|
|
||||||
# final. cleanup_orphaned_temp_files sweeps ones left behind; the min-age guard
|
|
||||||
# keeps it from deleting an in-flight download's staging file mid-write.
|
|
||||||
IMAGES_ROOT = Path("/images")
|
|
||||||
TEMP_STAGING_SUFFIXES = (".part", ".partial")
|
|
||||||
ORPHAN_TEMP_MIN_AGE_HOURS = 6
|
|
||||||
|
|
||||||
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
# Audit 2026-06-02: per-entity recovery sweep thresholds. Each must be
|
||||||
# > the entity's longest legitimate runtime (its task's time_limit + a
|
# > the entity's longest legitimate runtime (its task's time_limit + a
|
||||||
@@ -349,34 +339,6 @@ def cleanup_old_tasks() -> int:
|
|||||||
return result.rowcount or 0
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.cleanup_orphaned_temp_files")
|
|
||||||
def cleanup_orphaned_temp_files() -> int:
|
|
||||||
"""Delete orphaned .part/.partial staging files under the images root, left by
|
|
||||||
a download/import killed mid-write. Only removes files older than
|
|
||||||
ORPHAN_TEMP_MIN_AGE_HOURS so an in-flight download's staging file is never
|
|
||||||
pulled out from under it. Returns the count removed."""
|
|
||||||
if not IMAGES_ROOT.is_dir():
|
|
||||||
return 0
|
|
||||||
cutoff = datetime.now(UTC).timestamp() - ORPHAN_TEMP_MIN_AGE_HOURS * 3600
|
|
||||||
removed = 0
|
|
||||||
for path in IMAGES_ROOT.rglob("*"):
|
|
||||||
if path.suffix not in TEMP_STAGING_SUFFIXES or not path.is_file():
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
if path.stat().st_mtime >= cutoff:
|
|
||||||
continue # too fresh — may be an active download
|
|
||||||
path.unlink()
|
|
||||||
removed += 1
|
|
||||||
except OSError as exc:
|
|
||||||
log.warning("cleanup_orphaned_temp_files: %s: %s", path, exc)
|
|
||||||
if removed:
|
|
||||||
log.info(
|
|
||||||
"cleanup_orphaned_temp_files: removed %d orphaned staging file(s)",
|
|
||||||
removed,
|
|
||||||
)
|
|
||||||
return removed
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_task_runs")
|
||||||
def recover_stalled_task_runs() -> int:
|
def recover_stalled_task_runs() -> int:
|
||||||
"""Flip task_run rows stuck in 'running' past their queue-specific
|
"""Flip task_run rows stuck in 'running' past their queue-specific
|
||||||
@@ -776,62 +738,89 @@ def recover_stalled_library_audit_runs() -> int:
|
|||||||
return recovered
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
def _recover_stalled_runs(model, *, stall_minutes: int, keep_runs: int, label: str) -> int:
|
|
||||||
"""Shared recovery + retention sweep for the head run-tracking tables
|
|
||||||
(HeadTrainingRun / HeadAutoApplyRun, which share the
|
|
||||||
status/last_progress_at/started_at/finished_at/error/id columns): flip 'running'
|
|
||||||
rows with no progress past `stall_minutes` to 'error', then prune to the last
|
|
||||||
`keep_runs` (rule 89). Returns the number recovered. NOTE the two other recover
|
|
||||||
tasks are deliberately NOT folded in — library-audit has no prune tail and
|
|
||||||
backup uses a single started_at cutoff."""
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
now = datetime.now(UTC)
|
|
||||||
cutoff = now - timedelta(minutes=stall_minutes)
|
|
||||||
with SessionLocal() as session:
|
|
||||||
result = session.execute(
|
|
||||||
update(model)
|
|
||||||
.where(model.status == "running")
|
|
||||||
.where(func.coalesce(model.last_progress_at, model.started_at) < cutoff)
|
|
||||||
.values(
|
|
||||||
status="error", finished_at=now,
|
|
||||||
error=f"stranded by recovery sweep (no progress for {stall_minutes} min)",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
keep = session.execute(
|
|
||||||
select(model.id).order_by(model.id.desc()).limit(keep_runs)
|
|
||||||
).scalars().all()
|
|
||||||
if keep:
|
|
||||||
session.execute(delete(model).where(model.id.not_in(keep)))
|
|
||||||
session.commit()
|
|
||||||
recovered = result.rowcount or 0
|
|
||||||
if recovered:
|
|
||||||
log.info("%s: recovered %d rows", label, recovered)
|
|
||||||
return recovered
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_training_runs")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_training_runs")
|
||||||
def recover_stalled_head_training_runs() -> int:
|
def recover_stalled_head_training_runs() -> int:
|
||||||
"""Flip HeadTrainingRun rows stuck in 'running' past the stall threshold to
|
"""Flip HeadTrainingRun rows stuck in 'running' past the stall threshold to
|
||||||
'error', and prune old runs to the last HEAD_TRAINING_KEEP_RUNS (retention,
|
'error', and prune old runs to the last HEAD_TRAINING_KEEP_RUNS (retention,
|
||||||
rule 89). Runs every 5 min on the maintenance lane; no-op when idle."""
|
rule 89). Runs every 5 min on the maintenance lane; no-op when idle."""
|
||||||
return _recover_stalled_runs(
|
SessionLocal = _sync_session_factory()
|
||||||
HeadTrainingRun,
|
now = datetime.now(UTC)
|
||||||
stall_minutes=HEAD_TRAINING_STALL_THRESHOLD_MINUTES,
|
cutoff = now - timedelta(minutes=HEAD_TRAINING_STALL_THRESHOLD_MINUTES)
|
||||||
keep_runs=HEAD_TRAINING_KEEP_RUNS,
|
with SessionLocal() as session:
|
||||||
label="recover_stalled_head_training_runs",
|
result = session.execute(
|
||||||
)
|
update(HeadTrainingRun)
|
||||||
|
.where(HeadTrainingRun.status == "running")
|
||||||
|
.where(
|
||||||
|
func.coalesce(
|
||||||
|
HeadTrainingRun.last_progress_at, HeadTrainingRun.started_at
|
||||||
|
)
|
||||||
|
< cutoff
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
status="error", finished_at=now,
|
||||||
|
error=(
|
||||||
|
f"stranded by recovery sweep (no progress for "
|
||||||
|
f"{HEAD_TRAINING_STALL_THRESHOLD_MINUTES} min)"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
keep = session.execute(
|
||||||
|
select(HeadTrainingRun.id).order_by(HeadTrainingRun.id.desc())
|
||||||
|
.limit(HEAD_TRAINING_KEEP_RUNS)
|
||||||
|
).scalars().all()
|
||||||
|
if keep:
|
||||||
|
session.execute(
|
||||||
|
delete(HeadTrainingRun).where(HeadTrainingRun.id.not_in(keep))
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
recovered = result.rowcount or 0
|
||||||
|
if recovered:
|
||||||
|
log.info(
|
||||||
|
"recover_stalled_head_training_runs: recovered %d rows", recovered
|
||||||
|
)
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_auto_apply_runs")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_head_auto_apply_runs")
|
||||||
def recover_stalled_head_auto_apply_runs() -> int:
|
def recover_stalled_head_auto_apply_runs() -> int:
|
||||||
"""Flip stalled HeadAutoApplyRun 'running' rows to 'error' + prune to the
|
"""Flip stalled HeadAutoApplyRun 'running' rows to 'error' + prune to the
|
||||||
last HEAD_AUTO_APPLY_KEEP_RUNS (retention, rule 89). 5-min maintenance lane."""
|
last HEAD_AUTO_APPLY_KEEP_RUNS (retention, rule 89). 5-min maintenance lane."""
|
||||||
return _recover_stalled_runs(
|
SessionLocal = _sync_session_factory()
|
||||||
HeadAutoApplyRun,
|
now = datetime.now(UTC)
|
||||||
stall_minutes=HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES,
|
cutoff = now - timedelta(minutes=HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES)
|
||||||
keep_runs=HEAD_AUTO_APPLY_KEEP_RUNS,
|
with SessionLocal() as session:
|
||||||
label="recover_stalled_head_auto_apply_runs",
|
result = session.execute(
|
||||||
)
|
update(HeadAutoApplyRun)
|
||||||
|
.where(HeadAutoApplyRun.status == "running")
|
||||||
|
.where(
|
||||||
|
func.coalesce(
|
||||||
|
HeadAutoApplyRun.last_progress_at, HeadAutoApplyRun.started_at
|
||||||
|
)
|
||||||
|
< cutoff
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
status="error", finished_at=now,
|
||||||
|
error=(
|
||||||
|
f"stranded by recovery sweep (no progress for "
|
||||||
|
f"{HEAD_AUTO_APPLY_STALL_THRESHOLD_MINUTES} min)"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
keep = session.execute(
|
||||||
|
select(HeadAutoApplyRun.id).order_by(HeadAutoApplyRun.id.desc())
|
||||||
|
.limit(HEAD_AUTO_APPLY_KEEP_RUNS)
|
||||||
|
).scalars().all()
|
||||||
|
if keep:
|
||||||
|
session.execute(
|
||||||
|
delete(HeadAutoApplyRun).where(HeadAutoApplyRun.id.not_in(keep))
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
recovered = result.rowcount or 0
|
||||||
|
if recovered:
|
||||||
|
log.info(
|
||||||
|
"recover_stalled_head_auto_apply_runs: recovered %d rows", recovered
|
||||||
|
)
|
||||||
|
return recovered
|
||||||
|
|
||||||
|
|
||||||
# Keep ~6 months of daily head-metric snapshots (enough to see tuning trends).
|
# Keep ~6 months of daily head-metric snapshots (enough to see tuning trends).
|
||||||
@@ -1020,96 +1009,6 @@ def cleanup_old_download_events() -> int:
|
|||||||
return result.rowcount or 0
|
return result.rowcount or 0
|
||||||
|
|
||||||
|
|
||||||
def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
|
||||||
"""One keyset-paginated pass over posts whose title matches a WIP tier, applying
|
|
||||||
`tag_id` (stamped `source`) to their images. Shared by the hard + soft tiers
|
|
||||||
(#1458 / #1474). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
|
||||||
`matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
|
|
||||||
count newly applied."""
|
|
||||||
from ..models import Post
|
|
||||||
from ..models.image_provenance import ImageProvenance
|
|
||||||
from ..services.wip_title import apply_wip_image_tags
|
|
||||||
|
|
||||||
applied = 0
|
|
||||||
last_id = 0
|
|
||||||
while True:
|
|
||||||
rows = session.execute(
|
|
||||||
select(Post.id, Post.post_title)
|
|
||||||
.where(Post.id > last_id)
|
|
||||||
.where(Post.post_title.is_not(None))
|
|
||||||
.where(or_(*[Post.post_title.ilike(p) for p in prefilter]))
|
|
||||||
.order_by(Post.id.asc())
|
|
||||||
.limit(WIP_BACKFILL_PAGE)
|
|
||||||
).all()
|
|
||||||
if not rows:
|
|
||||||
break
|
|
||||||
last_id = rows[-1][0]
|
|
||||||
match_ids = [pid for pid, title in rows if matcher(title)]
|
|
||||||
if match_ids:
|
|
||||||
image_ids = session.execute(
|
|
||||||
select(ImageProvenance.image_record_id)
|
|
||||||
.where(ImageProvenance.post_id.in_(match_ids))
|
|
||||||
).scalars().all()
|
|
||||||
applied += apply_wip_image_tags(session, image_ids, tag_id, source=source)
|
|
||||||
session.commit()
|
|
||||||
return applied
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.maintenance.backfill_wip_title_tags",
|
|
||||||
# Coarse-prefiltered scan over posts; the candidate set is small on a typical
|
|
||||||
# library, but bound it like the other full-library sweeps.
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def backfill_wip_title_tags() -> int:
|
|
||||||
"""Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
|
|
||||||
images — the operator-triggered back-catalogue catch-up (task #1458 hard tier +
|
|
||||||
#1474 soft tier). New imports are tagged live by the importer; this covers the
|
|
||||||
existing library.
|
|
||||||
|
|
||||||
HARD tier ("WIP"/"work in progress") always runs (the operator triggered the
|
|
||||||
scan); the SOFT tier (sketch/doodle, provisional source) runs only when
|
|
||||||
wip_soft_title_tagging_enabled, AFTER hard so a title matching both keeps the
|
|
||||||
trained hard tag (ON CONFLICT DO NOTHING). Keyset-paginated, restart-safe.
|
|
||||||
|
|
||||||
Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
|
|
||||||
posts and silently undo a manual WIP removal, so it stays an explicit operator
|
|
||||||
action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
|
|
||||||
"""
|
|
||||||
from ..models import ImportSettings
|
|
||||||
from ..services.wip_title import (
|
|
||||||
SOFT_WIP_TITLE_SQL_PREFILTER,
|
|
||||||
WIP_TITLE_SOFT_SOURCE,
|
|
||||||
WIP_TITLE_SOURCE,
|
|
||||||
WIP_TITLE_SQL_PREFILTER,
|
|
||||||
matches_soft_wip_title,
|
|
||||||
matches_wip_title,
|
|
||||||
resolve_wip_tag_id,
|
|
||||||
)
|
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
tag_id = resolve_wip_tag_id(session)
|
|
||||||
if tag_id is None:
|
|
||||||
log.warning(
|
|
||||||
"backfill_wip_title_tags: no `wip` system tag present; nothing to do"
|
|
||||||
)
|
|
||||||
return 0
|
|
||||||
settings = ImportSettings.load_sync(session)
|
|
||||||
applied = _backfill_wip_tier(
|
|
||||||
session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
|
|
||||||
WIP_TITLE_SOURCE,
|
|
||||||
)
|
|
||||||
if settings.wip_soft_title_tagging_enabled:
|
|
||||||
applied += _backfill_wip_tier(
|
|
||||||
session, tag_id, SOFT_WIP_TITLE_SQL_PREFILTER, matches_soft_wip_title,
|
|
||||||
WIP_TITLE_SOFT_SOURCE,
|
|
||||||
)
|
|
||||||
if applied:
|
|
||||||
log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
|
|
||||||
return applied
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.vacuum_analyze")
|
@celery.task(name="backend.app.tasks.maintenance.vacuum_analyze")
|
||||||
def vacuum_analyze() -> dict:
|
def vacuum_analyze() -> dict:
|
||||||
"""Periodic VACUUM (ANALYZE) over the high-churn tables (VACUUM_TABLES) to
|
"""Periodic VACUUM (ANALYZE) over the high-churn tables (VACUUM_TABLES) to
|
||||||
|
|||||||
+31
-140
@@ -105,7 +105,9 @@ def embed_image(self, image_id: int) -> dict:
|
|||||||
record = session.get(ImageRecord, image_id)
|
record = session.get(ImageRecord, image_id)
|
||||||
if record is None:
|
if record is None:
|
||||||
return {"status": "missing", "image_id": image_id}
|
return {"status": "missing", "image_id": image_id}
|
||||||
settings = MLSettings.load_sync(session)
|
settings = session.execute(
|
||||||
|
select(MLSettings).where(MLSettings.id == 1)
|
||||||
|
).scalar_one()
|
||||||
|
|
||||||
src = Path(record.path)
|
src = Path(record.path)
|
||||||
is_vid = _is_video(src)
|
is_vid = _is_video(src)
|
||||||
@@ -326,11 +328,6 @@ def train_heads(self, run_id: int) -> str:
|
|||||||
run.status = "ready"
|
run.status = "ready"
|
||||||
run.finished_at = datetime.now(UTC)
|
run.finished_at = datetime.now(UTC)
|
||||||
session.commit()
|
session.commit()
|
||||||
# A retrain folds the day's accepts/rejects into the heads; refresh the
|
|
||||||
# CCIP character prototypes on the SAME trigger (#1317) so the Retrain
|
|
||||||
# button + nightly run keep character matching current too. Cheap no-op
|
|
||||||
# when references didn't change.
|
|
||||||
refresh_character_prototypes.delay()
|
|
||||||
return "ready"
|
return "ready"
|
||||||
|
|
||||||
|
|
||||||
@@ -354,10 +351,7 @@ def scheduled_train_heads() -> str:
|
|||||||
if running is not None:
|
if running is not None:
|
||||||
return "already running"
|
return "already running"
|
||||||
run = HeadTrainingRun(
|
run = HeadTrainingRun(
|
||||||
# Nightly = FULL reconcile (refit every head) so sampled-negative +
|
params={"source": "scheduled"}, status="running",
|
||||||
# hygiene drift is folded in; the manual Retrain button stays
|
|
||||||
# incremental (#1317 p2).
|
|
||||||
params={"source": "scheduled", "full": True}, status="running",
|
|
||||||
last_progress_at=datetime.now(UTC),
|
last_progress_at=datetime.now(UTC),
|
||||||
)
|
)
|
||||||
session.add(run)
|
session.add(run)
|
||||||
@@ -367,31 +361,6 @@ def scheduled_train_heads() -> str:
|
|||||||
return "dispatched"
|
return "dispatched"
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.ml.refresh_character_prototypes",
|
|
||||||
# Global-gate no-op when idle; a rebuild touches only changed characters. The
|
|
||||||
# initial (cold) build loads the whole reference set once, in the BACKGROUND
|
|
||||||
# — never on a /suggestions request.
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def refresh_character_prototypes(full: bool = False) -> str:
|
|
||||||
"""Incrementally refresh the CCIP character-prototype store (#1317, m138):
|
|
||||||
cheap global-gate skip when nothing changed, else rebuild only the characters
|
|
||||||
whose references moved. Triggered by a ~15-min beat, after every head retrain
|
|
||||||
(train_heads), and nightly with full=True. Returns a short status."""
|
|
||||||
from ..services.ml.character_prototypes import (
|
|
||||||
refresh_character_prototypes as _refresh,
|
|
||||||
)
|
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
result = _refresh(session, full=full)
|
|
||||||
return (
|
|
||||||
"skipped" if result["skipped"]
|
|
||||||
else f"rebuilt={result['rebuilt']} removed={result['removed']}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
@celery.task(
|
||||||
name="backend.app.tasks.ml.apply_head_tags",
|
name="backend.app.tasks.ml.apply_head_tags",
|
||||||
bind=True,
|
bind=True,
|
||||||
@@ -486,10 +455,15 @@ def scheduled_ccip_auto_apply() -> str:
|
|||||||
from sqlalchemy import select as sa_select
|
from sqlalchemy import select as sa_select
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
|
||||||
from ..models import ImageRegion, MLSettings, Tag, TagKind
|
from ..models import ImageRegion, MLSettings, Tag, TagKind, TagSuggestionRejection
|
||||||
from ..models.tag import image_tag
|
from ..models.tag import image_tag
|
||||||
from ..services.ml.ccip import _FIGURE_KINDS
|
|
||||||
from ..services.ml.training_data import _applied_or_rejected, _l2norm
|
fig = ("face", "figure")
|
||||||
|
|
||||||
|
def _l2(m):
|
||||||
|
n = np.linalg.norm(m, axis=1, keepdims=True)
|
||||||
|
n[n == 0] = 1.0
|
||||||
|
return m / n
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
SessionLocal = _sync_session_factory()
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
@@ -514,7 +488,7 @@ def scheduled_ccip_auto_apply() -> str:
|
|||||||
)
|
)
|
||||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||||
.where(Tag.kind == TagKind.character)
|
.where(Tag.kind == TagKind.character)
|
||||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
.where(ImageRegion.kind.in_(fig))
|
||||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||||
.where(ImageRegion.image_record_id.in_(single))
|
.where(ImageRegion.image_record_id.in_(single))
|
||||||
).all()
|
).all()
|
||||||
@@ -525,16 +499,29 @@ def scheduled_ccip_auto_apply() -> str:
|
|||||||
for tid, vec in ref_rows:
|
for tid, vec in ref_rows:
|
||||||
by_char.setdefault(tid, []).append(vec)
|
by_char.setdefault(tid, []).append(vec)
|
||||||
ref_tags = list(by_char)
|
ref_tags = list(by_char)
|
||||||
mats = [_l2norm(np.asarray(by_char[t], dtype=np.float32), np) for t in ref_tags]
|
mats = [_l2(np.asarray(by_char[t], dtype=np.float32)) for t in ref_tags]
|
||||||
allref = np.vstack(mats) # (total, 768)
|
allref = np.vstack(mats) # (total, 768)
|
||||||
seg = np.cumsum([0] + [len(m) for m in mats])[:-1] # per-char start
|
seg = np.cumsum([0] + [len(m) for m in mats])[:-1] # per-char start
|
||||||
|
|
||||||
# Per character: images that already carry OR rejected the tag — skip.
|
# Per character: images that already carry OR rejected the tag — skip.
|
||||||
skip = _applied_or_rejected(session, ref_tags)
|
skip = {t: set() for t in ref_tags}
|
||||||
|
for t in ref_tags:
|
||||||
|
for (iid,) in session.execute(
|
||||||
|
sa_select(image_tag.c.image_record_id).where(
|
||||||
|
image_tag.c.tag_id == t
|
||||||
|
)
|
||||||
|
):
|
||||||
|
skip[t].add(iid)
|
||||||
|
for (iid,) in session.execute(
|
||||||
|
sa_select(TagSuggestionRejection.image_record_id).where(
|
||||||
|
TagSuggestionRejection.tag_id == t
|
||||||
|
)
|
||||||
|
):
|
||||||
|
skip[t].add(iid)
|
||||||
|
|
||||||
img_ids = list(session.execute(
|
img_ids = list(session.execute(
|
||||||
sa_select(ImageRegion.image_record_id)
|
sa_select(ImageRegion.image_record_id)
|
||||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS), ImageRegion.ccip_embedding.is_not(None))
|
.where(ImageRegion.kind.in_(fig), ImageRegion.ccip_embedding.is_not(None))
|
||||||
.distinct()
|
.distinct()
|
||||||
).scalars())
|
).scalars())
|
||||||
|
|
||||||
@@ -546,7 +533,7 @@ def scheduled_ccip_auto_apply() -> str:
|
|||||||
sa_select(ImageRegion.image_record_id, ImageRegion.ccip_embedding)
|
sa_select(ImageRegion.image_record_id, ImageRegion.ccip_embedding)
|
||||||
.where(
|
.where(
|
||||||
ImageRegion.image_record_id.in_(chunk),
|
ImageRegion.image_record_id.in_(chunk),
|
||||||
ImageRegion.kind.in_(_FIGURE_KINDS),
|
ImageRegion.kind.in_(fig),
|
||||||
ImageRegion.ccip_embedding.is_not(None),
|
ImageRegion.ccip_embedding.is_not(None),
|
||||||
)
|
)
|
||||||
).all()
|
).all()
|
||||||
@@ -554,7 +541,7 @@ def scheduled_ccip_auto_apply() -> str:
|
|||||||
for iid, vec in rows:
|
for iid, vec in rows:
|
||||||
by_img.setdefault(iid, []).append(vec)
|
by_img.setdefault(iid, []).append(vec)
|
||||||
for iid, vecs in by_img.items():
|
for iid, vecs in by_img.items():
|
||||||
q = _l2norm(np.asarray(vecs, dtype=np.float32), np) # (nq, 768)
|
q = _l2(np.asarray(vecs, dtype=np.float32)) # (nq, 768)
|
||||||
colmax = (q @ allref.T).max(axis=0) # (total,)
|
colmax = (q @ allref.T).max(axis=0) # (total,)
|
||||||
charmax = np.maximum.reduceat(colmax, seg) # (n_chars,)
|
charmax = np.maximum.reduceat(colmax, seg) # (n_chars,)
|
||||||
for ci in np.where(charmax >= thr)[0]:
|
for ci in np.where(charmax >= thr)[0]:
|
||||||
@@ -572,99 +559,3 @@ def scheduled_ccip_auto_apply() -> str:
|
|||||||
applied += 1
|
applied += 1
|
||||||
session.commit()
|
session.commit()
|
||||||
return f"applied={applied}"
|
return f"applied={applied}"
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.ml.scheduled_presentation_auto_apply",
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def scheduled_presentation_auto_apply() -> str:
|
|
||||||
"""Auto-hide presentation chrome (banner) on a daily passive sweep (#141).
|
|
||||||
No-op unless presentation_auto_apply_enabled. Idempotent — already-tagged images
|
|
||||||
are skipped — so an interrupted run simply re-runs next cycle (that IS the
|
|
||||||
recovery). Wall-clock bounded by the task time limits."""
|
|
||||||
from ..services.ml.heads import system_tag_auto_apply_sweep
|
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
result = system_tag_auto_apply_sweep(session, mode="chrome")
|
|
||||||
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.ml.scheduled_process_auto_apply",
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def scheduled_process_auto_apply() -> str:
|
|
||||||
"""Auto-apply the PROCESS system tags (wip / editor screenshot) on a daily
|
|
||||||
passive sweep (#1464) — provisional source, ring-loud review guard, image stays
|
|
||||||
VISIBLE. No-op unless process_auto_apply_enabled (opt-in). Idempotent —
|
|
||||||
already-tagged/rejected images are skipped — so an interrupted run just re-runs
|
|
||||||
next cycle (the recovery). Wall-clock bounded by the task time limits."""
|
|
||||||
from ..services.ml.heads import system_tag_auto_apply_sweep
|
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
result = system_tag_auto_apply_sweep(session, mode="process")
|
|
||||||
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def scheduled_soft_wip_conflict_audit() -> str:
|
|
||||||
"""Ring-loud audit over the SOFT WIP-title cohort (#1474) — flag sketch/doodle
|
|
||||||
auto-tags that ALSO look like real content for review. No-op when there are no
|
|
||||||
content heads; idempotent (already-flagged images skipped). Runs regardless of
|
|
||||||
the process-sweep toggle, since soft-title tags come from the importer, not that
|
|
||||||
sweep. Wall-clock bounded by the task time limits."""
|
|
||||||
from ..services.ml.heads import soft_wip_conflict_audit
|
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
result = soft_wip_conflict_audit(session)
|
|
||||||
return f"scanned={result['n_scanned']} flagged={result['n_flagged']}"
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
|
|
||||||
def prune_presentation_reviews() -> str:
|
|
||||||
"""Retention (rule 89): drop RESOLVED presentation-review flags older than 30
|
|
||||||
days — the operator has acted on them, so they're just history (#141)."""
|
|
||||||
from datetime import UTC, datetime, timedelta
|
|
||||||
|
|
||||||
from sqlalchemy import delete as sa_delete
|
|
||||||
|
|
||||||
from ..models import PresentationReview
|
|
||||||
|
|
||||||
cutoff = datetime.now(UTC) - timedelta(days=30)
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
res = session.execute(
|
|
||||||
sa_delete(PresentationReview).where(
|
|
||||||
PresentationReview.resolved_at.is_not(None),
|
|
||||||
PresentationReview.resolved_at < cutoff,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
session.commit()
|
|
||||||
return f"pruned={res.rowcount}"
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.ml.scheduled_retract_auto_tags",
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def scheduled_retract_auto_tags() -> str:
|
|
||||||
"""Soft auto-apply (milestone 139): retract standing head_auto/ccip_auto tags
|
|
||||||
the model no longer supports (score now below the auto-apply threshold),
|
|
||||||
skipping operator-confirmed ones. Silent (no hard negative). No-op unless the
|
|
||||||
respective auto-apply switch is on. Returns 'head=N ccip=M'."""
|
|
||||||
from ..services.ml.character_prototypes import retract_auto_applied_ccip
|
|
||||||
from ..services.ml.heads import retract_auto_applied_heads
|
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
n_head = retract_auto_applied_heads(session)
|
|
||||||
with SessionLocal() as session:
|
|
||||||
n_ccip = retract_auto_applied_ccip(session)
|
|
||||||
return f"head={n_head} ccip={n_ccip}"
|
|
||||||
|
|||||||
@@ -1,398 +0,0 @@
|
|||||||
"""Post-text translation Celery tasks (milestone 143 + re-translate m146).
|
|
||||||
|
|
||||||
Backfills non-English post title/description to the target language via the
|
|
||||||
self-hosted Interpreter LAN service, storing the result + engine_version so
|
|
||||||
viewing is instant and re-runs are cache-fast. Sync (Celery workers are sync),
|
|
||||||
same pattern as the other backfill sweeps. No-op unless translation is enabled,
|
|
||||||
a base URL is set, and the service is healthy.
|
|
||||||
|
|
||||||
Curator does NO language detection of its own: each field's translation is
|
|
||||||
accepted or rejected purely on Interpreter's reported engine + detected language
|
|
||||||
+ confidence (see ``_accept``) — a short English title Interpreter mis-labels as
|
|
||||||
a European language scores below the latin-script floor and is kept as-is.
|
|
||||||
|
|
||||||
Two entry points:
|
|
||||||
- ``translate_posts`` — the periodic untranslated sweep (every 8h;
|
|
||||||
translated_source_lang IS NULL only), one bounded chunk per fire. The Settings
|
|
||||||
"Translate now" button calls it with ``drain=True`` to chase the tail
|
|
||||||
run-until-done and clear the whole backlog in a single press.
|
|
||||||
- ``retranslate_posts`` — after a model change, resets the stored translation
|
|
||||||
columns for a scoped set of posts (all, or a given set of artists) so the
|
|
||||||
untranslated selection re-runs them, then chases the tail until drained
|
|
||||||
(run-until-done). The Interpreter cache keys on engine_version: a changed
|
|
||||||
model genuinely re-translates, an unchanged one is ~1ms cache-fast.
|
|
||||||
"""
|
|
||||||
import logging
|
|
||||||
from datetime import UTC, datetime
|
|
||||||
|
|
||||||
from celery.exceptions import SoftTimeLimitExceeded
|
|
||||||
from sqlalchemy import func, or_, select, update
|
|
||||||
|
|
||||||
from ..celery_app import celery
|
|
||||||
from ..models import ImportSettings, Post
|
|
||||||
from ..services import interpreter_client as ic
|
|
||||||
from ..utils.text import html_to_plain
|
|
||||||
from ._sync_engine import sync_session_factory as _sync_session_factory
|
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
# Bound one run's work so it commits progress rather than dying at the time
|
|
||||||
# limit; the rest resumes next cycle (idempotent — only untranslated posts are
|
|
||||||
# picked, so an interrupted run just re-runs = rule-89 recovery).
|
|
||||||
_MAX_POSTS_PER_RUN = 300
|
|
||||||
# Short gap between run-until-done chunks so a big re-translate finishes on its
|
|
||||||
# own without monopolising the maintenance_long worker in one uninterrupted run.
|
|
||||||
_RETRANSLATE_COUNTDOWN = 5
|
|
||||||
# When Interpreter drains mid-chunk (graceful restart), resume the bulk
|
|
||||||
# re-translate after the service's Retry-After hint if it sent one, else this
|
|
||||||
# default; capped so a bogus/huge hint can't park the work longer than the daily
|
|
||||||
# beat's own safety net would.
|
|
||||||
_INTERRUPT_BACKOFF = 60
|
|
||||||
_INTERRUPT_BACKOFF_MAX = 900
|
|
||||||
|
|
||||||
# --- Translation acceptance gate (Scribe rule 133; contract note #1347) -------
|
|
||||||
# Curator does NO language detection of its own — it accepts or rejects a
|
|
||||||
# translation purely on Interpreter's reported detection. CJK is script-detected
|
|
||||||
# (kana/hangul/kanji) and reliably high-confidence, so ja/ko/zh are trusted
|
|
||||||
# outright (pure-kanji Japanese can even come back labelled zh ~0.75, still
|
|
||||||
# correctly translated). Latin-script detection is a real statistical
|
|
||||||
# probability, so a short English title Interpreter mis-labels as a European
|
|
||||||
# language scores low; require it to clear this floor, else keep the original.
|
|
||||||
# Calibrated 2026-07-09 against fresh (uncached) probes once Interpreter returned
|
|
||||||
# real langdetect confidence: genuine German detected at 1.0, while a correctly-
|
|
||||||
# detected but ambiguous latin string dipped to 0.86 — so a single constant can't win.
|
|
||||||
# The floor is now operator-tunable (ImportSettings.translation_min_confidence,
|
|
||||||
# milestone 155) with a stricter 0.90 default; per-post overrides
|
|
||||||
# (Post.translation_override) rescue the false negatives it skips. Genuine German
|
|
||||||
# and mis-flagged short English both land ~0.86, and single-word mis-flags sit at
|
|
||||||
# a confident 1.0 no floor catches. Real langdetect fixed some cases at the
|
|
||||||
# source (some short English titles now detect as English → passthrough), so this
|
|
||||||
# floor is a safety net, not the primary fix. Re-tune via the "Test translation"
|
|
||||||
# box (send fresh text — cache hits report 1.0).
|
|
||||||
_CJK_LANGS = frozenset({"ja", "ko", "zh"})
|
|
||||||
_DEFAULT_MIN_LATIN_CONFIDENCE = 0.90
|
|
||||||
|
|
||||||
|
|
||||||
def _interrupt_backoff(retry_after) -> int:
|
|
||||||
"""Seconds to wait before resuming after an Interpreter interruption: the
|
|
||||||
server's Retry-After hint (capped), else the default."""
|
|
||||||
if retry_after and retry_after > 0:
|
|
||||||
return int(min(retry_after, _INTERRUPT_BACKOFF_MAX))
|
|
||||||
return _INTERRUPT_BACKOFF
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.translation.translate_posts",
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def translate_posts(drain: bool = False) -> str:
|
|
||||||
"""Translate untranslated non-English post title/description (default target
|
|
||||||
en) via Interpreter. Per-post [title, description] batch → per-post detected
|
|
||||||
language. Passthrough / already-target posts are marked handled (source ==
|
|
||||||
target) with no stored translation. 503 or a connection error interrupts the
|
|
||||||
run (retry next cycle); 400 stops it (fix config); posts done so far stay
|
|
||||||
committed. No-op unless configured + healthy. Returns a summary string.
|
|
||||||
|
|
||||||
``drain`` (the Settings "Translate now" button) chases the tail
|
|
||||||
run-until-done: on a clean chunk with work still remaining it re-enqueues
|
|
||||||
itself until the untranslated backlog is zero, like ``retranslate_posts``. The
|
|
||||||
periodic beat leaves it False → one bounded chunk per fire (the 8h cadence
|
|
||||||
drives the rest, enough for the trickle of newly-imported posts)."""
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
ready = _translation_config(session)
|
|
||||||
if isinstance(ready, str):
|
|
||||||
return ready
|
|
||||||
base_url, target, min_confidence = ready
|
|
||||||
posts = _select_untranslated(session, None, _MAX_POSTS_PER_RUN)
|
|
||||||
status, translated, retry_after = _translate_batch(
|
|
||||||
session, posts, base_url, target, min_confidence,
|
|
||||||
)
|
|
||||||
# Drain mode (manual "Translate now"): chase the tail until the backlog is
|
|
||||||
# zero, so one press clears the whole pile instead of one 300-chunk.
|
|
||||||
# Termination is guaranteed — every handled post leaves the untranslated
|
|
||||||
# set, so the remaining count strictly decreases each chunk.
|
|
||||||
if status == "ok" and drain:
|
|
||||||
remaining = _count_untranslated(session, None)
|
|
||||||
if remaining:
|
|
||||||
translate_posts.apply_async(
|
|
||||||
(), {"drain": True}, countdown=_RETRANSLATE_COUNTDOWN,
|
|
||||||
)
|
|
||||||
log.info(
|
|
||||||
"translate_posts: draining — %d remaining → re-enqueued",
|
|
||||||
remaining,
|
|
||||||
)
|
|
||||||
# Interpreter drained mid-chunk (graceful restart): don't make a manual
|
|
||||||
# "Translate now" wait a whole beat cycle — re-enqueue after its
|
|
||||||
# Retry-After hint (or the default backoff), preserving `drain` so a bulk
|
|
||||||
# drain resumes cleanly. Self-terminating: once the service is down the
|
|
||||||
# health gate returns "interpreter unavailable" early.
|
|
||||||
elif status == "interrupted" and _count_untranslated(session, None):
|
|
||||||
delay = _interrupt_backoff(retry_after)
|
|
||||||
translate_posts.apply_async((), {"drain": drain}, countdown=delay)
|
|
||||||
log.info(
|
|
||||||
"translate_posts: interrupted (service draining) → retry in %ss",
|
|
||||||
delay,
|
|
||||||
)
|
|
||||||
return _summary(status, translated, len(posts))
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.translation.retranslate_posts",
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def retranslate_posts(artist_ids=None, _reset_done=False) -> str:
|
|
||||||
"""Re-translate posts after a model change. On the first call resets the
|
|
||||||
stored translation columns to NULL for the scoped posts — all artists when
|
|
||||||
``artist_ids`` is falsy, else ``WHERE artist_id IN artist_ids`` — so the
|
|
||||||
normal untranslated selection re-runs them through Interpreter. Then handles
|
|
||||||
one bounded chunk and re-enqueues itself until the scoped backlog is drained
|
|
||||||
(run-until-done; ``_reset_done`` guards against wiping fresh work on the
|
|
||||||
follow-up chunks). No reset happens unless the service is configured +
|
|
||||||
healthy, so translations are never cleared when they can't be rebuilt."""
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
ready = _translation_config(session)
|
|
||||||
if isinstance(ready, str):
|
|
||||||
return ready
|
|
||||||
base_url, target, min_confidence = ready
|
|
||||||
|
|
||||||
if not _reset_done:
|
|
||||||
n = _reset_translations(session, artist_ids)
|
|
||||||
log.info(
|
|
||||||
"retranslate_posts: reset %d post(s) (artist_ids=%s)",
|
|
||||||
n, artist_ids,
|
|
||||||
)
|
|
||||||
|
|
||||||
posts = _select_untranslated(session, artist_ids, _MAX_POSTS_PER_RUN)
|
|
||||||
status, translated, retry_after = _translate_batch(
|
|
||||||
session, posts, base_url, target, min_confidence,
|
|
||||||
)
|
|
||||||
|
|
||||||
# run-until-done: chase the tail when the chunk finished cleanly.
|
|
||||||
# Termination is guaranteed: every handled post leaves the NULL set, so
|
|
||||||
# the remaining count strictly decreases each chunk.
|
|
||||||
if status == "ok":
|
|
||||||
remaining = _count_untranslated(session, artist_ids)
|
|
||||||
if remaining:
|
|
||||||
retranslate_posts.apply_async(
|
|
||||||
(),
|
|
||||||
{"artist_ids": artist_ids, "_reset_done": True},
|
|
||||||
countdown=_RETRANSLATE_COUNTDOWN,
|
|
||||||
)
|
|
||||||
log.info(
|
|
||||||
"retranslate_posts: %d remaining → re-enqueued", remaining,
|
|
||||||
)
|
|
||||||
# Interpreter drained mid-chunk (graceful restart / 429 / 5xx): don't
|
|
||||||
# stall the bulk re-translate until the daily beat — resume it after the
|
|
||||||
# service's Retry-After hint (or the default backoff), with
|
|
||||||
# _reset_done=True so the fresh work is never re-wiped. Self-terminating:
|
|
||||||
# if the service is still down next run, _translation_config's health gate
|
|
||||||
# returns "interpreter unavailable" early and nothing re-enqueues.
|
|
||||||
elif status == "interrupted" and _count_untranslated(session, artist_ids):
|
|
||||||
delay = _interrupt_backoff(retry_after)
|
|
||||||
retranslate_posts.apply_async(
|
|
||||||
(),
|
|
||||||
{"artist_ids": artist_ids, "_reset_done": True},
|
|
||||||
countdown=delay,
|
|
||||||
)
|
|
||||||
log.info(
|
|
||||||
"retranslate_posts: interrupted (service draining) → retry in %ss",
|
|
||||||
delay,
|
|
||||||
)
|
|
||||||
return _summary(status, translated, len(posts))
|
|
||||||
|
|
||||||
|
|
||||||
def _translation_config(session):
|
|
||||||
"""Resolve (base_url, target, min_confidence) if translation is enabled +
|
|
||||||
healthy, else a short status string ("disabled" / "interpreter unavailable")
|
|
||||||
for the caller to return. Centralises the guard so translate/retranslate agree
|
|
||||||
exactly, including the operator-tunable acceptance floor."""
|
|
||||||
cfg = ImportSettings.load_sync(session)
|
|
||||||
if not cfg.translation_enabled or not cfg.interpreter_base_url.strip():
|
|
||||||
return "disabled"
|
|
||||||
base_url = cfg.interpreter_base_url.strip()
|
|
||||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
|
||||||
min_confidence = cfg.translation_min_confidence
|
|
||||||
if not ic.health(base_url):
|
|
||||||
return "interpreter unavailable"
|
|
||||||
return base_url, target, min_confidence
|
|
||||||
|
|
||||||
|
|
||||||
def _untranslated_filter(stmt, artist_ids):
|
|
||||||
"""Add the untranslated-post predicate (+ optional artist scope) to a
|
|
||||||
select/count over Post. Untranslated = translated_source_lang IS NULL with
|
|
||||||
some text to translate."""
|
|
||||||
stmt = stmt.where(
|
|
||||||
Post.translated_source_lang.is_(None)
|
|
||||||
).where(or_(
|
|
||||||
Post.post_title.is_not(None), Post.description.is_not(None),
|
|
||||||
))
|
|
||||||
if artist_ids:
|
|
||||||
stmt = stmt.where(Post.artist_id.in_(artist_ids))
|
|
||||||
return stmt
|
|
||||||
|
|
||||||
|
|
||||||
def _select_untranslated(session, artist_ids, limit):
|
|
||||||
return session.execute(
|
|
||||||
_untranslated_filter(select(Post), artist_ids).limit(limit)
|
|
||||||
).scalars().all()
|
|
||||||
|
|
||||||
|
|
||||||
def _count_untranslated(session, artist_ids) -> int:
|
|
||||||
return int(session.execute(
|
|
||||||
_untranslated_filter(select(func.count(Post.id)), artist_ids)
|
|
||||||
).scalar_one())
|
|
||||||
|
|
||||||
|
|
||||||
def _reset_translations(session, artist_ids) -> int:
|
|
||||||
"""Clear the stored translation columns for scoped, already-handled posts so
|
|
||||||
the untranslated sweep re-runs them. Only touches rows that were translated
|
|
||||||
(translated_source_lang IS NOT NULL) — untranslated rows are already NULL.
|
|
||||||
Skips 'keep original' posts (translation_override = 'original') so that choice
|
|
||||||
survives a Re-translate-all (milestone 155). Returns the row count reset
|
|
||||||
(commits it)."""
|
|
||||||
stmt = (
|
|
||||||
update(Post)
|
|
||||||
.where(Post.translated_source_lang.is_not(None))
|
|
||||||
.where(Post.translation_override != "original")
|
|
||||||
.values(
|
|
||||||
post_title_translated=None,
|
|
||||||
description_translated=None,
|
|
||||||
translated_source_lang=None,
|
|
||||||
translation_engine_version=None,
|
|
||||||
translated_at=None,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if artist_ids:
|
|
||||||
stmt = stmt.where(Post.artist_id.in_(artist_ids))
|
|
||||||
n = session.execute(stmt).rowcount
|
|
||||||
session.commit()
|
|
||||||
return n
|
|
||||||
|
|
||||||
|
|
||||||
def _translate_batch(session, posts, base_url: str, target: str, min_confidence: float):
|
|
||||||
"""Translate each post in the chunk with a per-post commit (an interrupted
|
|
||||||
run keeps progress). Returns (status, translated, retry_after) where status is
|
|
||||||
one of "ok" / "interrupted" (unavailable/drain) / "stopped" (400) / "timeout";
|
|
||||||
retry_after is the server's Retry-After hint in seconds on an interrupt, else
|
|
||||||
None."""
|
|
||||||
translated = 0
|
|
||||||
for post in posts:
|
|
||||||
try:
|
|
||||||
translated += _translate_one(session, post, base_url, target, min_confidence)
|
|
||||||
session.commit()
|
|
||||||
except ic.InterpreterUnavailable as e:
|
|
||||||
session.rollback()
|
|
||||||
return "interrupted", translated, getattr(e, "retry_after", None)
|
|
||||||
except ic.InterpreterBadRequest as e:
|
|
||||||
session.rollback()
|
|
||||||
log.warning("translate bad request: %s", e)
|
|
||||||
return "stopped", translated, None
|
|
||||||
except SoftTimeLimitExceeded:
|
|
||||||
return "timeout", translated, None
|
|
||||||
return "ok", translated, None
|
|
||||||
|
|
||||||
|
|
||||||
def _summary(status: str, translated: int, scanned: int) -> str:
|
|
||||||
if status == "interrupted":
|
|
||||||
return f"interrupted (service down) — translated={translated}"
|
|
||||||
if status == "stopped":
|
|
||||||
return f"stopped (bad request) — translated={translated}"
|
|
||||||
if status == "timeout":
|
|
||||||
return f"time limit — translated={translated}"
|
|
||||||
return f"translated={translated} scanned={scanned}"
|
|
||||||
|
|
||||||
|
|
||||||
def _accept(
|
|
||||||
language: str, confidence, min_confidence: float = _DEFAULT_MIN_LATIN_CONFIDENCE,
|
|
||||||
) -> bool:
|
|
||||||
"""Should curator store Interpreter's translation, or keep the original?
|
|
||||||
Consumes ONLY Interpreter's own detection (curator does none, Scribe rule
|
|
||||||
133): CJK languages are trusted regardless of the reported number; a
|
|
||||||
latin-script detection must clear ``min_confidence`` (the operator-tunable
|
|
||||||
ImportSettings.translation_min_confidence). A missing confidence fails OPEN
|
|
||||||
(accept), so a service that omits the field never silently drops a
|
|
||||||
translation. Per-post ``force`` overrides bypass this entirely (see
|
|
||||||
``_translate_one``)."""
|
|
||||||
if (language or "").split("-", 1)[0].lower() in _CJK_LANGS:
|
|
||||||
return True
|
|
||||||
if confidence is None:
|
|
||||||
return True
|
|
||||||
return confidence >= min_confidence
|
|
||||||
|
|
||||||
|
|
||||||
def _translate_field(
|
|
||||||
text: str, base_url: str, target: str, min_confidence: float, force: bool = False,
|
|
||||||
):
|
|
||||||
"""Translate ONE field independently. Returns (translated, source_lang,
|
|
||||||
engine_version), or (None, None, None) when there's nothing to store — empty
|
|
||||||
text, already the target language, a passthrough (engine "none"), or a
|
|
||||||
latin-script detection the acceptance gate rejects as too low-confidence (kept
|
|
||||||
as the original). ``force`` (a per-post 'force' override) bypasses the
|
|
||||||
confidence gate so a legitimately-foreign title Interpreter is only ~0.86 sure
|
|
||||||
about is still stored; a genuine passthrough stores nothing regardless (there
|
|
||||||
is no translation to force). Per-field (not aggregate first-item) detection so
|
|
||||||
a non-English description still translates when the title is already English."""
|
|
||||||
if not text:
|
|
||||||
return None, None, None
|
|
||||||
res = ic.translate([text], base_url=base_url, target=target)
|
|
||||||
detected = res["detected_lang"] or target
|
|
||||||
if detected == target or res["engine"] == "none":
|
|
||||||
return None, None, None
|
|
||||||
# Trust only Interpreter's own detection (Scribe rule 133): keep the original
|
|
||||||
# when a latin-script detection doesn't clear the confidence floor, so a short
|
|
||||||
# English title mis-labelled as e.g. German isn't rewritten into the archive.
|
|
||||||
# A 'force' override skips the gate — the operator is overriding a rejection.
|
|
||||||
if not force and not _accept(
|
|
||||||
detected, res.get("detected_confidence"), min_confidence,
|
|
||||||
):
|
|
||||||
return None, None, None
|
|
||||||
return res["translations"][0], detected, res["engine_version"]
|
|
||||||
|
|
||||||
|
|
||||||
def _store_translation(post, title_res, desc_res, target: str) -> int:
|
|
||||||
"""Apply per-field (translated, source_lang, engine_version) results to a post
|
|
||||||
in place. Returns 1 if a translation was stored, 0 when the post is all
|
|
||||||
passthrough/empty — in which case the translated columns are CLEARED and the
|
|
||||||
post is marked handled (translated_source_lang = target). Clearing (not just
|
|
||||||
leaving) matters on re-apply: a 'keep original' override, or a re-translate
|
|
||||||
that now rejects a previously-accepted mis-flag, must remove the stale
|
|
||||||
translation, not keep it."""
|
|
||||||
title_tr, title_lang, title_ev = title_res
|
|
||||||
desc_tr, desc_lang, desc_ev = desc_res
|
|
||||||
if title_tr is None and desc_tr is None:
|
|
||||||
post.post_title_translated = None
|
|
||||||
post.description_translated = None
|
|
||||||
post.translated_source_lang = target
|
|
||||||
post.translation_engine_version = None
|
|
||||||
post.translated_at = None
|
|
||||||
return 0
|
|
||||||
post.post_title_translated = title_tr
|
|
||||||
post.description_translated = desc_tr
|
|
||||||
# Source lang = whichever field was actually non-target (for a mixed post,
|
|
||||||
# the translated field's language, not the already-English one).
|
|
||||||
post.translated_source_lang = title_lang or desc_lang
|
|
||||||
post.translation_engine_version = title_ev or desc_ev
|
|
||||||
post.translated_at = datetime.now(UTC)
|
|
||||||
return 1
|
|
||||||
|
|
||||||
|
|
||||||
def _translate_one(
|
|
||||||
session, post, base_url: str, target: str, min_confidence: float,
|
|
||||||
) -> int:
|
|
||||||
"""Translate a post's title/description in place, each field independently,
|
|
||||||
honoring the per-post override (milestone 155): 'original' keeps the original
|
|
||||||
(clears any stored translation, no Interpreter call); 'force' translates even
|
|
||||||
below the confidence floor; 'auto' (default) runs the acceptance gate. Returns
|
|
||||||
1 if it stored a translation, else 0 (still marks the post handled so the sweep
|
|
||||||
won't revisit it)."""
|
|
||||||
if post.translation_override == "original":
|
|
||||||
return _store_translation(post, (None, None, None), (None, None, None), target)
|
|
||||||
force = post.translation_override == "force"
|
|
||||||
title = (post.post_title or "").strip()
|
|
||||||
desc = (html_to_plain(post.description) if post.description else "") or ""
|
|
||||||
desc = desc.strip()
|
|
||||||
title_res = _translate_field(title, base_url, target, min_confidence, force=force)
|
|
||||||
desc_res = _translate_field(desc, base_url, target, min_confidence, force=force)
|
|
||||||
return _store_translation(post, title_res, desc_res, target)
|
|
||||||
@@ -8,35 +8,6 @@
|
|||||||
# run `docker compose up` from this directory and switches images to
|
# run `docker compose up` from this directory and switches images to
|
||||||
# local builds + DEBUG logging.
|
# local builds + DEBUG logging.
|
||||||
|
|
||||||
# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a
|
|
||||||
# time, START the new task before stopping the old (zero-downtime via the ingress
|
|
||||||
# mesh), and if the new task doesn't reach a healthy state within `monitor`, roll
|
|
||||||
# back to the previous image automatically. `monitor` is sized above web's
|
|
||||||
# healthcheck start_period so a broken image that never goes healthy is caught.
|
|
||||||
# Plain `docker compose up` ignores `deploy:` (it warns + skips), so the dev
|
|
||||||
# override is unaffected. Referenced by each long-lived service below.
|
|
||||||
x-deploy-policy: &deploy_policy
|
|
||||||
update_config:
|
|
||||||
order: start-first
|
|
||||||
failure_action: rollback
|
|
||||||
monitor: 90s
|
|
||||||
rollback_config:
|
|
||||||
order: start-first
|
|
||||||
restart_policy:
|
|
||||||
condition: any
|
|
||||||
delay: 10s
|
|
||||||
|
|
||||||
# Worker liveness: ping THIS container's celery node over the broker. Lenient
|
|
||||||
# (60s interval, 3 retries, 60s start_period) so a transient broker blip never
|
|
||||||
# false-flags a worker into a rollback. `$$HOSTNAME` → `$HOSTNAME` for the shell;
|
|
||||||
# celery's default node name is celery@<hostname> (the container id).
|
|
||||||
x-celery-healthcheck: &celery_healthcheck
|
|
||||||
test: ["CMD-SHELL", "celery -A backend.app.celery_app:celery inspect ping -d celery@$$HOSTNAME --timeout 10 >/dev/null 2>&1"]
|
|
||||||
interval: 60s
|
|
||||||
timeout: 15s
|
|
||||||
retries: 3
|
|
||||||
start_period: 60s
|
|
||||||
|
|
||||||
services:
|
services:
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:7-alpine
|
||||||
@@ -76,24 +47,6 @@ services:
|
|||||||
web:
|
web:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||||
command: ["web"]
|
command: ["web"]
|
||||||
# Graceful shutdown: give the container time to drain in-flight work on a
|
|
||||||
# deploy (docker SIGTERMs, then SIGKILLs after this window — default is only
|
|
||||||
# 10s, far too short for real jobs). Hypercorn/Celery both warm-shut-down on
|
|
||||||
# SIGTERM; per-lane values sized to typical task length. Anything that still
|
|
||||||
# outruns the window is re-queued (task_reject_on_worker_lost) and re-driven
|
|
||||||
# by the 5-min recovery sweeps, so a kill never corrupts. web = short HTTP
|
|
||||||
# requests + the occasional file download.
|
|
||||||
stop_grace_period: 30s
|
|
||||||
# Liveness for rolling deploys: /api/health is a no-DB 200 (just proves the
|
|
||||||
# app booted + serves HTTP after `alembic upgrade head`). start_period covers
|
|
||||||
# the migration + boot so a slow start isn't mis-flagged.
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8080/api/health', timeout=5).status==200 else 1)\""]
|
|
||||||
interval: 15s
|
|
||||||
timeout: 6s
|
|
||||||
retries: 3
|
|
||||||
start_period: 40s
|
|
||||||
deploy: *deploy_policy
|
|
||||||
ports:
|
ports:
|
||||||
- "${PORT:-8080}:8080"
|
- "${PORT:-8080}:8080"
|
||||||
environment: &app_env
|
environment: &app_env
|
||||||
@@ -124,10 +77,6 @@ services:
|
|||||||
worker:
|
worker:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||||
command: ["worker"]
|
command: ["worker"]
|
||||||
# Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
|
|
||||||
stop_grace_period: 90s
|
|
||||||
healthcheck: *celery_healthcheck
|
|
||||||
deploy: *deploy_policy
|
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
CELERY_QUEUES: default,import,thumbnail,download
|
CELERY_QUEUES: default,import,thumbnail,download
|
||||||
@@ -144,10 +93,6 @@ services:
|
|||||||
scheduler:
|
scheduler:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||||
command: ["scheduler"]
|
command: ["scheduler"]
|
||||||
# Quick maintenance/scan lane + beat — short tasks, modest drain window.
|
|
||||||
stop_grace_period: 60s
|
|
||||||
healthcheck: *celery_healthcheck
|
|
||||||
deploy: *deploy_policy
|
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
CELERY_QUEUES: maintenance,scan
|
CELERY_QUEUES: maintenance,scan
|
||||||
@@ -165,12 +110,6 @@ services:
|
|||||||
maintenance-long:
|
maintenance-long:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||||
command: ["worker"]
|
command: ["worker"]
|
||||||
# Longest lane (DB backups, library audits, translation backfill) — give it
|
|
||||||
# the most room to finish a chunk gracefully. Chunked + idempotent, so a job
|
|
||||||
# that still outruns this resumes cleanly next run rather than corrupting.
|
|
||||||
stop_grace_period: 180s
|
|
||||||
healthcheck: *celery_healthcheck
|
|
||||||
deploy: *deploy_policy
|
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
CELERY_QUEUES: maintenance_long
|
CELERY_QUEUES: maintenance_long
|
||||||
@@ -186,10 +125,6 @@ services:
|
|||||||
ml-worker:
|
ml-worker:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev
|
||||||
command: ["ml-worker"]
|
command: ["ml-worker"]
|
||||||
# A single GPU inference pass can run tens of seconds — let it finish.
|
|
||||||
stop_grace_period: 120s
|
|
||||||
healthcheck: *celery_healthcheck
|
|
||||||
deploy: *deploy_policy
|
|
||||||
environment:
|
environment:
|
||||||
<<: *app_env
|
<<: *app_env
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@@ -31,68 +31,6 @@ browser.runtime.onInstalled.addListener(() => ensureInitialized());
|
|||||||
browser.runtime.onStartup.addListener(() => ensureInitialized());
|
browser.runtime.onStartup.addListener(() => ensureInitialized());
|
||||||
ensureInitialized().catch(e => console.error('init failed:', e));
|
ensureInitialized().catch(e => console.error('init failed:', e));
|
||||||
|
|
||||||
// ---- Extension self-update check (#1489) ----
|
|
||||||
// Installed per-instance from the operator's FC host, so Firefox's static
|
|
||||||
// update_url can't apply (each instance has a different host). Instead ask the
|
|
||||||
// configured backend for the latest published version and nudge the operator to
|
|
||||||
// reinstall the freshly-signed XPI — surfaced as a popup banner (on demand) and
|
|
||||||
// a toolbar badge (daily). /api/extension/manifest is public and returns
|
|
||||||
// {version, latest_url, sha256}; the XPI is served from the web root (not /api).
|
|
||||||
|
|
||||||
function versionIsNewer(candidate, current) {
|
|
||||||
// Dotted numeric compare so 1.0.10 > 1.0.9 (a plain string compare wouldn't).
|
|
||||||
const a = String(candidate).split('.').map(n => parseInt(n, 10) || 0);
|
|
||||||
const b = String(current).split('.').map(n => parseInt(n, 10) || 0);
|
|
||||||
for (let i = 0; i < Math.max(a.length, b.length); i++) {
|
|
||||||
if ((a[i] || 0) !== (b[i] || 0)) return (a[i] || 0) > (b[i] || 0);
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function checkForUpdateInfo() {
|
|
||||||
await ensureInitialized();
|
|
||||||
if (!api.isConfigured()) return { updateAvailable: false, configured: false };
|
|
||||||
let info;
|
|
||||||
try {
|
|
||||||
info = await api.getExtensionManifest();
|
|
||||||
} catch (e) {
|
|
||||||
return { updateAvailable: false, error: e.message };
|
|
||||||
}
|
|
||||||
const currentVersion = browser.runtime.getManifest().version;
|
|
||||||
const latestVersion = info && info.version ? info.version : null;
|
|
||||||
// latest_url is served from the web root, not the JSON API.
|
|
||||||
const base = api.webRoot();
|
|
||||||
return {
|
|
||||||
updateAvailable: !!latestVersion && versionIsNewer(latestVersion, currentVersion),
|
|
||||||
currentVersion,
|
|
||||||
latestVersion,
|
|
||||||
xpiUrl: info && info.latest_url ? `${base}${info.latest_url}` : null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
async function refreshUpdateBadge() {
|
|
||||||
let r;
|
|
||||||
try { r = await checkForUpdateInfo(); } catch { return; }
|
|
||||||
try {
|
|
||||||
await browser.action.setBadgeText({ text: r.updateAvailable ? '↑' : '' });
|
|
||||||
if (r.updateAvailable) {
|
|
||||||
await browser.action.setBadgeBackgroundColor({ color: '#F4BA7A' });
|
|
||||||
await browser.action.setTitle({ title: `FabledCurator — update available (v${r.latestVersion})` });
|
|
||||||
} else {
|
|
||||||
await browser.action.setTitle({ title: 'FabledCurator' });
|
|
||||||
}
|
|
||||||
} catch { /* action API unavailable — non-fatal */ }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Daily proactive check (needs the "alarms" permission). create() is idempotent
|
|
||||||
// by name, so re-running it on each event-page load is safe.
|
|
||||||
browser.alarms.create('fc-update-check', { periodInMinutes: 24 * 60, delayInMinutes: 1 });
|
|
||||||
browser.alarms.onAlarm.addListener((alarm) => {
|
|
||||||
if (alarm.name === 'fc-update-check') refreshUpdateBadge();
|
|
||||||
});
|
|
||||||
browser.runtime.onStartup.addListener(() => refreshUpdateBadge());
|
|
||||||
browser.runtime.onInstalled.addListener(() => refreshUpdateBadge());
|
|
||||||
|
|
||||||
// ---- Discord token capture via webRequest ----
|
// ---- Discord token capture via webRequest ----
|
||||||
|
|
||||||
browser.webRequest.onBeforeSendHeaders.addListener(
|
browser.webRequest.onBeforeSendHeaders.addListener(
|
||||||
@@ -210,21 +148,6 @@ browser.webRequest.onBeforeRedirect.addListener(
|
|||||||
{ urls: ['https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback*'] },
|
{ urls: ['https://app-api.pixiv.net/web/v1/users/auth/pixiv/callback*'] },
|
||||||
);
|
);
|
||||||
|
|
||||||
// Extract → verify → upload one cookie-auth platform. Returns a structured
|
|
||||||
// outcome so the two callers (EXPORT_COOKIES single, EXPORT_ALL_COOKIES) shape
|
|
||||||
// their own response + skip semantics. Verifies the captured cookies are
|
|
||||||
// actually live BEFORE uploading, so a confirmed-stale session doesn't overwrite
|
|
||||||
// good FC-side credentials; platforms with no verify config (v.ok === null) fall
|
|
||||||
// through to upload.
|
|
||||||
async function exportPlatformCookies(key) {
|
|
||||||
const cookies = await extractCookiesForPlatform(key);
|
|
||||||
if (cookies.length === 0) return { status: 'empty' };
|
|
||||||
const v = await verifyCookiesForPlatform(key);
|
|
||||||
if (v.ok === false) return { status: 'stale', reason: v.reason, cookieCount: cookies.length };
|
|
||||||
await api.uploadCredentials(key, 'cookies', toNetscapeFormat(cookies));
|
|
||||||
return { status: 'ok', cookieCount: cookies.length, verified: v.ok === true };
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---- Message router ----
|
// ---- Message router ----
|
||||||
|
|
||||||
browser.runtime.onMessage.addListener(async (msg) => {
|
browser.runtime.onMessage.addListener(async (msg) => {
|
||||||
@@ -269,14 +192,22 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
if (!platform) return { error: `Unknown platform: ${key}` };
|
if (!platform) return { error: `Unknown platform: ${key}` };
|
||||||
try {
|
try {
|
||||||
if (platform.authType === 'cookies') {
|
if (platform.authType === 'cookies') {
|
||||||
const r = await exportPlatformCookies(key);
|
const cookies = await extractCookiesForPlatform(key);
|
||||||
if (r.status === 'empty') return { error: 'No cookies found — log in first.' };
|
if (cookies.length === 0) return { error: 'No cookies found — log in first.' };
|
||||||
if (r.status === 'stale') {
|
// Verify the captured cookies are actually live BEFORE
|
||||||
|
// uploading. Skips upload on confirmed-stale sessions so we
|
||||||
|
// don't overwrite FC-side credentials with garbage. Platforms
|
||||||
|
// without a verify config (verify.ok === null) fall through
|
||||||
|
// to upload as before.
|
||||||
|
const v = await verifyCookiesForPlatform(key);
|
||||||
|
if (v.ok === false) {
|
||||||
return {
|
return {
|
||||||
error: `Captured ${r.cookieCount} ${platform.name} cookies but they don't appear authenticated (${r.reason}). Log in again in this browser, then retry.`,
|
error: `Captured ${cookies.length} ${platform.name} cookies but they don't appear authenticated (${v.reason}). Log in again in this browser, then retry.`,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return { success: true, cookieCount: r.cookieCount, verified: r.verified };
|
const data = toNetscapeFormat(cookies);
|
||||||
|
await api.uploadCredentials(key, 'cookies', data);
|
||||||
|
return { success: true, cookieCount: cookies.length, verified: v.ok === true };
|
||||||
}
|
}
|
||||||
if (key === 'discord') {
|
if (key === 'discord') {
|
||||||
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
||||||
@@ -304,10 +235,18 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
try {
|
||||||
const r = await exportPlatformCookies(key);
|
const cookies = await extractCookiesForPlatform(key);
|
||||||
if (r.status === 'empty') results[key] = { skipped: true, reason: 'no cookies' };
|
if (cookies.length === 0) {
|
||||||
else if (r.status === 'stale') results[key] = { error: `verify failed: ${r.reason}` };
|
results[key] = { skipped: true, reason: 'no cookies' };
|
||||||
else results[key] = { success: true, cookieCount: r.cookieCount, verified: r.verified };
|
continue;
|
||||||
|
}
|
||||||
|
const v = await verifyCookiesForPlatform(key);
|
||||||
|
if (v.ok === false) {
|
||||||
|
results[key] = { error: `verify failed: ${v.reason}` };
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
await api.uploadCredentials(key, 'cookies', toNetscapeFormat(cookies));
|
||||||
|
results[key] = { success: true, cookieCount: cookies.length, verified: v.ok === true };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
results[key] = { error: e.message };
|
results[key] = { error: e.message };
|
||||||
}
|
}
|
||||||
@@ -344,9 +283,11 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
}
|
}
|
||||||
|
|
||||||
case 'OPEN_ARTIST_PAGE': {
|
case 'OPEN_ARTIST_PAGE': {
|
||||||
// The SPA artist route (/artist/:slug) is served from the web root, not
|
// apiUrl is configured with the /api suffix (see
|
||||||
// the JSON API — see api.webRoot().
|
// options/options.html placeholder); the SPA artist route is
|
||||||
const base = api.webRoot();
|
// /artist/:slug, served from the same origin. Strip /api so the
|
||||||
|
// browser-level URL hits the Vue router, not the JSON API.
|
||||||
|
const base = (api.baseUrl || '').replace(/\/+$/, '').replace(/\/api$/, '');
|
||||||
const slug = encodeURIComponent(msg.slug || '');
|
const slug = encodeURIComponent(msg.slug || '');
|
||||||
if (!base || !slug) return { error: 'apiUrl or slug missing' };
|
if (!base || !slug) return { error: 'apiUrl or slug missing' };
|
||||||
try {
|
try {
|
||||||
@@ -357,9 +298,6 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
case 'CHECK_UPDATE':
|
|
||||||
return await checkForUpdateInfo();
|
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return { error: `Unknown message type: ${msg.type}` };
|
return { error: `Unknown message type: ${msg.type}` };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -89,19 +89,6 @@ class FabledCuratorAPI {
|
|||||||
const qs = new URLSearchParams({ url }).toString();
|
const qs = new URLSearchParams({ url }).toString();
|
||||||
return this.request('GET', `/extension/probe?${qs}`);
|
return this.request('GET', `/extension/probe?${qs}`);
|
||||||
}
|
}
|
||||||
// Latest published extension version on this instance — drives the in-app
|
|
||||||
// update prompt. Public endpoint (no key needed, but request() sends it
|
|
||||||
// harmlessly). Returns {version, xpi_url, latest_url, sha256}.
|
|
||||||
getExtensionManifest() {
|
|
||||||
return this.request('GET', '/extension/manifest');
|
|
||||||
}
|
|
||||||
|
|
||||||
// The web/SPA root: baseUrl with the trailing slash + `/api` suffix stripped.
|
|
||||||
// Where the Vue router (artist pages) and the served XPI live, NOT the JSON
|
|
||||||
// API. Used by OPEN_ARTIST_PAGE + the self-update check.
|
|
||||||
webRoot() {
|
|
||||||
return (this.baseUrl || '').replace(/\/+$/, '').replace(/\/api$/, '');
|
|
||||||
}
|
|
||||||
|
|
||||||
// Connection test = the cheapest read with auth.
|
// Connection test = the cheapest read with auth.
|
||||||
testConnection() {
|
testConnection() {
|
||||||
|
|||||||
@@ -86,16 +86,7 @@ const PLATFORMS = {
|
|||||||
* script to decide whether to show the floating "Add as source" button.
|
* script to decide whether to show the floating "Add as source" button.
|
||||||
*/
|
*/
|
||||||
const PLATFORM_ARTIST_PATTERNS = {
|
const PLATFORM_ARTIST_PATTERNS = {
|
||||||
// Patreon serves the same creator under three URL shapes (see backend
|
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?!home$|search\b|messages\b|notifications\b|library\b|settings\b|posts\b|c\/)[^/?#]+\/?$/i,
|
||||||
// patreon_resolver._VANITY_RE): bare `patreon.com/Atole`, `c/` prefix, and
|
|
||||||
// `cw/` "creator workspace" — the last is the URL you land on once you're
|
|
||||||
// SUBSCRIBED, which is exactly when the button matters. Match all three, and
|
|
||||||
// drop the single-segment end-anchor so a creator's inner page
|
|
||||||
// (…/cw/Atole/posts, …/Atole/membership) also injects the button. Nav pages
|
|
||||||
// (home/search/…/posts permalink) stay excluded. Mirrors extension_service
|
|
||||||
// ._PLATFORM_PATTERNS — keep in sync (operator-flagged 2026-07-13: button
|
|
||||||
// vanished once subscribed because the old pattern only matched the bare root).
|
|
||||||
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i,
|
|
||||||
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
||||||
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
||||||
deviantart: /^https?:\/\/(www\.)?deviantart\.com\/(?!home$|watch\b|tag\b|browse\b)[^/?#]+\/?$/i,
|
deviantart: /^https?:\/\/(www\.)?deviantart\.com\/(?!home$|watch\b|tag\b|browse\b)[^/?#]+\/?$/i,
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"manifest_version": 3,
|
"manifest_version": 3,
|
||||||
"name": "FabledCurator",
|
"name": "FabledCurator",
|
||||||
"version": "1.0.9",
|
"version": "1.0.7",
|
||||||
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
"description": "Export cookies from supported platforms to FabledCurator and add creators as sources in one click.",
|
||||||
|
|
||||||
"browser_specific_settings": {
|
"browser_specific_settings": {
|
||||||
@@ -22,8 +22,7 @@
|
|||||||
"tabs",
|
"tabs",
|
||||||
"activeTab",
|
"activeTab",
|
||||||
"webRequest",
|
"webRequest",
|
||||||
"webRequestBlocking",
|
"webRequestBlocking"
|
||||||
"alarms"
|
|
||||||
],
|
],
|
||||||
|
|
||||||
"host_permissions": [
|
"host_permissions": [
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "fabledcurator-extension",
|
"name": "fabledcurator-extension",
|
||||||
"version": "1.0.9",
|
"version": "1.0.7",
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "Firefox extension for FabledCurator",
|
"description": "Firefox extension for FabledCurator",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@@ -10,6 +10,6 @@
|
|||||||
"sign": "web-ext sign --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
"sign": "web-ext sign --source-dir=. --no-config-discovery --ignore-files package.json package-lock.json web-ext-artifacts node_modules README.md .gitignore --channel=unlisted --api-key=$WEB_EXT_API_KEY --api-secret=$WEB_EXT_API_SECRET"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"web-ext": "^10.0.0"
|
"web-ext": "^8.0.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,17 +72,6 @@ body {
|
|||||||
.btn.block { display: block; width: 100%; margin-top: 8px; }
|
.btn.block { display: block; width: 100%; margin-top: 8px; }
|
||||||
.btn.link { background: none; color: var(--on-surface-variant); padding: 4px; }
|
.btn.link { background: none; color: var(--on-surface-variant); padding: 4px; }
|
||||||
.btn.link:hover { color: var(--accent); }
|
.btn.link:hover { color: var(--accent); }
|
||||||
.btn.small { padding: 6px 12px; font-size: 13px; }
|
|
||||||
|
|
||||||
/* In-app update prompt (accent-tinted so it reads as an actionable notice). */
|
|
||||||
.update-banner {
|
|
||||||
display: flex; align-items: center; gap: 10px;
|
|
||||||
margin: 10px 10px 0; padding: 10px 12px;
|
|
||||||
background: rgba(244, 186, 122, 0.12);
|
|
||||||
border: 1px solid rgba(244, 186, 122, 0.4);
|
|
||||||
border-radius: 6px;
|
|
||||||
}
|
|
||||||
#update-text { flex: 1; font-size: 13px; }
|
|
||||||
|
|
||||||
.source-row .play {
|
.source-row .play {
|
||||||
background: none; border: none; color: var(--on-surface-variant);
|
background: none; border: none; color: var(--on-surface-variant);
|
||||||
|
|||||||
@@ -20,11 +20,6 @@
|
|||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section id="main-content" class="main hidden">
|
<section id="main-content" class="main hidden">
|
||||||
<div id="update-banner" class="update-banner hidden">
|
|
||||||
<span id="update-text"></span>
|
|
||||||
<button id="update-btn" class="btn primary small">Update</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<nav class="tabs">
|
<nav class="tabs">
|
||||||
<button class="tab active" data-tab="platforms">Platforms</button>
|
<button class="tab active" data-tab="platforms">Platforms</button>
|
||||||
<button class="tab" data-tab="sources">Sources</button>
|
<button class="tab" data-tab="sources">Sources</button>
|
||||||
|
|||||||
+12
-33
@@ -2,15 +2,6 @@ document.addEventListener('DOMContentLoaded', init);
|
|||||||
|
|
||||||
const CONNECTION_TEST_INTERVAL = 2 * 60 * 1000;
|
const CONNECTION_TEST_INTERVAL = 2 * 60 * 1000;
|
||||||
|
|
||||||
// A centered muted note div — the loading / empty state shared by the platform
|
|
||||||
// and sources lists.
|
|
||||||
function mutedNote(text) {
|
|
||||||
const d = document.createElement('div');
|
|
||||||
d.style.cssText = 'text-align:center;padding:18px;color:var(--on-surface-variant);';
|
|
||||||
d.textContent = text;
|
|
||||||
return d;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function init() {
|
async function init() {
|
||||||
try {
|
try {
|
||||||
const cfg = await browser.runtime.sendMessage({ type: 'GET_CONFIG' });
|
const cfg = await browser.runtime.sendMessage({ type: 'GET_CONFIG' });
|
||||||
@@ -23,7 +14,6 @@ async function init() {
|
|||||||
setupEventListeners();
|
setupEventListeners();
|
||||||
showPlatformsLoading();
|
showPlatformsLoading();
|
||||||
testConnectionIfNeeded();
|
testConnectionIfNeeded();
|
||||||
checkForUpdate();
|
|
||||||
loadPlatformStatus().catch(e => showError(`Failed to load platforms: ${e.message}`));
|
loadPlatformStatus().catch(e => showError(`Failed to load platforms: ${e.message}`));
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
showSetupRequired();
|
showSetupRequired();
|
||||||
@@ -47,7 +37,10 @@ function showSetupRequired() {
|
|||||||
function showPlatformsLoading() {
|
function showPlatformsLoading() {
|
||||||
const c = document.getElementById('platforms-list');
|
const c = document.getElementById('platforms-list');
|
||||||
c.textContent = '';
|
c.textContent = '';
|
||||||
c.appendChild(mutedNote('Loading platforms…'));
|
const d = document.createElement('div');
|
||||||
|
d.style.cssText = 'text-align:center;padding:18px;color:var(--on-surface-variant);';
|
||||||
|
d.textContent = 'Loading platforms…';
|
||||||
|
c.appendChild(d);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function testConnectionIfNeeded() {
|
async function testConnectionIfNeeded() {
|
||||||
@@ -70,26 +63,6 @@ function updateConnectionDot(connected) {
|
|||||||
d.title = connected ? 'Connected to FabledCurator' : 'Disconnected';
|
d.title = connected ? 'Connected to FabledCurator' : 'Disconnected';
|
||||||
}
|
}
|
||||||
|
|
||||||
// Nudge to reinstall when the configured instance publishes a newer signed XPI
|
|
||||||
// (the extension is self-hosted, so there's no Firefox auto-update). Never
|
|
||||||
// blocks the popup — a failed check just leaves the banner hidden.
|
|
||||||
async function checkForUpdate() {
|
|
||||||
try {
|
|
||||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_UPDATE' });
|
|
||||||
if (r && r.updateAvailable && r.xpiUrl) showUpdateBanner(r);
|
|
||||||
} catch { /* non-fatal */ }
|
|
||||||
}
|
|
||||||
|
|
||||||
function showUpdateBanner(r) {
|
|
||||||
document.getElementById('update-text').textContent =
|
|
||||||
`Update available — v${r.latestVersion} (installed v${r.currentVersion})`;
|
|
||||||
// Opening the signed XPI triggers Firefox's native install prompt.
|
|
||||||
document.getElementById('update-btn').addEventListener('click', () => {
|
|
||||||
browser.tabs.create({ url: r.xpiUrl });
|
|
||||||
});
|
|
||||||
document.getElementById('update-banner').classList.remove('hidden');
|
|
||||||
}
|
|
||||||
|
|
||||||
async function loadPlatformStatus() {
|
async function loadPlatformStatus() {
|
||||||
const status = await browser.runtime.sendMessage({ type: 'GET_PLATFORM_STATUS' });
|
const status = await browser.runtime.sendMessage({ type: 'GET_PLATFORM_STATUS' });
|
||||||
const c = document.getElementById('platforms-list');
|
const c = document.getElementById('platforms-list');
|
||||||
@@ -189,7 +162,10 @@ async function exportAllCookies() {
|
|||||||
async function loadSources() {
|
async function loadSources() {
|
||||||
const c = document.getElementById('sources-list');
|
const c = document.getElementById('sources-list');
|
||||||
c.textContent = '';
|
c.textContent = '';
|
||||||
c.appendChild(mutedNote('Loading sources…'));
|
const d = document.createElement('div');
|
||||||
|
d.style.cssText = 'text-align:center;padding:18px;color:var(--on-surface-variant);';
|
||||||
|
d.textContent = 'Loading sources…';
|
||||||
|
c.appendChild(d);
|
||||||
const r = await browser.runtime.sendMessage({ type: 'LIST_SOURCES' });
|
const r = await browser.runtime.sendMessage({ type: 'LIST_SOURCES' });
|
||||||
c.textContent = '';
|
c.textContent = '';
|
||||||
if (r.error) {
|
if (r.error) {
|
||||||
@@ -200,7 +176,10 @@ async function loadSources() {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!r.sources || r.sources.length === 0) {
|
if (!r.sources || r.sources.length === 0) {
|
||||||
c.appendChild(mutedNote('No sources yet.'));
|
const empty = document.createElement('div');
|
||||||
|
empty.style.cssText = 'text-align:center;padding:18px;color:var(--on-surface-variant);';
|
||||||
|
empty.textContent = 'No sources yet.';
|
||||||
|
c.appendChild(empty);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
for (const src of r.sources) c.appendChild(createSourceRow(src));
|
for (const src of r.sources) c.appendChild(createSourceRow(src));
|
||||||
|
|||||||
+13
-11
@@ -4,28 +4,30 @@
|
|||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=24"
|
"node": ">=22"
|
||||||
},
|
},
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"dev": "vite",
|
"dev": "vite",
|
||||||
"build": "vite build",
|
"build": "vite build",
|
||||||
"preview": "vite preview",
|
"preview": "vite preview",
|
||||||
"test:unit": "vitest run"
|
"test:unit": "vitest run",
|
||||||
|
"check": "vue-tsc --noEmit"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"vue": "^3.5.0",
|
"vue": "^3.4.0",
|
||||||
"vue-router": "^5.0.0",
|
"vue-router": "^4.3.0",
|
||||||
"pinia": "^3.0.0",
|
"pinia": "^2.1.0",
|
||||||
"vuetify": "^4.0.0",
|
"vuetify": "^3.5.0",
|
||||||
"@mdi/font": "^7.4.0"
|
"@mdi/font": "^7.4.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@vitejs/plugin-vue": "^6.0.0",
|
"@vitejs/plugin-vue": "^5.0.0",
|
||||||
"vite": "^8.0.0",
|
"vite": "^5.2.0",
|
||||||
"vite-plugin-vuetify": "^2.1.0",
|
"vue-tsc": "^2.0.0",
|
||||||
|
"vite-plugin-vuetify": "^2.0.0",
|
||||||
"sass": "^1.71.0",
|
"sass": "^1.71.0",
|
||||||
"vitest": "^4.0.0",
|
"vitest": "^2.1.0",
|
||||||
"@vue/test-utils": "^2.4.0",
|
"@vue/test-utils": "^2.4.0",
|
||||||
"happy-dom": "^20.0.0"
|
"happy-dom": "^15.0.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,11 +18,11 @@ const route = useRoute()
|
|||||||
<style scoped>
|
<style scoped>
|
||||||
.fc-content {
|
.fc-content {
|
||||||
min-height: 100vh;
|
min-height: 100vh;
|
||||||
/* NO padding-top: the TopNav is position:sticky, so it already reserves its
|
/* Push initial viewport content below the sticky TopNav. Without
|
||||||
own space in the v-app flex column — content flows directly below it. The
|
this, some views' first rows / form fields / table headers can
|
||||||
old 64px padding-top was a leftover from a FIXED navbar and double-counted
|
end up obscured by the navbar (depending on parent overflow
|
||||||
that space, leaving a large empty band at the top of EVERY view (and pushing
|
context interacting with position: sticky). Scrolled-down content
|
||||||
the full-height calc(100vh - 64px) views down so they overflowed). Removed
|
still slides under the nav — the gradient-fade design is intact. */
|
||||||
2026-07-13. Scrolled content still slides under the sticky nav as before. */
|
padding-top: 64px;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<template>
|
<template>
|
||||||
<v-snackbar
|
<v-snackbar
|
||||||
v-model="show" :color="color" location="bottom right" timeout="4000"
|
v-model="show" :color="color" location="bottom right" timeout="4000"
|
||||||
min-height="68" elevation="4"
|
multi-line elevation="4"
|
||||||
>
|
>
|
||||||
{{ message }}
|
{{ message }}
|
||||||
<template #actions>
|
<template #actions>
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<template>
|
<template>
|
||||||
<header ref="navEl" class="fc-topnav" :class="{ 'fc-topnav--chrome': hasStickyChrome }">
|
<header class="fc-topnav">
|
||||||
<div class="fc-nav-left">
|
<div class="fc-nav-left">
|
||||||
<RouterLink :to="FRONT_DOOR" class="fc-brand" aria-label="FabledCurator home">
|
<RouterLink :to="FRONT_DOOR" class="fc-brand" aria-label="FabledCurator home">
|
||||||
<img src="/favicon.svg" alt="" class="fc-brand__glyph" width="22" height="22" />
|
<img src="/favicon.svg" alt="" class="fc-brand__glyph" width="22" height="22" />
|
||||||
@@ -64,39 +64,13 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
|
import { computed, onMounted } from 'vue'
|
||||||
import { useRoute } from 'vue-router'
|
|
||||||
import router, { FRONT_DOOR } from '../router.js'
|
import router, { FRONT_DOOR } from '../router.js'
|
||||||
import { useSystemStore } from '../stores/system.js'
|
import { useSystemStore } from '../stores/system.js'
|
||||||
import PipelineStatusChip from './PipelineStatusChip.vue'
|
import PipelineStatusChip from './PipelineStatusChip.vue'
|
||||||
|
|
||||||
const system = useSystemStore()
|
const system = useSystemStore()
|
||||||
|
onMounted(() => system.refreshHealth())
|
||||||
// Publish the nav's REAL height as --fc-nav-h so full-height workspaces
|
|
||||||
// (Explore/Subscriptions) and sticky sub-headers pin to it exactly instead of a
|
|
||||||
// hardcoded 64px that Vuetify 4's MD3 sizing broke — the Explore breadcrumb was
|
|
||||||
// tucking under a taller nav (#1481). ResizeObserver keeps it live as the nav
|
|
||||||
// reflows (per-view teleported actions, mobile breakpoint, chip state changes).
|
|
||||||
const navEl = ref(null)
|
|
||||||
let navRO = null
|
|
||||||
onMounted(() => {
|
|
||||||
system.refreshHealth()
|
|
||||||
if (navEl.value && 'ResizeObserver' in window) {
|
|
||||||
navRO = new ResizeObserver(() => {
|
|
||||||
const h = navEl.value?.offsetHeight
|
|
||||||
if (h) document.documentElement.style.setProperty('--fc-nav-h', `${h}px`)
|
|
||||||
})
|
|
||||||
navRO.observe(navEl.value)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
onBeforeUnmount(() => { navRO?.disconnect() })
|
|
||||||
|
|
||||||
// Views that pin a sticky sub-header (filter bar / tabs) directly under the nav
|
|
||||||
// declare `meta.stickyChrome`. On those, the nav doesn't fade to transparent at
|
|
||||||
// its bottom — it hands off at the shared seam alpha so the sub-header can
|
|
||||||
// continue the SAME fade (see .fc-chrome-continues in app.css). One gradient.
|
|
||||||
const route = useRoute()
|
|
||||||
const hasStickyChrome = computed(() => !!route.meta?.stickyChrome)
|
|
||||||
|
|
||||||
// Every route with a meta.title is a nav entry. Order by meta.navOrder —
|
// Every route with a meta.title is a nav entry. Order by meta.navOrder —
|
||||||
// router.getRoutes() does NOT guarantee declaration order, so explicit numbers
|
// router.getRoutes() does NOT guarantee declaration order, so explicit numbers
|
||||||
@@ -145,35 +119,16 @@ const health = computed(() => {
|
|||||||
align-items: center;
|
align-items: center;
|
||||||
gap: 1rem;
|
gap: 1rem;
|
||||||
padding: 0.75rem 1rem;
|
padding: 0.75rem 1rem;
|
||||||
/* Obsidian (#14171A) fade — content scrolls under it. Holds high (0.92 →
|
/* Obsidian (#14171A = 20,23,26) gradient fade — content scrolls under it. */
|
||||||
0.84) through the top half, then eases to transparent over the bottom
|
|
||||||
quarter so it tails off softly instead of a straight line to a hard edge
|
|
||||||
(operator 2026-07-13). Shared --fc-chrome-rgb keeps it in sync with the
|
|
||||||
sub-header continuation. */
|
|
||||||
background: linear-gradient(
|
background: linear-gradient(
|
||||||
to bottom,
|
to bottom,
|
||||||
rgba(var(--fc-chrome-rgb), 0.92) 0%,
|
rgba(20, 23, 26, 0.92) 0%,
|
||||||
rgba(var(--fc-chrome-rgb), 0.84) 50%,
|
rgba(20, 23, 26, 0.65) 60%,
|
||||||
rgba(var(--fc-chrome-rgb), 0.55) 75%,
|
rgba(20, 23, 26, 0) 100%
|
||||||
rgba(var(--fc-chrome-rgb), 0) 100%
|
|
||||||
);
|
);
|
||||||
backdrop-filter: blur(2px);
|
backdrop-filter: blur(2px);
|
||||||
-webkit-backdrop-filter: blur(2px);
|
-webkit-backdrop-filter: blur(2px);
|
||||||
}
|
}
|
||||||
/* On a view with a sticky sub-header pinned beneath (meta.stickyChrome), the nav
|
|
||||||
stops fading at the shared seam alpha instead of going fully transparent — the
|
|
||||||
sub-header (.fc-chrome-continues) picks the fade up from there, so the two read
|
|
||||||
as one continuous gradient. Compound selector out-specifies .fc-topnav so it
|
|
||||||
wins regardless of Vite's production CSS ordering. --fc-chrome-* come from the
|
|
||||||
global :root in app.css (custom props inherit into scoped styles). */
|
|
||||||
.fc-topnav.fc-topnav--chrome {
|
|
||||||
background: linear-gradient(
|
|
||||||
to bottom,
|
|
||||||
rgba(var(--fc-chrome-rgb), 0.92) 0%,
|
|
||||||
rgba(var(--fc-chrome-rgb), 0.84) 60%,
|
|
||||||
rgba(var(--fc-chrome-rgb), var(--fc-chrome-seam)) 100%
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
.fc-brand {
|
.fc-brand {
|
||||||
display: flex;
|
display: flex;
|
||||||
|
|||||||
@@ -101,47 +101,6 @@
|
|||||||
</v-card>
|
</v-card>
|
||||||
</v-dialog>
|
</v-dialog>
|
||||||
|
|
||||||
<section class="fc-artist-mgmt__sec">
|
|
||||||
<h2 class="fc-h2">Translation</h2>
|
|
||||||
<p class="fc-muted text-body-2 mb-3">
|
|
||||||
Re-translate every post by {{ overview.name }} through the current
|
|
||||||
Interpreter model — clears the stored translations and rebuilds them.
|
|
||||||
Use this after switching translation models to refresh just this artist.
|
|
||||||
</p>
|
|
||||||
<div class="d-flex align-center flex-wrap" style="gap: 10px;">
|
|
||||||
<v-btn
|
|
||||||
size="small" variant="tonal" rounded="pill"
|
|
||||||
prepend-icon="mdi-translate"
|
|
||||||
:loading="retranslating" :disabled="!translationEnabled"
|
|
||||||
@click="confirmRetranslate = true"
|
|
||||||
>Re-translate posts</v-btn>
|
|
||||||
<span v-if="!translationEnabled" class="fc-muted text-caption">
|
|
||||||
Translation is off — enable it in Settings → Maintenance.
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
|
|
||||||
<!-- m146: per-artist re-translate — clears + rebuilds this artist's stored
|
|
||||||
translations (used after a translation-model change). -->
|
|
||||||
<v-dialog v-model="confirmRetranslate" max-width="440">
|
|
||||||
<v-card>
|
|
||||||
<v-card-title class="fc-h2">Re-translate {{ overview.name }}?</v-card-title>
|
|
||||||
<v-card-text class="text-body-2">
|
|
||||||
Clears the stored translation for every post by
|
|
||||||
<strong>{{ overview.name }}</strong> and re-runs them through your
|
|
||||||
current Interpreter model. Unchanged text comes back from the cache
|
|
||||||
instantly. Runs in the background until done.
|
|
||||||
</v-card-text>
|
|
||||||
<v-card-actions>
|
|
||||||
<v-spacer />
|
|
||||||
<v-btn variant="text" @click="confirmRetranslate = false">Cancel</v-btn>
|
|
||||||
<v-btn color="accent" :loading="retranslating" @click="onRetranslate">
|
|
||||||
Re-translate
|
|
||||||
</v-btn>
|
|
||||||
</v-card-actions>
|
|
||||||
</v-card>
|
|
||||||
</v-dialog>
|
|
||||||
|
|
||||||
<section class="fc-artist-mgmt__sec">
|
<section class="fc-artist-mgmt__sec">
|
||||||
<h2 class="fc-h2">Danger zone</h2>
|
<h2 class="fc-h2">Danger zone</h2>
|
||||||
<ArtistDangerZone
|
<ArtistDangerZone
|
||||||
@@ -154,11 +113,9 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onMounted, ref } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import { useRouter, RouterLink } from 'vue-router'
|
import { useRouter, RouterLink } from 'vue-router'
|
||||||
|
|
||||||
import { useApi } from '../../composables/useApi.js'
|
|
||||||
import { useImportStore } from '../../stores/import.js'
|
|
||||||
import { useSourcesStore } from '../../stores/sources.js'
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
import { toast } from '../../utils/toast.js'
|
import { toast } from '../../utils/toast.js'
|
||||||
import ArtistDangerZone from './ArtistDangerZone.vue'
|
import ArtistDangerZone from './ArtistDangerZone.vue'
|
||||||
@@ -168,38 +125,8 @@ const props = defineProps({
|
|||||||
})
|
})
|
||||||
|
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
const api = useApi()
|
|
||||||
const importStore = useImportStore()
|
|
||||||
const sources = useSourcesStore()
|
const sources = useSourcesStore()
|
||||||
|
|
||||||
// m146: per-artist re-translate. Gated on translation being enabled globally
|
|
||||||
// (the endpoint 400s otherwise) — read the shared import settings once.
|
|
||||||
const translationEnabled = ref(false)
|
|
||||||
const retranslating = ref(false)
|
|
||||||
const confirmRetranslate = ref(false)
|
|
||||||
|
|
||||||
onMounted(async () => {
|
|
||||||
try {
|
|
||||||
await importStore.loadSettings()
|
|
||||||
translationEnabled.value = !!importStore.settings?.translation_enabled
|
|
||||||
} catch { /* non-fatal — the button just stays disabled */ }
|
|
||||||
})
|
|
||||||
|
|
||||||
async function onRetranslate () {
|
|
||||||
retranslating.value = true
|
|
||||||
try {
|
|
||||||
await api.post('/api/settings/translation/retranslate', {
|
|
||||||
body: { artist_id: props.overview.id },
|
|
||||||
})
|
|
||||||
confirmRetranslate.value = false
|
|
||||||
toast({ text: `Re-translating ${props.overview.name}…`, type: 'success' })
|
|
||||||
} catch (e) {
|
|
||||||
toast({ text: `Re-translate failed: ${e.message}`, type: 'error' })
|
|
||||||
} finally {
|
|
||||||
retranslating.value = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// #130: move a source into another artist.
|
// #130: move a source into another artist.
|
||||||
const moveOpen = ref(false)
|
const moveOpen = ref(false)
|
||||||
const moveSource = ref(null)
|
const moveSource = ref(null)
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
filter, applied retroactively to the existing library.
|
filter, applied retroactively to the existing library.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<v-row density="compact">
|
<v-row dense>
|
||||||
<v-col cols="6">
|
<v-col cols="6">
|
||||||
<v-text-field
|
<v-text-field
|
||||||
v-model.number="minW" label="Min width (px)" type="number"
|
v-model.number="minW" label="Min width (px)" type="number"
|
||||||
|
|||||||
@@ -13,7 +13,7 @@
|
|||||||
cadence as the transparency audit.
|
cadence as the transparency audit.
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
<v-row density="compact">
|
<v-row dense>
|
||||||
<v-col cols="6">
|
<v-col cols="6">
|
||||||
<v-text-field
|
<v-text-field
|
||||||
v-model.number="threshold" label="Threshold (0–1)"
|
v-model.number="threshold" label="Threshold (0–1)"
|
||||||
|
|||||||
@@ -1,56 +0,0 @@
|
|||||||
<!--
|
|
||||||
Canonical settings number field (DRY pass #161): a compact numeric v-text-field
|
|
||||||
with a built-in clamp to [min,max] on commit. Hand-rolled identically across the
|
|
||||||
ML settings cards (HeadsCard x6, CropProposersCard, VideoEmbeddingCard).
|
|
||||||
|
|
||||||
The clamp is the point: the cards previously sent Number(raw) straight to the
|
|
||||||
API, so an out-of-range value bounced off the API's 400 validator (only
|
|
||||||
TranslationCard clamped). This is now the single home for that clamp.
|
|
||||||
|
|
||||||
Binds `modelValue` (v-model) and emits `change` on blur/enter AFTER clamping, so
|
|
||||||
the parent's save reads the already-clamped value — same as the prior
|
|
||||||
`v-model.number` + `@change=save` pattern.
|
|
||||||
-->
|
|
||||||
<template>
|
|
||||||
<v-text-field
|
|
||||||
:model-value="modelValue"
|
|
||||||
:label="label"
|
|
||||||
type="number"
|
|
||||||
:min="min"
|
|
||||||
:max="max"
|
|
||||||
:step="step"
|
|
||||||
:disabled="disabled"
|
|
||||||
:density="density" hide-details
|
|
||||||
:style="{ maxWidth }"
|
|
||||||
@update:model-value="v => emit('update:modelValue', v)"
|
|
||||||
@change="onCommit"
|
|
||||||
/>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script setup>
|
|
||||||
const props = defineProps({
|
|
||||||
modelValue: { type: [Number, String], default: null },
|
|
||||||
label: { type: String, default: '' },
|
|
||||||
min: { type: [Number, String], default: null },
|
|
||||||
max: { type: [Number, String], default: null },
|
|
||||||
step: { type: [Number, String], default: 1 },
|
|
||||||
maxWidth: { type: String, default: '200px' },
|
|
||||||
density: { type: String, default: 'compact' },
|
|
||||||
disabled: { type: Boolean, default: false },
|
|
||||||
})
|
|
||||||
const emit = defineEmits(['update:modelValue', 'change'])
|
|
||||||
|
|
||||||
function onCommit() {
|
|
||||||
// On blur/enter: coerce to a number and clamp to [min,max] so an out-of-range
|
|
||||||
// value never reaches the API. props.modelValue reflects the latest keystroke
|
|
||||||
// (kept in sync by the passthrough above); re-emit the clamped number, then let
|
|
||||||
// the parent persist.
|
|
||||||
let n = Number(props.modelValue)
|
|
||||||
if (!Number.isNaN(n)) {
|
|
||||||
if (props.min !== null && props.min !== '') n = Math.max(Number(props.min), n)
|
|
||||||
if (props.max !== null && props.max !== '') n = Math.min(Number(props.max), n)
|
|
||||||
if (n !== Number(props.modelValue)) emit('update:modelValue', n)
|
|
||||||
}
|
|
||||||
emit('change')
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
<!--
|
|
||||||
Canonical settings toggle row (DRY pass #161): an accent icon + an uppercase
|
|
||||||
.fc-section-h label + a right-aligned switch. Hand-rolled identically in the
|
|
||||||
ML settings cards (HeadsCard x3, CropProposersCard, MLBackfillCard).
|
|
||||||
|
|
||||||
Two-way binds `modelValue` (so the parent switch state stays optimistic) AND
|
|
||||||
emits `change` with the new boolean, so the parent can persist + revert on
|
|
||||||
failure — matching the prior `v-model` + `@update:model-value=handler` pattern.
|
|
||||||
-->
|
|
||||||
<template>
|
|
||||||
<div class="d-flex align-center mb-1" style="gap: 10px;">
|
|
||||||
<v-icon v-if="icon" size="18" :color="iconColor">{{ icon }}</v-icon>
|
|
||||||
<span class="fc-section-h">{{ label }}</span>
|
|
||||||
<v-switch
|
|
||||||
:model-value="modelValue"
|
|
||||||
:loading="loading"
|
|
||||||
:disabled="disabled"
|
|
||||||
hide-details density="compact" color="success" class="ml-auto"
|
|
||||||
@update:model-value="onSwitch"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script setup>
|
|
||||||
defineProps({
|
|
||||||
modelValue: { type: Boolean, default: false },
|
|
||||||
label: { type: String, default: '' },
|
|
||||||
icon: { type: String, default: '' },
|
|
||||||
// Icon tint. Default accent; pass null for the theme default (e.g. when a row
|
|
||||||
// is off). null (not undefined) so the default doesn't override it.
|
|
||||||
iconColor: { type: String, default: 'accent' },
|
|
||||||
loading: { type: Boolean, default: false },
|
|
||||||
disabled: { type: Boolean, default: false },
|
|
||||||
})
|
|
||||||
const emit = defineEmits(['update:modelValue', 'change'])
|
|
||||||
|
|
||||||
function onSwitch(v) {
|
|
||||||
const b = !!v
|
|
||||||
emit('update:modelValue', b)
|
|
||||||
emit('change', b)
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
@@ -14,10 +14,10 @@
|
|||||||
@update:search="onSearch"
|
@update:search="onSearch"
|
||||||
@update:model-value="onPick"
|
@update:model-value="onPick"
|
||||||
>
|
>
|
||||||
<template #item="{ props: itemProps, internalItem }">
|
<template #item="{ props: itemProps, item }">
|
||||||
<v-list-item v-bind="itemProps" :title="internalItem.raw.name">
|
<v-list-item v-bind="itemProps" :title="item.raw.name">
|
||||||
<template #subtitle>
|
<template #subtitle>
|
||||||
{{ internalItem.raw.fandom_name ? `character · ${internalItem.raw.fandom_name}` : internalItem.raw.kind }}
|
{{ item.raw.fandom_name ? `character · ${item.raw.fandom_name}` : item.raw.kind }}
|
||||||
</template>
|
</template>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
</template>
|
</template>
|
||||||
|
|||||||
@@ -34,15 +34,6 @@
|
|||||||
:color="store.filter.no_artist ? 'accent' : undefined"
|
:color="store.filter.no_artist ? 'accent' : undefined"
|
||||||
@click="toggleFlag('no_artist')"
|
@click="toggleFlag('no_artist')"
|
||||||
>No artist<span class="fc-facets__count">{{ facetCount('no_artist') }}</span></v-chip>
|
>No artist<span class="fc-facets__count">{{ facetCount('no_artist') }}</span></v-chip>
|
||||||
<!-- Reveal presentation chrome (banner / editor screenshot) the gallery
|
|
||||||
hides by default (milestone 141). A browse-mode toggle, not a count. -->
|
|
||||||
<v-chip
|
|
||||||
size="small" label
|
|
||||||
:variant="store.filter.include_hidden ? 'flat' : 'tonal'"
|
|
||||||
:color="store.filter.include_hidden ? 'accent' : undefined"
|
|
||||||
title="Show hidden — reveal banners / editor screenshots the gallery sets aside by default"
|
|
||||||
@click="toggleFlag('include_hidden')"
|
|
||||||
>Show hidden</v-chip>
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-filterbar-wrap fc-chrome-continues">
|
<div class="fc-filterbar-wrap">
|
||||||
<div class="fc-filterbar">
|
<div class="fc-filterbar">
|
||||||
<v-autocomplete
|
<v-autocomplete
|
||||||
v-model="selected"
|
v-model="selected"
|
||||||
@@ -13,14 +13,14 @@
|
|||||||
@update:search="onSearch"
|
@update:search="onSearch"
|
||||||
@update:model-value="onPick"
|
@update:model-value="onPick"
|
||||||
>
|
>
|
||||||
<template #item="{ props: itemProps, internalItem }">
|
<template #item="{ props: itemProps, item }">
|
||||||
<v-list-item v-bind="itemProps" :title="internalItem.raw.name">
|
<v-list-item v-bind="itemProps" :title="item.raw.name">
|
||||||
<template #prepend>
|
<template #prepend>
|
||||||
<v-icon size="small">{{ iconFor(internalItem.raw) }}</v-icon>
|
<v-icon size="small">{{ iconFor(item.raw) }}</v-icon>
|
||||||
</template>
|
</template>
|
||||||
<template #subtitle>
|
<template #subtitle>
|
||||||
{{ internalItem.raw.kind === 'artist' ? 'artist'
|
{{ item.raw.kind === 'artist' ? 'artist'
|
||||||
: (internalItem.raw.fandom_name ? `character · ${internalItem.raw.fandom_name}` : internalItem.raw.kind) }}
|
: (item.raw.fandom_name ? `character · ${item.raw.fandom_name}` : item.raw.kind) }}
|
||||||
</template>
|
</template>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
</template>
|
</template>
|
||||||
@@ -160,7 +160,7 @@ let debounce = null
|
|||||||
const refineCount = computed(() => {
|
const refineCount = computed(() => {
|
||||||
const f = store.filter
|
const f = store.filter
|
||||||
return (f.platform ? 1 : 0) + (f.untagged ? 1 : 0) + (f.no_artist ? 1 : 0)
|
return (f.platform ? 1 : 0) + (f.untagged ? 1 : 0) + (f.no_artist ? 1 : 0)
|
||||||
+ (f.include_hidden ? 1 : 0) + (f.date_from ? 1 : 0) + (f.date_to ? 1 : 0)
|
+ (f.date_from ? 1 : 0) + (f.date_to ? 1 : 0)
|
||||||
})
|
})
|
||||||
const hasRefineFilters = computed(() => refineCount.value > 0)
|
const hasRefineFilters = computed(() => refineCount.value > 0)
|
||||||
|
|
||||||
@@ -306,17 +306,27 @@ function pushFilter(mutate) {
|
|||||||
frosted block pinned directly under the 64px TopNav and continuous with it. */
|
frosted block pinned directly under the 64px TopNav and continuous with it. */
|
||||||
.fc-filterbar-wrap {
|
.fc-filterbar-wrap {
|
||||||
position: sticky;
|
position: sticky;
|
||||||
top: var(--fc-nav-h, 64px); /* pins at the nav's real measured bottom (#1481) */
|
top: 64px;
|
||||||
z-index: 5;
|
z-index: 5;
|
||||||
/* Attach to the TopNav: cancel the v-container's top padding (pt-2 = 8px)
|
/* Attach to the TopNav: cancel the v-container's top padding (pt-2 = 8px)
|
||||||
so the bar sits flush at 64px even at scroll 0 — without this it detaches
|
so the bar sits flush at 64px even at scroll 0 — without this it detaches
|
||||||
and a gap shows through when scrolled to the top. */
|
and a gap shows through when scrolled to the top. */
|
||||||
margin-top: -8px;
|
margin-top: -8px;
|
||||||
margin-bottom: 12px;
|
margin-bottom: 12px;
|
||||||
/* The frost itself (obsidian fade + blur) is the shared .fc-chrome-continues
|
/* EXACT same gradiated obsidian (#14171A = 20,23,26) frost as the TopNav so
|
||||||
primitive: it CONTINUES the TopNav's fade from the seam alpha to transparent
|
the two read as one continuous piece of chrome — images scroll visibly
|
||||||
rather than re-darkening, so the nav + bar read as one gradient (operator
|
under both. The nav's gradient fades to transparent at ITS bottom; this
|
||||||
2026-07-13). This block only owns the sticky positioning now. */
|
bar re-darkens at its top, so a faint seam (the page/image showing through
|
||||||
|
the nav's transparent edge) separates them when scrolled to the very top,
|
||||||
|
while under-scroll they frost as one. */
|
||||||
|
background: linear-gradient(
|
||||||
|
to bottom,
|
||||||
|
rgba(20, 23, 26, 0.92) 0%,
|
||||||
|
rgba(20, 23, 26, 0.65) 60%,
|
||||||
|
rgba(20, 23, 26, 0) 100%
|
||||||
|
);
|
||||||
|
backdrop-filter: blur(2px);
|
||||||
|
-webkit-backdrop-filter: blur(2px);
|
||||||
}
|
}
|
||||||
.fc-filterbar {
|
.fc-filterbar {
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -331,20 +341,6 @@ function pushFilter(mutate) {
|
|||||||
.fc-filterbar-wrap :deep(.v-btn-group) {
|
.fc-filterbar-wrap :deep(.v-btn-group) {
|
||||||
background-color: rgba(20, 23, 26, 0.72);
|
background-color: rgba(20, 23, 26, 0.72);
|
||||||
}
|
}
|
||||||
/* Media toggle (All / Images / Videos) as ONE cohesive segmented control.
|
|
||||||
FC's global VBtn { rounded: 'pill' } default made Vuetify 4 pill-round each
|
|
||||||
SEGMENT individually, so the rounded ends collided at the joins — the shapes
|
|
||||||
landed awkwardly on the button edges (operator 2026-07-13). Square the inner
|
|
||||||
segments (over the pill utility's !important) and clip the group to a single
|
|
||||||
8px outline (matches the chips/tiles rounding elsewhere in the app). Radius
|
|
||||||
only — no height change, so the bar height and nav offset are untouched. */
|
|
||||||
.fc-filterbar-wrap :deep(.v-btn-toggle) {
|
|
||||||
border-radius: 8px;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
.fc-filterbar-wrap :deep(.v-btn-toggle .v-btn) {
|
|
||||||
border-radius: 0 !important;
|
|
||||||
}
|
|
||||||
.fc-filterbar__search { max-width: 320px; min-width: 200px; }
|
.fc-filterbar__search { max-width: 320px; min-width: 200px; }
|
||||||
.fc-filterbar__chips { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
|
.fc-filterbar__chips { display: flex; align-items: center; gap: 6px; flex-wrap: wrap; }
|
||||||
/* The tag chips' bodies toggle include/exclude — signal they're clickable. */
|
/* The tag chips' bodies toggle include/exclude — signal they're clickable. */
|
||||||
|
|||||||
@@ -1,157 +0,0 @@
|
|||||||
<template>
|
|
||||||
<!-- System-tag auto-applies (chrome hides / process WIP tags) that ALSO looked
|
|
||||||
like real content — surfaced PROACTIVELY atop the gallery whenever there's
|
|
||||||
something to review (NOT gated on the Show-hidden toggle, so misfires can't
|
|
||||||
go unnoticed), most-concerning first, with keep / remove (#141, #1464).
|
|
||||||
Renders nothing when there's nothing to review. -->
|
|
||||||
<section v-if="items.length" class="fc-review" aria-label="Auto-tagged images to review">
|
|
||||||
<div class="fc-review__head">
|
|
||||||
<v-icon size="18" color="warning">mdi-alert-outline</v-icon>
|
|
||||||
<span class="fc-review__title">
|
|
||||||
{{ items.length }} auto-tagged {{ items.length === 1 ? 'image' : 'images' }}
|
|
||||||
may be real content — review
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<div class="fc-review__cards">
|
|
||||||
<div
|
|
||||||
v-for="it in items" :key="keyOf(it)" class="fc-review-card"
|
|
||||||
>
|
|
||||||
<img
|
|
||||||
:src="it.thumbnail_url" :alt="it.tag_name"
|
|
||||||
class="fc-review-card__thumb" loading="lazy"
|
|
||||||
>
|
|
||||||
<div class="fc-review-card__body">
|
|
||||||
<div
|
|
||||||
class="fc-review-card__conflict"
|
|
||||||
:title="`Scored ${Math.round(it.conflict_score * 100)}% on “${it.conflict_name || 'a content tag'}”`"
|
|
||||||
>
|
|
||||||
also looks like <strong>{{ it.conflict_name || 'content' }}</strong>
|
|
||||||
</div>
|
|
||||||
<div class="fc-review-card__tag">{{ tagLine(it) }}</div>
|
|
||||||
<div class="fc-review-card__acts">
|
|
||||||
<button
|
|
||||||
type="button" class="fc-review-btn fc-review-btn--keep"
|
|
||||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')"
|
|
||||||
>{{ keepLabel(it) }}</button>
|
|
||||||
<button
|
|
||||||
type="button" class="fc-review-btn fc-review-btn--unhide"
|
|
||||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')"
|
|
||||||
>{{ removeLabel(it) }}</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script setup>
|
|
||||||
import { onMounted, ref } from 'vue'
|
|
||||||
import { useApi } from '../../composables/useApi.js'
|
|
||||||
import { toast } from '../../utils/toast.js'
|
|
||||||
|
|
||||||
const api = useApi()
|
|
||||||
|
|
||||||
const items = ref([])
|
|
||||||
const busy = ref([])
|
|
||||||
|
|
||||||
function keyOf(it) { return `${it.image_id}:${it.tag_id}` }
|
|
||||||
// Chrome flags hide the image (keep-hidden / un-hide); process flags leave it
|
|
||||||
// visible and just tagged (keep-tag / remove-tag). Same endpoints, different words.
|
|
||||||
function tagLine(it) { return (it.mode === 'process' ? 'auto-tagged ' : 'hidden as ') + it.tag_name }
|
|
||||||
function keepLabel(it) { return it.mode === 'process' ? 'Keep tag' : 'Keep hidden' }
|
|
||||||
function removeLabel(it) { return it.mode === 'process' ? 'Remove tag' : 'Un-hide' }
|
|
||||||
|
|
||||||
async function load() {
|
|
||||||
// Fetched unconditionally on mount — the strip prompts for pending misfires
|
|
||||||
// even when the operator is browsing normally (Show-hidden off).
|
|
||||||
try {
|
|
||||||
const body = await api.get('/api/gallery/hidden-review')
|
|
||||||
items.value = body.items || []
|
|
||||||
} catch { items.value = [] }
|
|
||||||
}
|
|
||||||
|
|
||||||
async function resolve(it, action) {
|
|
||||||
const k = keyOf(it)
|
|
||||||
busy.value = [...busy.value, k]
|
|
||||||
try {
|
|
||||||
await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`)
|
|
||||||
items.value = items.value.filter((x) => keyOf(x) !== k)
|
|
||||||
if (action === 'unhide') {
|
|
||||||
const verb = it.mode === 'process' ? 'Removed' : 'Un-hidden'
|
|
||||||
toast({ text: `${verb} — “${it.tag_name}” removed; it'll train the head`, type: 'success' })
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
toast({
|
|
||||||
text: `Could not ${action === 'keep' ? 'keep hidden' : 'un-hide'}: ${e.message}`,
|
|
||||||
type: 'error',
|
|
||||||
})
|
|
||||||
} finally {
|
|
||||||
busy.value = busy.value.filter((x) => x !== k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
onMounted(load)
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<style scoped>
|
|
||||||
.fc-review {
|
|
||||||
margin-bottom: 14px;
|
|
||||||
padding: 12px;
|
|
||||||
border: 1px solid rgb(var(--v-theme-warning), 0.4);
|
|
||||||
background: rgb(var(--v-theme-warning), 0.06);
|
|
||||||
border-radius: 8px;
|
|
||||||
}
|
|
||||||
.fc-review__head {
|
|
||||||
display: flex; align-items: center; gap: 8px; margin-bottom: 10px;
|
|
||||||
}
|
|
||||||
.fc-review__title {
|
|
||||||
font-size: 14px; font-weight: 600;
|
|
||||||
color: rgb(var(--v-theme-on-surface));
|
|
||||||
}
|
|
||||||
.fc-review__cards {
|
|
||||||
display: flex; gap: 10px; overflow-x: auto; padding-bottom: 4px;
|
|
||||||
}
|
|
||||||
.fc-review-card {
|
|
||||||
flex: 0 0 auto; width: 150px;
|
|
||||||
display: flex; flex-direction: column;
|
|
||||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
|
||||||
border-radius: 6px; overflow: hidden;
|
|
||||||
background: rgb(var(--v-theme-surface));
|
|
||||||
}
|
|
||||||
.fc-review-card__thumb {
|
|
||||||
width: 100%; height: 96px; object-fit: cover; display: block;
|
|
||||||
background: rgb(var(--v-theme-surface-light));
|
|
||||||
}
|
|
||||||
.fc-review-card__body { padding: 6px 8px; }
|
|
||||||
.fc-review-card__conflict {
|
|
||||||
font-size: 11px; color: rgb(var(--v-theme-on-surface));
|
|
||||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
|
||||||
}
|
|
||||||
.fc-review-card__conflict strong { color: rgb(var(--v-theme-warning)); }
|
|
||||||
.fc-review-card__tag {
|
|
||||||
font-size: 10px; color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
margin: 1px 0 6px;
|
|
||||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
|
||||||
}
|
|
||||||
.fc-review-card__acts { display: flex; gap: 4px; }
|
|
||||||
.fc-review-btn {
|
|
||||||
flex: 1; font-size: 11px; padding: 3px 4px; border-radius: 4px;
|
|
||||||
cursor: pointer; border: 1px solid transparent;
|
|
||||||
}
|
|
||||||
.fc-review-btn:disabled { opacity: 0.5; cursor: default; }
|
|
||||||
.fc-review-btn--keep {
|
|
||||||
background: transparent;
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
border-color: rgb(var(--v-theme-on-surface-variant), 0.4);
|
|
||||||
}
|
|
||||||
.fc-review-btn--keep:hover:not(:disabled) {
|
|
||||||
background: rgb(var(--v-theme-on-surface-variant), 0.1);
|
|
||||||
}
|
|
||||||
.fc-review-btn--unhide {
|
|
||||||
background: rgb(var(--v-theme-accent)); color: #fff;
|
|
||||||
}
|
|
||||||
.fc-review-btn--unhide:hover:not(:disabled) { opacity: 0.9; }
|
|
||||||
.fc-review-btn:focus-visible {
|
|
||||||
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 1px;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
<template>
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Treat as alias for…</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<p class="text-caption mb-2">
|
||||||
|
Pick the existing tag the model's prediction should map to. All
|
||||||
|
future predictions of this name will resolve to your tag.
|
||||||
|
</p>
|
||||||
|
<v-autocomplete
|
||||||
|
v-model="selectedId"
|
||||||
|
v-model:menu="menuOpen"
|
||||||
|
:items="results"
|
||||||
|
:item-title="(t) => t.fandom_name ? `${t.name} — ${t.fandom_name}` : t.name"
|
||||||
|
:item-value="(t) => t.id"
|
||||||
|
:loading="loading"
|
||||||
|
label="Canonical tag"
|
||||||
|
no-filter clearable density="compact"
|
||||||
|
@update:search="onSearch"
|
||||||
|
@keydown.enter.capture="onEnter"
|
||||||
|
/>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn @click="$emit('cancel')">Cancel</v-btn>
|
||||||
|
<v-btn
|
||||||
|
color="primary" rounded="pill" :disabled="!selectedId"
|
||||||
|
@click="$emit('confirm', selectedId)"
|
||||||
|
>Use this tag</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref } from 'vue'
|
||||||
|
import { useApi } from '../../composables/useApi.js'
|
||||||
|
import { useAcceptOnEnter } from '../../composables/useAcceptOnEnter.js'
|
||||||
|
|
||||||
|
const props = defineProps({ category: { type: String, required: true } })
|
||||||
|
const emit = defineEmits(['confirm', 'cancel'])
|
||||||
|
|
||||||
|
const api = useApi()
|
||||||
|
const results = ref([])
|
||||||
|
const loading = ref(false)
|
||||||
|
const selectedId = ref(null)
|
||||||
|
let debounce = null
|
||||||
|
|
||||||
|
// Enter on the closed dropdown confirms the selection instead of re-opening it.
|
||||||
|
const { menuOpen, onEnter } = useAcceptOnEnter(() => {
|
||||||
|
if (selectedId.value != null) emit('confirm', selectedId.value)
|
||||||
|
})
|
||||||
|
|
||||||
|
function onSearch(q) {
|
||||||
|
if (debounce) clearTimeout(debounce)
|
||||||
|
debounce = setTimeout(async () => {
|
||||||
|
const query = (q || '').trim()
|
||||||
|
if (!query) { results.value = []; return }
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
// Scope the autocomplete to the prediction's category where it
|
||||||
|
// maps to a tag kind. Only 'character' surfaces as both a
|
||||||
|
// suggestion category and a tag kind now ('artist' + 'copyright'
|
||||||
|
// retired); other categories search unscoped.
|
||||||
|
const kind = props.category === 'character' ? 'character' : null
|
||||||
|
const params = { q: query, limit: 20 }
|
||||||
|
if (kind) params.kind = kind
|
||||||
|
results.value = await api.get('/api/tags/autocomplete', { params })
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
}, 200)
|
||||||
|
}
|
||||||
|
</script>
|
||||||
@@ -16,101 +16,20 @@
|
|||||||
transform: `translate(${panZoom.state.x}px, ${panZoom.state.y}px) scale(${panZoom.state.scale})`
|
transform: `translate(${panZoom.state.x}px, ${panZoom.state.y}px) scale(${panZoom.state.scale})`
|
||||||
}"
|
}"
|
||||||
draggable="false"
|
draggable="false"
|
||||||
@load="onImgLoad"
|
|
||||||
>
|
>
|
||||||
<!-- #1206 grounding overlay: tracks the <img>'s live rendered rect (so it's
|
|
||||||
correct under object-fit letterboxing + pan/zoom) and draws the crop
|
|
||||||
region a hovered suggestion came from. Null grounding → a subtle
|
|
||||||
whole-image frame ("the global vector won, not a crop"). -->
|
|
||||||
<div
|
|
||||||
v-if="hover && overlayBox"
|
|
||||||
class="fc-canvas__overlay"
|
|
||||||
:style="{
|
|
||||||
left: overlayBox.left + 'px', top: overlayBox.top + 'px',
|
|
||||||
width: overlayBox.width + 'px', height: overlayBox.height + 'px',
|
|
||||||
}"
|
|
||||||
>
|
|
||||||
<div
|
|
||||||
class="fc-canvas__region"
|
|
||||||
:class="{ 'fc-canvas__region--whole': !regionStyle }"
|
|
||||||
:style="regionStyle || {}"
|
|
||||||
>
|
|
||||||
<span class="fc-canvas__region-label">
|
|
||||||
<span v-if="regionTag" class="fc-canvas__region-tag">{{ regionTag }}</span>
|
|
||||||
<span class="fc-canvas__region-origin">{{ regionOrigin }}</span>
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
|
import { ref, watch } from 'vue'
|
||||||
import { usePanZoom } from '../../composables/usePanZoom.js'
|
import { usePanZoom } from '../../composables/usePanZoom.js'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({ src: String, alt: String })
|
||||||
src: String,
|
|
||||||
alt: String,
|
|
||||||
// Hovered-suggestion state, or null when nothing is hovered:
|
|
||||||
// null → no overlay
|
|
||||||
// { g: {bbox,kind,detector}, tag } → highlight that crop, label with the tag
|
|
||||||
// { g: null, tag } → whole-image frame (not localized)
|
|
||||||
// `tag` is the hovered concept's name (primary label line); `g` is where the
|
|
||||||
// crop came from (secondary provenance line).
|
|
||||||
hover: { type: Object, default: null },
|
|
||||||
})
|
|
||||||
const emit = defineEmits(['close-request'])
|
const emit = defineEmits(['close-request'])
|
||||||
const panZoom = usePanZoom()
|
const panZoom = usePanZoom()
|
||||||
|
|
||||||
const canvasEl = ref(null)
|
const canvasEl = ref(null)
|
||||||
const imgEl = ref(null)
|
const imgEl = ref(null)
|
||||||
const overlayBox = ref(null) // the <img>'s rect relative to the canvas
|
|
||||||
|
|
||||||
// The <img> renders aspect-fit + transformed, so its getBoundingClientRect is
|
|
||||||
// the ground truth for where the picture actually is. Position the overlay to
|
|
||||||
// match it, then place the region as a % within.
|
|
||||||
function measure () {
|
|
||||||
const img = imgEl.value
|
|
||||||
const canvas = canvasEl.value
|
|
||||||
if (!img || !canvas) { overlayBox.value = null; return }
|
|
||||||
const ir = img.getBoundingClientRect()
|
|
||||||
const cr = canvas.getBoundingClientRect()
|
|
||||||
if (!ir.width || !ir.height) { overlayBox.value = null; return }
|
|
||||||
overlayBox.value = {
|
|
||||||
left: ir.left - cr.left, top: ir.top - cr.top,
|
|
||||||
width: ir.width, height: ir.height,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const regionStyle = computed(() => {
|
|
||||||
const g = props.hover?.g
|
|
||||||
if (!g || !Array.isArray(g.bbox)) return null // null grounding → whole frame
|
|
||||||
const [rx, ry, rw, rh] = g.bbox
|
|
||||||
return {
|
|
||||||
left: `${rx * 100}%`, top: `${ry * 100}%`,
|
|
||||||
width: `${rw * 100}%`, height: `${rh * 100}%`,
|
|
||||||
}
|
|
||||||
})
|
|
||||||
// The hovered tag/suggestion name — the PRIMARY label line (what you care about).
|
|
||||||
const regionTag = computed(() => props.hover?.tag || '')
|
|
||||||
// Where the winning crop came from (the region's detector_version, else kind) —
|
|
||||||
// shown small + dimmed beneath the tag as provenance, not the headline. "whole
|
|
||||||
// image" = null grounding (the global vector won, not a crop).
|
|
||||||
const regionOrigin = computed(() => {
|
|
||||||
const g = props.hover?.g
|
|
||||||
return g ? (g.detector || g.kind || 'region') : 'whole image'
|
|
||||||
})
|
|
||||||
|
|
||||||
// Re-measure whenever hovering starts or the image could have moved.
|
|
||||||
watch(() => props.hover, (h) => { if (h) nextTick(measure) })
|
|
||||||
watch(
|
|
||||||
() => [panZoom.state.scale, panZoom.state.x, panZoom.state.y],
|
|
||||||
() => { if (props.hover) measure() },
|
|
||||||
)
|
|
||||||
function onResize () { if (props.hover) measure() }
|
|
||||||
function onImgLoad () { if (props.hover) measure() }
|
|
||||||
onMounted(() => window.addEventListener('resize', onResize))
|
|
||||||
onUnmounted(() => window.removeEventListener('resize', onResize))
|
|
||||||
|
|
||||||
// Reset zoom when the src changes (prev/next nav).
|
// Reset zoom when the src changes (prev/next nav).
|
||||||
watch(() => props.src, () => panZoom.reset())
|
watch(() => props.src, () => panZoom.reset())
|
||||||
@@ -160,53 +79,4 @@ function onCanvasClick(ev) {
|
|||||||
transition: transform 100ms ease-out;
|
transition: transform 100ms ease-out;
|
||||||
pointer-events: none;
|
pointer-events: none;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* Grounding overlay — never intercepts pan/zoom/click. */
|
|
||||||
.fc-canvas__overlay {
|
|
||||||
position: absolute;
|
|
||||||
pointer-events: none;
|
|
||||||
z-index: 2;
|
|
||||||
}
|
|
||||||
.fc-canvas__region {
|
|
||||||
position: absolute;
|
|
||||||
border: 2px solid rgb(var(--v-theme-accent));
|
|
||||||
border-radius: 3px;
|
|
||||||
box-shadow: 0 0 0 9999px rgba(0, 0, 0, 0.28); /* dim everything outside it */
|
|
||||||
transition: all 90ms ease-out;
|
|
||||||
}
|
|
||||||
/* Null grounding: a subtle full-image frame, no dimming — it says "this tag
|
|
||||||
came from the whole image, not a crop." */
|
|
||||||
.fc-canvas__region--whole {
|
|
||||||
inset: 0;
|
|
||||||
border-style: dashed;
|
|
||||||
border-color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
box-shadow: none;
|
|
||||||
}
|
|
||||||
.fc-canvas__region-label {
|
|
||||||
position: absolute;
|
|
||||||
top: 0; left: 0;
|
|
||||||
transform: translateY(-100%);
|
|
||||||
display: flex; flex-direction: column;
|
|
||||||
line-height: 1.25;
|
|
||||||
padding: 2px 6px;
|
|
||||||
white-space: nowrap;
|
|
||||||
background: rgb(var(--v-theme-accent));
|
|
||||||
color: rgb(var(--v-theme-on-surface));
|
|
||||||
border-radius: 3px 3px 0 0;
|
|
||||||
}
|
|
||||||
/* The hovered tag is the headline. */
|
|
||||||
.fc-canvas__region-tag {
|
|
||||||
font-size: 12px;
|
|
||||||
font-weight: 600;
|
|
||||||
}
|
|
||||||
/* Crop origin is provenance, not the point — subordinate: smaller, dimmed, and
|
|
||||||
monospace (it's a detector token like booru:head, not prose). */
|
|
||||||
.fc-canvas__region-origin {
|
|
||||||
font-size: 10px;
|
|
||||||
font-family: 'JetBrains Mono', monospace;
|
|
||||||
color: rgb(var(--v-theme-on-surface), 0.72);
|
|
||||||
}
|
|
||||||
.fc-canvas__region--whole .fc-canvas__region-label {
|
|
||||||
background: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
}
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -60,7 +60,6 @@
|
|||||||
<template v-if="modal.current">
|
<template v-if="modal.current">
|
||||||
<ImageCanvas
|
<ImageCanvas
|
||||||
v-if="!isVideo" :src="modal.current.image_url" :alt="`Image ${modal.current.id}`"
|
v-if="!isVideo" :src="modal.current.image_url" :alt="`Image ${modal.current.id}`"
|
||||||
:hover="hoverGrounding"
|
|
||||||
@close-request="$emit('close')"
|
@close-request="$emit('close')"
|
||||||
/>
|
/>
|
||||||
<VideoCanvas
|
<VideoCanvas
|
||||||
@@ -95,7 +94,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onMounted, onUnmounted, provide, ref, watch } from 'vue'
|
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||||
import { useModalStore } from '../../stores/modal.js'
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import { arrowNavAllowed, isTextEntry } from '../../utils/textEntry.js'
|
import { arrowNavAllowed, isTextEntry } from '../../utils/textEntry.js'
|
||||||
import ImageCanvas from './ImageCanvas.vue'
|
import ImageCanvas from './ImageCanvas.vue'
|
||||||
@@ -111,13 +110,6 @@ const modal = useModalStore()
|
|||||||
const rootEl = ref(null)
|
const rootEl = ref(null)
|
||||||
const showHelp = ref(false)
|
const showHelp = ref(false)
|
||||||
|
|
||||||
// #1206: a deep descendant (SuggestionItem) sets this on hover to the crop that
|
|
||||||
// produced a suggestion; ImageCanvas draws it over the image. provide/inject
|
|
||||||
// avoids relaying a hover event through TagPanel → SuggestionsPanel → group.
|
|
||||||
// null = nothing hovered; { g: grounding|null } = hovering (g null → whole-image).
|
|
||||||
const hoverGrounding = ref(null)
|
|
||||||
provide('fcSuggestionHover', hoverGrounding)
|
|
||||||
|
|
||||||
const isVideo = computed(() =>
|
const isVideo = computed(() =>
|
||||||
modal.current?.mime && modal.current.mime.startsWith('video/')
|
modal.current?.mime && modal.current.mime.startsWith('video/')
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -42,23 +42,13 @@
|
|||||||
· {{ e.post.attachment_count }} files
|
· {{ e.post.attachment_count }} files
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div v-if="hasTranslation(e)" class="fc-prov__actions">
|
|
||||||
<a
|
|
||||||
href="#" @click.prevent="toggleOrig(e.provenance_id)"
|
|
||||||
>{{ showOrig[e.provenance_id] ? 'Show translation' : `Show original${langLabel(e)}` }}</a>
|
|
||||||
</div>
|
|
||||||
<div v-if="e.post.description_html" class="fc-prov__actions">
|
<div v-if="e.post.description_html" class="fc-prov__actions">
|
||||||
<a
|
<a
|
||||||
href="#" @click.prevent="toggleDesc(e.provenance_id)"
|
href="#" @click.prevent="toggleDesc(e.provenance_id)"
|
||||||
>{{ expanded[e.provenance_id] ? 'Hide description ▴' : 'Show description ▾' }}</a>
|
>{{ expanded[e.provenance_id] ? 'Hide description ▴' : 'Show description ▾' }}</a>
|
||||||
</div>
|
</div>
|
||||||
<!-- Translated body = plain text; original = sanitized HTML (#143). -->
|
|
||||||
<p
|
|
||||||
v-if="showTranslated(e) && expanded[e.provenance_id] && e.post.description_translated"
|
|
||||||
class="fc-prov__desc"
|
|
||||||
>{{ e.post.description_translated }}</p>
|
|
||||||
<div
|
<div
|
||||||
v-else-if="e.post.description_html && expanded[e.provenance_id]"
|
v-if="e.post.description_html && expanded[e.provenance_id]"
|
||||||
class="fc-prov__desc" v-html="e.post.description_html"
|
class="fc-prov__desc" v-html="e.post.description_html"
|
||||||
/>
|
/>
|
||||||
</article>
|
</article>
|
||||||
@@ -127,12 +117,8 @@ const effectiveImage = computed(() => props.image ?? modal.current)
|
|||||||
// 2026-05-28. Reset when the viewed image changes.
|
// 2026-05-28. Reset when the viewed image changes.
|
||||||
const expanded = reactive({})
|
const expanded = reactive({})
|
||||||
function toggleDesc(id) { expanded[id] = !expanded[id] }
|
function toggleDesc(id) { expanded[id] = !expanded[id] }
|
||||||
// Per-entry "show the original (untranslated) text" toggle (#143).
|
|
||||||
const showOrig = reactive({})
|
|
||||||
function toggleOrig(id) { showOrig[id] = !showOrig[id] }
|
|
||||||
watch(() => effectiveId.value, () => {
|
watch(() => effectiveId.value, () => {
|
||||||
for (const k of Object.keys(expanded)) delete expanded[k]
|
for (const k of Object.keys(expanded)) delete expanded[k]
|
||||||
for (const k of Object.keys(showOrig)) delete showOrig[k]
|
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(
|
watch(
|
||||||
@@ -171,23 +157,10 @@ const show = computed(() => {
|
|||||||
const attachments = computed(() => state.value?.attachments || [])
|
const attachments = computed(() => state.value?.attachments || [])
|
||||||
|
|
||||||
function postDate(e) { return formatPostDate(e.post.date) }
|
function postDate(e) { return formatPostDate(e.post.date) }
|
||||||
// Translation-forward (#143): show the English by default when present.
|
|
||||||
function hasTranslation(e) {
|
|
||||||
return !!(e.post.title_translated || e.post.description_translated)
|
|
||||||
}
|
|
||||||
function showTranslated(e) {
|
|
||||||
return hasTranslation(e) && !showOrig[e.provenance_id]
|
|
||||||
}
|
|
||||||
function langLabel(e) {
|
|
||||||
return e.post.translated_source_lang ? ` (${e.post.translated_source_lang})` : ''
|
|
||||||
}
|
|
||||||
function postTitle(e) {
|
function postTitle(e) {
|
||||||
// Titles can arrive as stored HTML (e.g. "<strong>…</strong>"); render
|
// Titles can arrive as stored HTML (e.g. "<strong>…</strong>"); render
|
||||||
// as plain text (the CSS makes it bold).
|
// as plain text (the CSS makes it bold).
|
||||||
const t = showTranslated(e) && e.post.title_translated
|
return toPlainText(e.post.title) || `Post ${e.post.external_post_id}`
|
||||||
? e.post.title_translated
|
|
||||||
: e.post.title
|
|
||||||
return toPlainText(t) || `Post ${e.post.external_post_id}`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function openPost(postId, artistId) {
|
function openPost(postId, artistId) {
|
||||||
|
|||||||
@@ -3,14 +3,15 @@
|
|||||||
name, score, and action buttons as one "object" (operator-asked
|
name, score, and action buttons as one "object" (operator-asked
|
||||||
2026-06-01). The row itself is informational; the green ✓ / red ✗
|
2026-06-01). The row itself is informational; the green ✓ / red ✗
|
||||||
verdict pair + 3-dot alias menu are the action affordances. -->
|
verdict pair + 3-dot alias menu are the action affordances. -->
|
||||||
<div
|
<div class="fc-suggestion" :class="{ 'fc-suggestion--rejected': suggestion.rejected }">
|
||||||
class="fc-suggestion" :class="{ 'fc-suggestion--rejected': suggestion.rejected }"
|
|
||||||
@mouseenter="onEnter" @mouseleave="onLeave"
|
|
||||||
>
|
|
||||||
<span class="fc-suggestion__name">
|
<span class="fc-suggestion__name">
|
||||||
{{ suggestion.display_name }}
|
{{ suggestion.display_name }}
|
||||||
<span v-if="suggestion.rejected" class="fc-suggestion__rejected-tag"
|
<span v-if="suggestion.rejected" class="fc-suggestion__rejected-tag"
|
||||||
title="You rejected this for this image — un-reject to recover">rejected</span>
|
title="You rejected this for this image — un-reject to recover">rejected</span>
|
||||||
|
<span v-else-if="suggestion.creates_new_tag" class="fc-suggestion__new"
|
||||||
|
title="No matching tag yet — accepting creates it">+ new</span>
|
||||||
|
<span v-else-if="suggestion.via_alias" class="fc-suggestion__alias"
|
||||||
|
:title="`Mapped from the tagger's “${suggestion.raw_name}” via an alias`">alias</span>
|
||||||
</span>
|
</span>
|
||||||
<span class="fc-suggestion__score">{{ scorePct }}</span>
|
<span class="fc-suggestion__score">{{ scorePct }}</span>
|
||||||
<!-- Green ✓ / red ✗ pair (operator-asked 2026-06-28) mirrors the eval
|
<!-- Green ✓ / red ✗ pair (operator-asked 2026-06-28) mirrors the eval
|
||||||
@@ -42,26 +43,48 @@
|
|||||||
@click="$emit('dismiss', suggestion)"
|
@click="$emit('dismiss', suggestion)"
|
||||||
><v-icon size="16">mdi-close</v-icon></button>
|
><v-icon size="16">mdi-close</v-icon></button>
|
||||||
</div>
|
</div>
|
||||||
|
<!-- Modal-safe kebab is baked into KebabMenu (this row lives in the
|
||||||
|
teleported image modal — #711). Only rendered when an alias action
|
||||||
|
applies — dismiss now lives on the red ✗, so a centroid hit with no
|
||||||
|
alias option has no menu. -->
|
||||||
|
<KebabMenu
|
||||||
|
v-if="hasMenu"
|
||||||
|
class="fc-suggestion__menu" size="small" variant="outlined"
|
||||||
|
:label="`More actions for ${suggestion.display_name}`"
|
||||||
|
>
|
||||||
|
<!-- Alias is a tagger-prediction remap, so only offer it for tagger
|
||||||
|
suggestions with a raw model key that aren't already aliased.
|
||||||
|
Centroid hits (raw_name null) have nothing to alias. -->
|
||||||
|
<v-list-item
|
||||||
|
v-if="suggestion.raw_name && !suggestion.via_alias"
|
||||||
|
@click="$emit('alias', suggestion)"
|
||||||
|
>
|
||||||
|
<v-list-item-title>Treat as alias for…</v-list-item-title>
|
||||||
|
</v-list-item>
|
||||||
|
<v-list-item
|
||||||
|
v-if="suggestion.via_alias"
|
||||||
|
@click="$emit('remove-alias', suggestion)"
|
||||||
|
>
|
||||||
|
<v-list-item-title>Remove alias</v-list-item-title>
|
||||||
|
</v-list-item>
|
||||||
|
</KebabMenu>
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, inject } from 'vue'
|
import { computed } from 'vue'
|
||||||
|
import KebabMenu from '../common/KebabMenu.vue'
|
||||||
|
|
||||||
const props = defineProps({ suggestion: { type: Object, required: true } })
|
const props = defineProps({ suggestion: { type: Object, required: true } })
|
||||||
defineEmits(['accept', 'dismiss', 'undismiss'])
|
defineEmits(['accept', 'alias', 'remove-alias', 'dismiss', 'undismiss'])
|
||||||
|
|
||||||
// #1206: on hover, tell the image viewer which crop produced this suggestion so
|
|
||||||
// it highlights that region. Provided by ImageViewer/Explore; a no-op elsewhere.
|
|
||||||
const hover = inject('fcSuggestionHover', null)
|
|
||||||
function onEnter () {
|
|
||||||
if (hover) hover.value = { g: props.suggestion.grounding ?? null, tag: props.suggestion.display_name }
|
|
||||||
}
|
|
||||||
function onLeave () {
|
|
||||||
if (hover) hover.value = null
|
|
||||||
}
|
|
||||||
|
|
||||||
const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
||||||
|
// Kebab now only carries alias actions: show it when this suggestion can be
|
||||||
|
// aliased (raw model key, not yet aliased) or is already aliased (so it can be
|
||||||
|
// un-aliased). Centroid hits (no raw_name, no alias) have an empty menu → hide.
|
||||||
|
const hasMenu = computed(() =>
|
||||||
|
Boolean(props.suggestion.raw_name) || Boolean(props.suggestion.via_alias)
|
||||||
|
)
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
@@ -83,6 +106,26 @@ const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
|||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
.fc-suggestion__new {
|
||||||
|
display: inline-block;
|
||||||
|
font-size: 10px; font-weight: 600;
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
background: rgba(var(--v-theme-accent), 0.12);
|
||||||
|
border: 1px solid rgb(var(--v-theme-accent), 0.4);
|
||||||
|
padding: 1px 6px; border-radius: 999px;
|
||||||
|
margin-left: 6px;
|
||||||
|
text-transform: uppercase; letter-spacing: 0.04em;
|
||||||
|
}
|
||||||
|
.fc-suggestion__alias {
|
||||||
|
display: inline-block;
|
||||||
|
font-size: 10px; font-weight: 600;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
background: rgb(var(--v-theme-surface-light));
|
||||||
|
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||||
|
padding: 1px 6px; border-radius: 999px;
|
||||||
|
margin-left: 6px;
|
||||||
|
text-transform: uppercase; letter-spacing: 0.04em;
|
||||||
|
}
|
||||||
.fc-suggestion__score {
|
.fc-suggestion__score {
|
||||||
flex: 0 0 auto; min-width: 38px; text-align: right;
|
flex: 0 0 auto; min-width: 38px; text-align: right;
|
||||||
font-size: 11px;
|
font-size: 11px;
|
||||||
@@ -110,6 +153,10 @@ const scorePct = computed(() => `${Math.round(props.suggestion.score * 100)}%`)
|
|||||||
background: transparent; color: rgb(var(--v-theme-on-surface-variant));
|
background: transparent; color: rgb(var(--v-theme-on-surface-variant));
|
||||||
border: 1px solid rgb(var(--v-theme-on-surface-variant), 0.5);
|
border: 1px solid rgb(var(--v-theme-on-surface-variant), 0.5);
|
||||||
}
|
}
|
||||||
|
.fc-suggestion__menu {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
|
|
||||||
/* Rejected state: the row stays put (recovery), dimmed + red-edged so it
|
/* Rejected state: the row stays put (recovery), dimmed + red-edged so it
|
||||||
reads as "handled, negative" without shouting over live suggestions. */
|
reads as "handled, negative" without shouting over live suggestions. */
|
||||||
.fc-suggestion--rejected {
|
.fc-suggestion--rejected {
|
||||||
|
|||||||
@@ -1,31 +1,22 @@
|
|||||||
<template>
|
<template>
|
||||||
<div class="fc-sgroup">
|
<div class="fc-sgroup">
|
||||||
<div class="fc-sgroup__header-row">
|
<button
|
||||||
<button
|
v-if="collapsible"
|
||||||
v-if="collapsible"
|
class="fc-sgroup__header fc-sgroup__header--btn"
|
||||||
class="fc-sgroup__header fc-sgroup__header--btn"
|
@click="open = !open"
|
||||||
@click="open = !open"
|
>
|
||||||
>
|
<v-icon size="small">{{ open ? 'mdi-chevron-down' : 'mdi-chevron-right' }}</v-icon>
|
||||||
<v-icon size="small">{{ open ? 'mdi-chevron-down' : 'mdi-chevron-right' }}</v-icon>
|
{{ label }} ({{ items.length }})
|
||||||
{{ label }} ({{ items.length }})
|
</button>
|
||||||
</button>
|
<div v-else class="fc-sgroup__header">{{ label }}</div>
|
||||||
<div v-else class="fc-sgroup__header">{{ label }}</div>
|
|
||||||
<!-- "Confirm what's right, reject the rest": clears every still-unhandled
|
|
||||||
suggestion in this section at once. Reversible (each stays flagged
|
|
||||||
rejected with one-click un-reject), so no confirm dialog. -->
|
|
||||||
<button
|
|
||||||
v-if="rejectableCount > 0"
|
|
||||||
class="fc-sgroup__reject-rest" type="button"
|
|
||||||
:title="`Reject the ${rejectableCount} remaining ${label} suggestion${rejectableCount === 1 ? '' : 's'}`"
|
|
||||||
@click="$emit('reject-all')"
|
|
||||||
>Reject rest</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div v-show="open" class="fc-sgroup__items">
|
<div v-show="open" class="fc-sgroup__items">
|
||||||
<SuggestionItem
|
<SuggestionItem
|
||||||
v-for="(s, i) in items" :key="`${s.display_name}-${i}`"
|
v-for="(s, i) in items" :key="`${s.display_name}-${i}`"
|
||||||
:suggestion="s"
|
:suggestion="s"
|
||||||
@accept="$emit('accept', $event)"
|
@accept="$emit('accept', $event)"
|
||||||
|
@alias="$emit('alias', $event)"
|
||||||
|
@remove-alias="$emit('remove-alias', $event)"
|
||||||
@dismiss="$emit('dismiss', $event)"
|
@dismiss="$emit('dismiss', $event)"
|
||||||
@undismiss="$emit('undismiss', $event)"
|
@undismiss="$emit('undismiss', $event)"
|
||||||
/>
|
/>
|
||||||
@@ -34,7 +25,7 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import SuggestionItem from './SuggestionItem.vue'
|
import SuggestionItem from './SuggestionItem.vue'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
@@ -43,45 +34,24 @@ const props = defineProps({
|
|||||||
collapsible: { type: Boolean, default: false },
|
collapsible: { type: Boolean, default: false },
|
||||||
defaultOpen: { type: Boolean, default: true }
|
defaultOpen: { type: Boolean, default: true }
|
||||||
})
|
})
|
||||||
defineEmits(['accept', 'dismiss', 'undismiss', 'reject-all'])
|
defineEmits(['accept', 'alias', 'remove-alias', 'dismiss', 'undismiss'])
|
||||||
|
|
||||||
// Still-unhandled suggestions (not yet rejected) — how many "Reject rest" clears.
|
|
||||||
const rejectableCount = computed(() => props.items.filter((s) => !s.rejected).length)
|
|
||||||
const open = ref(props.collapsible ? props.defaultOpen : true)
|
const open = ref(props.collapsible ? props.defaultOpen : true)
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
.fc-sgroup { margin-bottom: 10px; }
|
.fc-sgroup { margin-bottom: 10px; }
|
||||||
.fc-sgroup__header-row {
|
|
||||||
display: flex; align-items: center; gap: 8px;
|
|
||||||
margin-bottom: 4px;
|
|
||||||
}
|
|
||||||
.fc-sgroup__header {
|
.fc-sgroup__header {
|
||||||
flex: 1 1 auto; min-width: 0;
|
|
||||||
font-family: 'Inter', sans-serif;
|
font-family: 'Inter', sans-serif;
|
||||||
font-size: 11px; font-weight: 600;
|
font-size: 11px; font-weight: 600;
|
||||||
text-transform: uppercase; letter-spacing: 0.06em;
|
text-transform: uppercase; letter-spacing: 0.06em;
|
||||||
color: rgb(var(--v-theme-on-surface-variant, var(--v-theme-on-surface)));
|
color: rgb(var(--v-theme-on-surface-variant, var(--v-theme-on-surface)));
|
||||||
|
margin-bottom: 4px;
|
||||||
}
|
}
|
||||||
.fc-sgroup__header--btn {
|
.fc-sgroup__header--btn {
|
||||||
display: flex; align-items: center; gap: 4px;
|
display: flex; align-items: center; gap: 4px;
|
||||||
background: none; border: none; cursor: pointer;
|
background: none; border: none; cursor: pointer;
|
||||||
padding: 0; text-align: left;
|
padding: 0; width: 100%; text-align: left;
|
||||||
font: inherit; text-transform: uppercase; letter-spacing: 0.06em;
|
font: inherit; text-transform: uppercase; letter-spacing: 0.06em;
|
||||||
}
|
}
|
||||||
/* Section-level "reject the remaining" — subtle until hovered so it doesn't
|
|
||||||
compete with the per-row verdicts. */
|
|
||||||
.fc-sgroup__reject-rest {
|
|
||||||
flex: 0 0 auto;
|
|
||||||
background: none; border: none; cursor: pointer;
|
|
||||||
font-family: 'Inter', sans-serif;
|
|
||||||
font-size: 10px; font-weight: 600;
|
|
||||||
text-transform: uppercase; letter-spacing: 0.04em;
|
|
||||||
color: rgb(var(--v-theme-error));
|
|
||||||
padding: 2px 5px; border-radius: 4px;
|
|
||||||
}
|
|
||||||
.fc-sgroup__reject-rest:hover { background: rgb(var(--v-theme-error), 0.1); }
|
|
||||||
.fc-sgroup__reject-rest:focus-visible {
|
|
||||||
outline: 2px solid rgb(var(--v-theme-error)); outline-offset: 1px;
|
|
||||||
}
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -20,39 +20,45 @@
|
|||||||
first, so false positives are easy to spot and reject (reject-to-train
|
first, so false positives are easy to spot and reject (reject-to-train
|
||||||
for these heads). Small set — collapsible, open by default. -->
|
for these heads). Small set — collapsible, open by default. -->
|
||||||
<SuggestionsCategoryGroup
|
<SuggestionsCategoryGroup
|
||||||
v-if="store.aboveByCategory.system && store.aboveByCategory.system.length"
|
v-if="store.byCategory.system && store.byCategory.system.length"
|
||||||
label="System" :items="store.aboveByCategory.system"
|
label="System" :items="store.byCategory.system"
|
||||||
collapsible :default-open="true"
|
collapsible :default-open="true"
|
||||||
@accept="onAccept"
|
@accept="onAccept" @alias="onAlias" @remove-alias="onRemoveAlias"
|
||||||
@dismiss="onDismiss" @undismiss="onUndismiss"
|
@dismiss="onDismiss" @undismiss="onUndismiss"
|
||||||
@reject-all="onRejectAll('system')"
|
|
||||||
/>
|
/>
|
||||||
<SuggestionsCategoryGroup
|
<SuggestionsCategoryGroup
|
||||||
v-for="cat in peopleCats" :key="cat"
|
v-for="cat in peopleCats" :key="cat"
|
||||||
v-show="store.aboveByCategory[cat] && store.aboveByCategory[cat].length"
|
v-show="store.byCategory[cat] && store.byCategory[cat].length"
|
||||||
:label="labelFor(cat)" :items="store.aboveByCategory[cat] || []"
|
:label="labelFor(cat)" :items="store.byCategory[cat] || []"
|
||||||
@accept="onAccept"
|
@accept="onAccept" @alias="onAlias" @remove-alias="onRemoveAlias"
|
||||||
@dismiss="onDismiss" @undismiss="onUndismiss"
|
@dismiss="onDismiss" @undismiss="onUndismiss"
|
||||||
@reject-all="onRejectAll(cat)"
|
|
||||||
/>
|
/>
|
||||||
<SuggestionsCategoryGroup
|
<SuggestionsCategoryGroup
|
||||||
v-if="store.aboveByCategory.general && store.aboveByCategory.general.length"
|
v-if="store.byCategory.general && store.byCategory.general.length"
|
||||||
label="General" :items="store.aboveByCategory.general"
|
label="General" :items="store.byCategory.general"
|
||||||
collapsible :default-open="true"
|
collapsible :default-open="true"
|
||||||
@accept="onAccept"
|
@accept="onAccept" @alias="onAlias" @remove-alias="onRemoveAlias"
|
||||||
@dismiss="onDismiss" @undismiss="onUndismiss"
|
@dismiss="onDismiss" @undismiss="onUndismiss"
|
||||||
@reject-all="onRejectAll('general')"
|
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<v-dialog v-model="aliasDialog" max-width="480">
|
||||||
|
<AliasPickerDialog
|
||||||
|
v-if="aliasTarget"
|
||||||
|
:category="aliasTarget.category"
|
||||||
|
@confirm="onAliasConfirm" @cancel="aliasDialog = false"
|
||||||
|
/>
|
||||||
|
</v-dialog>
|
||||||
</section>
|
</section>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { toast } from '../../utils/toast.js'
|
import { toast } from '../../utils/toast.js'
|
||||||
import { computed, watch } from 'vue'
|
import { computed, ref, watch } from 'vue'
|
||||||
import { useSuggestionsStore, CATEGORY_LABELS } from '../../stores/suggestions.js'
|
import { useSuggestionsStore, CATEGORY_LABELS } from '../../stores/suggestions.js'
|
||||||
import { useModalStore } from '../../stores/modal.js'
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import SuggestionsCategoryGroup from './SuggestionsCategoryGroup.vue'
|
import SuggestionsCategoryGroup from './SuggestionsCategoryGroup.vue'
|
||||||
|
import AliasPickerDialog from './AliasPickerDialog.vue'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
imageId: { type: Number, required: true },
|
imageId: { type: Number, required: true },
|
||||||
@@ -70,16 +76,6 @@ const emit = defineEmits(['accepted', 'dismissed'])
|
|||||||
// re-focus the tag input — same return-to-input behaviour as accept.
|
// re-focus the tag input — same return-to-input behaviour as accept.
|
||||||
function onDismiss (s) { store.dismiss(s); emit('dismissed') }
|
function onDismiss (s) { store.dismiss(s); emit('dismissed') }
|
||||||
function onUndismiss (s) { store.undismiss(s); emit('dismissed') }
|
function onUndismiss (s) { store.undismiss(s); emit('dismissed') }
|
||||||
// Section-level "Reject rest": dismiss every still-unhandled suggestion in the
|
|
||||||
// category, then return focus to the input like a single reject does.
|
|
||||||
async function onRejectAll (category) {
|
|
||||||
try {
|
|
||||||
await store.dismissRemaining(category)
|
|
||||||
emit('dismissed')
|
|
||||||
} catch (e) {
|
|
||||||
toast({ text: `Reject failed: ${e.message}`, type: 'error' })
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const store = useSuggestionsStore()
|
const store = useSuggestionsStore()
|
||||||
const modalStore = useModalStore()
|
const modalStore = useModalStore()
|
||||||
const host = props.host || modalStore
|
const host = props.host || modalStore
|
||||||
@@ -91,14 +87,13 @@ const peopleCats = ['character']
|
|||||||
function labelFor(c) { return CATEGORY_LABELS[c] || c }
|
function labelFor(c) { return CATEGORY_LABELS[c] || c }
|
||||||
|
|
||||||
const isEmpty = computed(() =>
|
const isEmpty = computed(() =>
|
||||||
Object.values(store.aboveByCategory).every(list => !list || list.length === 0)
|
Object.values(store.byCategory).every(list => !list || list.length === 0)
|
||||||
)
|
)
|
||||||
|
|
||||||
watch(() => props.imageId, (id) => {
|
watch(() => props.imageId, (id) => {
|
||||||
if (id == null) return
|
if (id == null) return
|
||||||
// One fetch (min=0) backs both surfaces: the panel reads aboveByCategory, the
|
store.load(id) // panel: curated, ≥ threshold
|
||||||
// typed tag-input dropdown reads the full set. No second request.
|
store.loadAll(id) // dropdown: full prediction set (low-confidence included)
|
||||||
store.load(id)
|
|
||||||
}, { immediate: true })
|
}, { immediate: true })
|
||||||
|
|
||||||
// After a successful accept/alias-accept, refresh the modal's current
|
// After a successful accept/alias-accept, refresh the modal's current
|
||||||
@@ -117,6 +112,31 @@ async function onAccept(s) {
|
|||||||
toast({ text: `Accept failed: ${e.message}`, type: 'error' })
|
toast({ text: `Accept failed: ${e.message}`, type: 'error' })
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const aliasDialog = ref(false)
|
||||||
|
const aliasTarget = ref(null)
|
||||||
|
function onAlias(s) { aliasTarget.value = s; aliasDialog.value = true }
|
||||||
|
async function onAliasConfirm(canonicalTagId) {
|
||||||
|
try {
|
||||||
|
await store.aliasAccept(aliasTarget.value, canonicalTagId)
|
||||||
|
aliasDialog.value = false
|
||||||
|
await host.reloadTags()
|
||||||
|
emit('accepted')
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Alias failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Undo the model-key→tag mapping behind an aliased suggestion. The store
|
||||||
|
// reloads suggestions so the prediction reverts to its raw form; the applied
|
||||||
|
// canonical tag (if any) stays, so no tag-rail reload is needed.
|
||||||
|
async function onRemoveAlias(s) {
|
||||||
|
try {
|
||||||
|
await store.removeAlias(s)
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Remove alias failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -17,14 +17,7 @@
|
|||||||
density="compact" class="fc-tag-autocomplete__list"
|
density="compact" class="fc-tag-autocomplete__list"
|
||||||
>
|
>
|
||||||
<template v-for="(row, idx) in rows" :key="row.key">
|
<template v-for="(row, idx) in rows" :key="row.key">
|
||||||
<!-- One unified row per DB tag (server autocomplete). Every suggestion is
|
<!-- Existing tag match (server autocomplete). -->
|
||||||
a canonical tag now, so when the model ALSO scored this tag for the
|
|
||||||
image the row just carries its confidence (🔧 %) in place of the
|
|
||||||
redundant kind label — the coloured leading icon already shows kind.
|
|
||||||
That means a searched tag that's also a suggestion shows its % on ONE
|
|
||||||
row, with no dedup and no flicker. Picking it emits pick-existing;
|
|
||||||
TagPanel.findPending routes a matching suggestion through accept() so
|
|
||||||
the acceptance is still recorded + it drops from the panel. -->
|
|
||||||
<v-list-item
|
<v-list-item
|
||||||
v-if="row.type === 'hit'"
|
v-if="row.type === 'hit'"
|
||||||
:active="idx === highlight" @click="onPickRow(row)"
|
:active="idx === highlight" @click="onPickRow(row)"
|
||||||
@@ -39,18 +32,32 @@
|
|||||||
<span v-if="row.hit.fandom_name" class="text-caption">— {{ row.hit.fandom_name }}</span>
|
<span v-if="row.hit.fandom_name" class="text-caption">— {{ row.hit.fandom_name }}</span>
|
||||||
</v-list-item-title>
|
</v-list-item-title>
|
||||||
<template #append>
|
<template #append>
|
||||||
<span
|
<span class="text-caption">{{ row.hit.kind }}</span>
|
||||||
v-if="row.sugg"
|
</template>
|
||||||
class="fc-tag-autocomplete__sugg-tag"
|
</v-list-item>
|
||||||
:class="{ 'fc-tag-autocomplete__sugg-tag--below': !row.sugg.above_threshold }"
|
|
||||||
:title="row.sugg.above_threshold
|
<!-- ML suggestion for THIS image that matches the typed query. Picking
|
||||||
? 'The model suggests this tag for this image'
|
it routes through the same accept path as the Suggestions panel, so
|
||||||
: 'The model scored this tag below its suggest threshold'"
|
it's recorded + drops out of the panel (operator-asked 2026-06-07). -->
|
||||||
>
|
<v-list-item
|
||||||
|
v-else-if="row.type === 'suggestion'"
|
||||||
|
:active="idx === highlight" @click="onPickRow(row)"
|
||||||
|
class="fc-tag-autocomplete__sugg"
|
||||||
|
>
|
||||||
|
<template #prepend>
|
||||||
|
<v-icon size="small" :color="store.colorFor(row.sugg.category)">
|
||||||
|
{{ iconFor(row.sugg.category) }}
|
||||||
|
</v-icon>
|
||||||
|
</template>
|
||||||
|
<v-list-item-title>
|
||||||
|
{{ row.sugg.display_name }}
|
||||||
|
<span v-if="row.sugg.creates_new_tag" class="text-caption">— new</span>
|
||||||
|
</v-list-item-title>
|
||||||
|
<template #append>
|
||||||
|
<span class="fc-tag-autocomplete__sugg-tag">
|
||||||
<v-icon size="x-small">mdi-auto-fix</v-icon>
|
<v-icon size="x-small">mdi-auto-fix</v-icon>
|
||||||
{{ scorePct(row.sugg) }}
|
{{ scorePct(row.sugg) }}
|
||||||
</span>
|
</span>
|
||||||
<span v-else class="text-caption">{{ row.hit.kind }}</span>
|
|
||||||
</template>
|
</template>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
|
|
||||||
@@ -90,13 +97,13 @@
|
|||||||
import { computed, nextTick, onMounted, ref, watch } from 'vue'
|
import { computed, nextTick, onMounted, ref, watch } from 'vue'
|
||||||
import { useTagStore } from '../../stores/tags.js'
|
import { useTagStore } from '../../stores/tags.js'
|
||||||
import { useSuggestionsStore } from '../../stores/suggestions.js'
|
import { useSuggestionsStore } from '../../stores/suggestions.js'
|
||||||
import { useInflightToken } from '../../composables/useInflightToken.js'
|
|
||||||
import FandomPicker from './FandomPicker.vue'
|
import FandomPicker from './FandomPicker.vue'
|
||||||
|
|
||||||
const emit = defineEmits(['pick-existing', 'pick-new', 'cancel'])
|
const emit = defineEmits(['pick-existing', 'pick-new', 'accept-suggestion', 'cancel'])
|
||||||
// The host surface's applied tags (host.current?.tags). The dropdown filters
|
// The host surface's applied tags (host.current?.tags). Both dropdown sections
|
||||||
// against this so a just-added tag drops out of the search the moment the chip
|
// filter against this so a just-added tag drops out of the search the moment
|
||||||
// rail updates, not after the next modal refresh (operator-asked 2026-07-03).
|
// the chip rail updates, not after the next modal refresh (operator-asked
|
||||||
|
// 2026-07-03).
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
appliedTags: { type: Array, default: () => [] },
|
appliedTags: { type: Array, default: () => [] },
|
||||||
})
|
})
|
||||||
@@ -120,16 +127,7 @@ const suggestions = useSuggestionsStore()
|
|||||||
const inputRef = ref(null)
|
const inputRef = ref(null)
|
||||||
function focusInput () {
|
function focusInput () {
|
||||||
if (window.matchMedia?.('(max-width: 900px)')?.matches) return
|
if (window.matchMedia?.('(max-width: 900px)')?.matches) return
|
||||||
// Focus the inner <input> with preventScroll: handing focus back after an
|
nextTick(() => inputRef.value?.focus?.())
|
||||||
// accept/reject must NOT yank the rail's scroll up to this field, which sits
|
|
||||||
// above the suggestions list (operator-flagged 2026-07-06 — "I have to scroll
|
|
||||||
// back down every time I apply/reject"). Vuetify's own .focus() forwards no
|
|
||||||
// options, so reach the element directly; fall back to it if it's not there.
|
|
||||||
nextTick(() => {
|
|
||||||
const el = inputRef.value?.$el?.querySelector?.('input')
|
|
||||||
if (el) el.focus({ preventScroll: true })
|
|
||||||
else inputRef.value?.focus?.()
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
onMounted(focusInput)
|
onMounted(focusInput)
|
||||||
// Exposed so the parent (TagPanel) can hand focus back to this field after an
|
// Exposed so the parent (TagPanel) can hand focus back to this field after an
|
||||||
@@ -176,26 +174,17 @@ const parsed = computed(() => {
|
|||||||
const parsedKind = computed(() => parsed.value.kind)
|
const parsedKind = computed(() => parsed.value.kind)
|
||||||
const parsedName = computed(() => parsed.value.name)
|
const parsedName = computed(() => parsed.value.name)
|
||||||
|
|
||||||
// Inflight guard: the debounce only clears the TIMER, so once a fetch has fired
|
|
||||||
// it still races later ones — a slower earlier-prefix response ("s") could land
|
|
||||||
// after "sex" and overwrite the dropdown with stale, wrong-prefix matches
|
|
||||||
// (operator-flagged 2026-07-06). Gate each response on a token so only the latest
|
|
||||||
// query's results are applied.
|
|
||||||
const acInflight = useInflightToken()
|
|
||||||
let debounceId = null
|
let debounceId = null
|
||||||
watch(query, () => {
|
watch(query, () => {
|
||||||
highlight.value = 0
|
highlight.value = 0
|
||||||
if (debounceId) clearTimeout(debounceId)
|
if (debounceId) clearTimeout(debounceId)
|
||||||
acInflight.cancel()
|
|
||||||
debounceId = setTimeout(async () => {
|
debounceId = setTimeout(async () => {
|
||||||
const q = parsedName.value
|
const q = parsedName.value
|
||||||
if (!q) { hits.value = []; return }
|
if (!q) { hits.value = []; return }
|
||||||
// Autocomplete across ALL kinds. When the user typed a prefix the
|
// Autocomplete across ALL kinds. When the user typed a prefix the
|
||||||
// matches list is naturally narrower because the parsed name is
|
// matches list is naturally narrower because the parsed name is
|
||||||
// shorter; we don't filter server-side by kind.
|
// shorter; we don't filter server-side by kind.
|
||||||
const t = acInflight.claim()
|
hits.value = await store.autocomplete(q, null, 10)
|
||||||
const res = await store.autocomplete(q, null, 10)
|
|
||||||
if (t.isCurrent()) hits.value = res
|
|
||||||
}, 200)
|
}, 200)
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -230,6 +219,9 @@ const createLabel = computed(() =>
|
|||||||
function scorePct (s) { return `${Math.round(s.score * 100)}%` }
|
function scorePct (s) { return `${Math.round(s.score * 100)}%` }
|
||||||
|
|
||||||
const appliedIds = computed(() => new Set(props.appliedTags.map(t => t.id)))
|
const appliedIds = computed(() => new Set(props.appliedTags.map(t => t.id)))
|
||||||
|
const appliedNames = computed(() =>
|
||||||
|
new Set(props.appliedTags.map(t => `${t.kind}:${(t.name || '').toLowerCase()}`)),
|
||||||
|
)
|
||||||
|
|
||||||
// Server matches minus the image's applied tags. allowCreate/sameNameCharExists
|
// Server matches minus the image's applied tags. allowCreate/sameNameCharExists
|
||||||
// keep reading the RAW hits — they reason about the tag universe (does this tag
|
// keep reading the RAW hits — they reason about the tag universe (does this tag
|
||||||
@@ -238,29 +230,49 @@ const visibleHits = computed(() =>
|
|||||||
hits.value.filter(h => !appliedIds.value.has(h.id)),
|
hits.value.filter(h => !appliedIds.value.has(h.id)),
|
||||||
)
|
)
|
||||||
|
|
||||||
// This image's suggestions keyed by canonical tag id, so a matching DB-tag row
|
// This image's suggestions that match the typed query, minus any the server
|
||||||
// can show the model's confidence inline. Every suggestion is a canonical tag
|
// autocomplete already returned (same name+kind) so a tag never shows twice.
|
||||||
// now (tagging-v2), so the id is the join key — no name/kind matching, no dedup,
|
// Sources the FULL prediction set (allByCategory, down to the store floor) — NOT
|
||||||
// no flicker. Rejected suggestions are excluded: a dismissed tag shouldn't
|
// the threshold-filtered panel list — so a low-confidence action/feature the
|
||||||
// advertise a score in the type-to-add dropdown (un-reject lives in the panel).
|
// model saw can be typed and accepted in canonical formatting instead of being
|
||||||
const suggByTagId = computed(() => {
|
// hand-entered as a custom tag (operator-asked 2026-06-09). The typed query is
|
||||||
const m = new Map()
|
// the only filter; the threshold no longer hides anything here.
|
||||||
for (const list of Object.values(suggestions.byCategory)) {
|
const suggestionHits = computed(() => {
|
||||||
|
const q = parsedName.value.toLowerCase()
|
||||||
|
if (!q) return []
|
||||||
|
const seen = new Set(visibleHits.value.map(h => `${h.kind}:${h.name.toLowerCase()}`))
|
||||||
|
const out = []
|
||||||
|
for (const list of Object.values(suggestions.allByCategory)) {
|
||||||
for (const s of list || []) {
|
for (const s of list || []) {
|
||||||
|
// Rejected suggestions now stay in allByCategory (flagged) so the panel
|
||||||
|
// can show + un-reject them; keep them OUT of the type-to-add dropdown,
|
||||||
|
// whose job is finding a tag to ADD (un-reject lives in the panel).
|
||||||
if (s.rejected) continue
|
if (s.rejected) continue
|
||||||
if (s.canonical_tag_id != null) m.set(s.canonical_tag_id, s)
|
// Already on the image (matched by canonical id or name+kind): nothing
|
||||||
|
// to add. Covers the window between an add and the next suggestions
|
||||||
|
// fetch, where the stale row would otherwise still be pickable.
|
||||||
|
if (s.canonical_tag_id != null && appliedIds.value.has(s.canonical_tag_id)) continue
|
||||||
|
const key = `${s.category}:${s.display_name.toLowerCase()}`
|
||||||
|
if (appliedNames.value.has(key)) continue
|
||||||
|
if (!s.display_name.toLowerCase().includes(q)) continue
|
||||||
|
if (seen.has(key)) continue
|
||||||
|
seen.add(key)
|
||||||
|
out.push(s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return m
|
// Best matches first; cap generously so a specific typed query surfaces its
|
||||||
|
// matches even when many predictions exist, while the list stays scrollable.
|
||||||
|
out.sort((a, b) => b.score - a.score)
|
||||||
|
return out.slice(0, 20)
|
||||||
})
|
})
|
||||||
|
|
||||||
// One ordered list backing both the rendered dropdown and keyboard nav, so the
|
// One ordered list backing both the rendered dropdown and keyboard nav, so the
|
||||||
// highlight index maps 1:1 to a row. Each hit is annotated with its suggestion
|
// highlight index maps 1:1 to a row regardless of which section it's in.
|
||||||
// (score) when the model scored that tag for this image.
|
|
||||||
const rows = computed(() => {
|
const rows = computed(() => {
|
||||||
const r = visibleHits.value.map(h => ({
|
const r = visibleHits.value.map(h => ({ type: 'hit', key: `h${h.id}`, hit: h }))
|
||||||
type: 'hit', key: `h${h.id}`, hit: h, sugg: suggByTagId.value.get(h.id) || null,
|
for (const s of suggestionHits.value) {
|
||||||
}))
|
r.push({ type: 'suggestion', key: `s${s.category}:${s.display_name}`, sugg: s })
|
||||||
|
}
|
||||||
if (allowCreate.value) r.push({ type: 'create', key: 'create' })
|
if (allowCreate.value) r.push({ type: 'create', key: 'create' })
|
||||||
return r
|
return r
|
||||||
})
|
})
|
||||||
@@ -282,9 +294,16 @@ function moveHighlight (delta) {
|
|||||||
// Dispatch a chosen dropdown row by its type.
|
// Dispatch a chosen dropdown row by its type.
|
||||||
function onPickRow (row) {
|
function onPickRow (row) {
|
||||||
if (row.type === 'hit') { emit('pick-existing', row.hit); reset() }
|
if (row.type === 'hit') { emit('pick-existing', row.hit); reset() }
|
||||||
|
else if (row.type === 'suggestion') { onPickSuggestion(row.sugg) }
|
||||||
else { onCreate() }
|
else { onCreate() }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Picking an ML suggestion hands it to the parent, which runs the same accept
|
||||||
|
// flow as the Suggestions panel (creates the tag if raw, records acceptance,
|
||||||
|
// drops it from the panel). reset() clears the query; the panel-drop makes it
|
||||||
|
// fall out of suggestionHits reactively.
|
||||||
|
function onPickSuggestion (s) { emit('accept-suggestion', s); reset() }
|
||||||
|
|
||||||
function onCreate () {
|
function onCreate () {
|
||||||
const name = parsedName.value
|
const name = parsedName.value
|
||||||
const kind = parsedKind.value
|
const kind = parsedKind.value
|
||||||
@@ -351,16 +370,14 @@ function reset () { query.value = ''; hits.value = []; highlight.value = 0 }
|
|||||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||||
border-radius: 6px;
|
border-radius: 6px;
|
||||||
}
|
}
|
||||||
/* The model-confidence badge on a row whose tag the model also scored. Accent
|
/* Mark ML-suggestion rows so they read as distinct from typed/known matches. */
|
||||||
when above the head's suggest threshold; muted when below (a low-confidence
|
.fc-tag-autocomplete__sugg {
|
||||||
match you can still type + pick). */
|
border-left: 2px solid rgb(var(--v-theme-accent), 0.5);
|
||||||
|
}
|
||||||
.fc-tag-autocomplete__sugg-tag {
|
.fc-tag-autocomplete__sugg-tag {
|
||||||
display: inline-flex; align-items: center; gap: 2px;
|
display: inline-flex; align-items: center; gap: 2px;
|
||||||
font-size: 11px;
|
font-size: 11px;
|
||||||
font-family: 'JetBrains Mono', monospace;
|
font-family: 'JetBrains Mono', monospace;
|
||||||
color: rgb(var(--v-theme-accent));
|
color: rgb(var(--v-theme-accent));
|
||||||
}
|
}
|
||||||
.fc-tag-autocomplete__sugg-tag--below {
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
}
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
<template>
|
<template>
|
||||||
<span class="fc-tag-chip" @mouseenter="onEnter" @mouseleave="onLeave">
|
<span class="fc-tag-chip">
|
||||||
<v-chip
|
<v-chip
|
||||||
size="default" :closable="!unconfirmedAuto"
|
size="small" closable
|
||||||
:color="store.colorFor(tag.kind)" variant="tonal"
|
:color="store.colorFor(tag.kind)" variant="tonal"
|
||||||
class="fc-tag-chip__nav"
|
class="fc-tag-chip__nav"
|
||||||
role="link"
|
role="link"
|
||||||
@@ -9,32 +9,14 @@
|
|||||||
@click="$emit('navigate', tag)"
|
@click="$emit('navigate', tag)"
|
||||||
@click:close="$emit('remove', tag.id)"
|
@click:close="$emit('remove', tag.id)"
|
||||||
>
|
>
|
||||||
<v-icon start size="small">{{ iconFor(tag.kind) }}</v-icon>
|
<v-icon start size="x-small">{{ iconFor(tag.kind) }}</v-icon>
|
||||||
<span class="fc-tag-chip__name">{{ tag.name }}</span><v-icon
|
{{ tag.name }}<v-icon
|
||||||
v-if="tag.is_system" end size="x-small" class="fc-tag-chip__system"
|
v-if="tag.is_system" end size="x-small" class="fc-tag-chip__system"
|
||||||
title="System tag — tagged items are excluded from training other concepts"
|
title="System tag — tagged items are excluded from training other concepts"
|
||||||
>mdi-shield-outline</v-icon><span
|
>mdi-shield-outline</v-icon><span
|
||||||
v-if="tag.fandom_id" class="fc-tag-chip__fandom"
|
v-if="tag.fandom_id" class="fc-tag-chip__fandom"
|
||||||
:title="tag.fandom_name ? `Fandom: ${tag.fandom_name}` : 'Has a fandom'"
|
:title="tag.fandom_name ? `Fandom: ${tag.fandom_name}` : 'Has a fandom'"
|
||||||
>→<template v-if="fandomLabel"> {{ fandomLabel }}</template></span>
|
>→<template v-if="fandomLabel"> {{ fandomLabel }}</template></span>
|
||||||
<!-- Provisional auto-tag: an in-pill yes/no pair REPLACES the ✕ (the pair
|
|
||||||
itself signals "auto" — no separate label, operator 2026-07-07). Yes
|
|
||||||
confirms it (trains + shields from retraction); No removes it (records
|
|
||||||
a negative). Both hand focus back to the tag input via TagPanel. -->
|
|
||||||
<span v-if="unconfirmedAuto" class="fc-tag-chip__verdict">
|
|
||||||
<button
|
|
||||||
type="button" class="fc-tag-chip__yes"
|
|
||||||
:title="`Yes — keep “${tag.name}” (trains the model, won't be retracted)`"
|
|
||||||
:aria-label="`Confirm ${tag.name}`"
|
|
||||||
@click.stop="$emit('confirm', tag)"
|
|
||||||
><v-icon size="15">mdi-check</v-icon></button>
|
|
||||||
<button
|
|
||||||
type="button" class="fc-tag-chip__no"
|
|
||||||
:title="`No — remove “${tag.name}”`"
|
|
||||||
:aria-label="`Reject ${tag.name}`"
|
|
||||||
@click.stop="$emit('remove', tag.id)"
|
|
||||||
><v-icon size="15">mdi-close</v-icon></button>
|
|
||||||
</span>
|
|
||||||
</v-chip>
|
</v-chip>
|
||||||
<!-- Modal-safe kebab is baked into KebabMenu (this chip lives in the
|
<!-- Modal-safe kebab is baked into KebabMenu (this chip lives in the
|
||||||
teleported image modal — #711). System tags hide it entirely: rename
|
teleported image modal — #711). System tags hide it entirely: rename
|
||||||
@@ -56,67 +38,14 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, inject } from 'vue'
|
import { computed } from 'vue'
|
||||||
import { useTagStore } from '../../stores/tags.js'
|
import { useTagStore } from '../../stores/tags.js'
|
||||||
import { useApi } from '../../composables/useApi.js'
|
|
||||||
import KebabMenu from '../common/KebabMenu.vue'
|
import KebabMenu from '../common/KebabMenu.vue'
|
||||||
|
|
||||||
const AUTO_SOURCES = ['head_auto', 'ccip_auto', 'ml_auto']
|
const props = defineProps({ tag: { type: Object, required: true } })
|
||||||
|
defineEmits(['remove', 'rename', 'set-fandom', 'navigate'])
|
||||||
const props = defineProps({
|
|
||||||
tag: { type: Object, required: true },
|
|
||||||
// When set (the tagging panels), hovering the chip asks the backend which crop
|
|
||||||
// region best explains this applied tag and lights it up on the image — the
|
|
||||||
// same overlay the suggestion rail uses (#1206 Step 4). Omitted elsewhere →
|
|
||||||
// the hover is inert (no injected target, or no image to ground against).
|
|
||||||
imageId: { type: Number, default: null },
|
|
||||||
})
|
|
||||||
defineEmits(['remove', 'rename', 'set-fandom', 'navigate', 'confirm'])
|
|
||||||
|
|
||||||
const store = useTagStore()
|
const store = useTagStore()
|
||||||
const api = useApi()
|
|
||||||
|
|
||||||
// An auto-applied tag the operator hasn't confirmed yet — provisional (milestone
|
|
||||||
// 139): it doesn't train the model and the retraction sweep can drop it. Shows an
|
|
||||||
// in-pill yes/no pair in place of the ✕. `source`/`confirmed` come from the
|
|
||||||
// applied-tags payload (list_for_image / get_image_with_tags).
|
|
||||||
const unconfirmedAuto = computed(() =>
|
|
||||||
AUTO_SOURCES.includes(props.tag.source) && !props.tag.confirmed
|
|
||||||
)
|
|
||||||
|
|
||||||
// #1206 Step 4: applied-tag grounding. `fcSuggestionHover` is provided by the
|
|
||||||
// image viewer / Explore host (a no-op elsewhere). Applied tags aren't scored
|
|
||||||
// live, so we fetch the winning region on demand and cache it per (image, tag).
|
|
||||||
const hover = inject('fcSuggestionHover', null)
|
|
||||||
const _groundingCache = new Map()
|
|
||||||
// Bumped on every enter/leave so a slow fetch that resolves after the pointer
|
|
||||||
// has moved on can't draw a stale overlay.
|
|
||||||
let hoverSeq = 0
|
|
||||||
|
|
||||||
async function onEnter () {
|
|
||||||
if (!hover || props.imageId == null) return
|
|
||||||
const seq = ++hoverSeq
|
|
||||||
const key = `${props.imageId}:${props.tag.id}`
|
|
||||||
let res = _groundingCache.get(key)
|
|
||||||
if (res === undefined) {
|
|
||||||
try {
|
|
||||||
res = await api.get(
|
|
||||||
`/api/images/${props.imageId}/tags/${props.tag.id}/grounding`
|
|
||||||
)
|
|
||||||
_groundingCache.set(key, res)
|
|
||||||
} catch {
|
|
||||||
return // best-effort — leave the overlay untouched on error
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (seq !== hoverSeq) return // pointer already left / moved to another chip
|
|
||||||
// No head → nothing to localize; don't draw an overlay at all. With a head,
|
|
||||||
// null grounding still draws the whole-image frame ("the global vector won").
|
|
||||||
if (res.has_head) hover.value = { g: res.grounding ?? null, tag: props.tag.name }
|
|
||||||
}
|
|
||||||
function onLeave () {
|
|
||||||
hoverSeq++
|
|
||||||
if (hover) hover.value = null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Show a character's fandom inline (truncated). Falls back to a bare arrow when
|
// Show a character's fandom inline (truncated). Falls back to a bare arrow when
|
||||||
// only fandom_id is known but the name wasn't resolved (older payloads).
|
// only fandom_id is known but the name wasn't resolved (older payloads).
|
||||||
@@ -136,30 +65,7 @@ function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
/* A chip must never exceed the rail — a long character+fandom chip used to
|
.fc-tag-chip { display: inline-flex; align-items: center; gap: 1px; }
|
||||||
overflow the right edge and clip its trailing control (operator-flagged
|
|
||||||
2026-07-06). The name is the elastic part: it ellipsis-truncates so the ✕ (or
|
|
||||||
the auto-tag yes/no pair) and fandom stay reachable. Full name is on the
|
|
||||||
chip's hover title. */
|
|
||||||
.fc-tag-chip { display: inline-flex; align-items: center; gap: 1px; max-width: 100%; min-width: 0; }
|
|
||||||
.fc-tag-chip__nav { max-width: 100%; min-width: 0; }
|
|
||||||
/* Tonal chips (esp. character = info) wash out against the dark rail — the fill
|
|
||||||
is intentionally faint. Give every tag chip a thin border in its OWN kind
|
|
||||||
colour (currentColor = the tonal chip's themed foreground) so the edge reads
|
|
||||||
clearly without touching the fill (operator-asked 2026-07-08). Theme-aware:
|
|
||||||
currentColor tracks the kind colour in either light or dark. */
|
|
||||||
.fc-tag-chip__nav { border: thin solid color-mix(in srgb, currentColor 55%, transparent); }
|
|
||||||
.fc-tag-chip__nav :deep(.v-chip__content) { min-width: 0; overflow: hidden; }
|
|
||||||
/* The bigger size=default chip widens Vuetify's negative start-margin on the
|
|
||||||
leading kind-icon, pulling it LEFT of the content box above — where that
|
|
||||||
overflow:hidden then clips the icon's left edge as a vertical slice
|
|
||||||
(operator-flagged 2026-07-08). Zero the pull so the icon sits inside the clip
|
|
||||||
box; the chip's own 12px padding keeps a comfortable left inset. */
|
|
||||||
.fc-tag-chip__nav :deep(.v-icon--start) { margin-inline-start: 0; }
|
|
||||||
.fc-tag-chip__name {
|
|
||||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
|
||||||
min-width: 0; flex: 0 1 auto;
|
|
||||||
}
|
|
||||||
/* The chip body navigates to the filtered gallery (#5); signal it's clickable.
|
/* The chip body navigates to the filtered gallery (#5); signal it's clickable.
|
||||||
The close ✕ (remove) and the sibling kebab stay as the explicit controls. */
|
The close ✕ (remove) and the sibling kebab stay as the explicit controls. */
|
||||||
.fc-tag-chip__nav { cursor: pointer; }
|
.fc-tag-chip__nav { cursor: pointer; }
|
||||||
@@ -171,27 +77,4 @@ function iconFor (k) { return KIND_ICONS[k] || 'mdi-tag' }
|
|||||||
.fc-tag-chip__kebab { opacity: 0.7; }
|
.fc-tag-chip__kebab { opacity: 0.7; }
|
||||||
.fc-tag-chip:hover .fc-tag-chip__kebab { opacity: 1; }
|
.fc-tag-chip:hover .fc-tag-chip__kebab { opacity: 1; }
|
||||||
.fc-tag-chip__fandom { opacity: 0.7; font-size: 0.85em; }
|
.fc-tag-chip__fandom { opacity: 0.7; font-size: 0.85em; }
|
||||||
/* Provisional auto-tag: a green ✓ / red ✗ pair in place of the ✕. The yes/no is
|
|
||||||
obvious enough on its own, so there's no "auto" label (operator-asked
|
|
||||||
2026-07-07). flex:0 0 auto keeps it visible while the name ellipsis-truncates. */
|
|
||||||
.fc-tag-chip__verdict {
|
|
||||||
flex: 0 0 auto; display: inline-flex; align-items: center; gap: 3px;
|
|
||||||
margin-left: 5px;
|
|
||||||
}
|
|
||||||
/* Solid-filled (white glyph on a full success/error circle) so accept/reject
|
|
||||||
stay well-defined even on a muted tonal chip — e.g. character (info) — instead
|
|
||||||
of a faint icon that only lit up on hover. Mirrors the Suggestions rail's
|
|
||||||
verdict buttons so the affordance reads identically (operator-asked 2026-07-08). */
|
|
||||||
.fc-tag-chip__yes, .fc-tag-chip__no {
|
|
||||||
display: inline-flex; align-items: center; justify-content: center;
|
|
||||||
width: 22px; height: 22px; padding: 0; border: none; border-radius: 50%;
|
|
||||||
color: #fff; cursor: pointer; opacity: 0.92;
|
|
||||||
transition: transform 0.1s, opacity 0.1s;
|
|
||||||
}
|
|
||||||
.fc-tag-chip__yes { background: rgb(var(--v-theme-success)); }
|
|
||||||
.fc-tag-chip__no { background: rgb(var(--v-theme-error)); }
|
|
||||||
.fc-tag-chip__yes:hover, .fc-tag-chip__no:hover { opacity: 1; transform: scale(1.1); }
|
|
||||||
.fc-tag-chip__yes:focus-visible, .fc-tag-chip__no:focus-visible {
|
|
||||||
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 2px;
|
|
||||||
}
|
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -4,9 +4,9 @@
|
|||||||
<div class="fc-tag-panel__chips">
|
<div class="fc-tag-panel__chips">
|
||||||
<TagChip
|
<TagChip
|
||||||
v-for="tag in host.current?.tags || []"
|
v-for="tag in host.current?.tags || []"
|
||||||
:key="tag.id" :tag="tag" :image-id="host.currentImageId"
|
:key="tag.id" :tag="tag"
|
||||||
@remove="onRemove" @rename="openRename" @set-fandom="openSetFandom"
|
@remove="onRemove" @rename="openRename" @set-fandom="openSetFandom"
|
||||||
@navigate="onNavigate" @confirm="onConfirm"
|
@navigate="onNavigate"
|
||||||
/>
|
/>
|
||||||
<span v-if="!host.current?.tags?.length" class="text-caption">No tags yet.</span>
|
<span v-if="!host.current?.tags?.length" class="text-caption">No tags yet.</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -17,6 +17,7 @@
|
|||||||
ref="tagInputRef"
|
ref="tagInputRef"
|
||||||
:applied-tags="host.current?.tags || []"
|
:applied-tags="host.current?.tags || []"
|
||||||
@pick-existing="onPickExisting" @pick-new="onPickNew"
|
@pick-existing="onPickExisting" @pick-new="onPickNew"
|
||||||
|
@accept-suggestion="onAcceptSuggestion"
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<v-alert v-if="errorMsg" type="error" variant="tonal" class="mt-2" closable>
|
<v-alert v-if="errorMsg" type="error" variant="tonal" class="mt-2" closable>
|
||||||
@@ -64,7 +65,6 @@
|
|||||||
<script setup>
|
<script setup>
|
||||||
import { ref } from 'vue'
|
import { ref } from 'vue'
|
||||||
import { useRouter } from 'vue-router'
|
import { useRouter } from 'vue-router'
|
||||||
import { useApi } from '../../composables/useApi.js'
|
|
||||||
import { useModalStore } from '../../stores/modal.js'
|
import { useModalStore } from '../../stores/modal.js'
|
||||||
import { useSuggestionsStore } from '../../stores/suggestions.js'
|
import { useSuggestionsStore } from '../../stores/suggestions.js'
|
||||||
import TagChip from './TagChip.vue'
|
import TagChip from './TagChip.vue'
|
||||||
@@ -83,7 +83,6 @@ const modalStore = useModalStore()
|
|||||||
const host = props.host || modalStore
|
const host = props.host || modalStore
|
||||||
const suggestions = useSuggestionsStore()
|
const suggestions = useSuggestionsStore()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
const api = useApi()
|
|
||||||
const errorMsg = ref(null)
|
const errorMsg = ref(null)
|
||||||
const tagInputRef = ref(null)
|
const tagInputRef = ref(null)
|
||||||
|
|
||||||
@@ -107,22 +106,6 @@ defineExpose({ focusTagInput })
|
|||||||
// Every tag mutation hands focus back to the input so the operator can keep
|
// Every tag mutation hands focus back to the input so the operator can keep
|
||||||
// typing the next tag without re-clicking — matches the accept-suggestion flow
|
// typing the next tag without re-clicking — matches the accept-suggestion flow
|
||||||
// (operator-asked 2026-06-26; the Explore workspace leans on this hard).
|
// (operator-asked 2026-06-26; the Explore workspace leans on this hard).
|
||||||
// Confirm/keep an auto-applied tag (milestone 139): records the affirmation so
|
|
||||||
// the tag becomes a training positive AND is shielded from the retraction sweep,
|
|
||||||
// then reloads so the chip drops its "auto" badge + Keep button.
|
|
||||||
async function onConfirm(tag) {
|
|
||||||
errorMsg.value = null
|
|
||||||
try {
|
|
||||||
await api.post(`/api/images/${host.currentImageId}/tags/${tag.id}/confirm`)
|
|
||||||
await host.reloadTags()
|
|
||||||
// Return the cursor to the tag input — its resting position in these views,
|
|
||||||
// matching every other chip action (remove/accept) so the keyboard flow
|
|
||||||
// never leaves the field (operator-asked 2026-07-07).
|
|
||||||
focusTagInput()
|
|
||||||
}
|
|
||||||
catch (e) { errorMsg.value = e.message }
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onRemove(tagId) {
|
async function onRemove(tagId) {
|
||||||
errorMsg.value = null
|
errorMsg.value = null
|
||||||
try {
|
try {
|
||||||
@@ -133,6 +116,7 @@ async function onRemove(tagId) {
|
|||||||
// until the next modal open (operator-asked 2026-07-03).
|
// until the next modal open (operator-asked 2026-07-03).
|
||||||
if (host.currentImageId != null) {
|
if (host.currentImageId != null) {
|
||||||
suggestions.load(host.currentImageId)
|
suggestions.load(host.currentImageId)
|
||||||
|
suggestions.loadAll(host.currentImageId)
|
||||||
}
|
}
|
||||||
focusTagInput()
|
focusTagInput()
|
||||||
}
|
}
|
||||||
@@ -176,6 +160,18 @@ async function onPickNew(payload) {
|
|||||||
}
|
}
|
||||||
catch (e) { errorMsg.value = e.message }
|
catch (e) { errorMsg.value = e.message }
|
||||||
}
|
}
|
||||||
|
// A suggestion picked from the autocomplete dropdown runs the SAME path as the
|
||||||
|
// Suggestions panel's Accept: the store creates the tag if it's raw, records the
|
||||||
|
// acceptance, and drops it from the panel; then we refresh the chip rail.
|
||||||
|
async function onAcceptSuggestion(s) {
|
||||||
|
errorMsg.value = null
|
||||||
|
try {
|
||||||
|
await suggestions.accept(s)
|
||||||
|
await host.reloadTags()
|
||||||
|
focusTagInput()
|
||||||
|
} catch (e) { errorMsg.value = e.message }
|
||||||
|
}
|
||||||
|
|
||||||
const renameDialog = ref(false)
|
const renameDialog = ref(false)
|
||||||
const renameTarget = ref(null)
|
const renameTarget = ref(null)
|
||||||
|
|
||||||
|
|||||||
@@ -58,31 +58,16 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="fc-post-card__text">
|
<div class="fc-post-card__text">
|
||||||
<h3 v-if="displayTitle" class="fc-post-card__title">{{ displayTitle }}</h3>
|
<h3 v-if="plainTitle" class="fc-post-card__title">{{ plainTitle }}</h3>
|
||||||
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
||||||
Post {{ post.external_post_id }}
|
Post {{ post.external_post_id }}
|
||||||
</h3>
|
</h3>
|
||||||
<!-- Translation toggle (#143): English shown by default; reveal the
|
|
||||||
original, labelled with its detected language. -->
|
|
||||||
<button
|
|
||||||
v-if="hasTranslation" type="button" class="fc-post-card__lang"
|
|
||||||
:title="showOriginal ? 'Show the English translation' : `Show the original${sourceLangLabel} text`"
|
|
||||||
@click.stop="showOriginal = !showOriginal"
|
|
||||||
>{{ showOriginal ? 'Show translation' : `Show original${sourceLangLabel}` }}</button>
|
|
||||||
|
|
||||||
<!-- Per-post translation override (#155): force a skipped translation on,
|
|
||||||
or keep the original for a confidently mis-flagged title. Only where
|
|
||||||
there's text to translate. -->
|
|
||||||
<PostTranslationControl
|
|
||||||
v-if="post.post_title || post.description_plain"
|
|
||||||
:post="post"
|
|
||||||
/>
|
|
||||||
|
|
||||||
<!-- Faithful (semantic) body render once expanded: backend-sanitized
|
<!-- Faithful (semantic) body render once expanded: backend-sanitized
|
||||||
HTML (headings, lists, links, inline images). Collapsed and the
|
HTML (headings, lists, links, inline images). Collapsed and the
|
||||||
no-detail fallback stay plain text. -->
|
no-detail fallback stay plain text. -->
|
||||||
<div
|
<div
|
||||||
v-if="hasDescription && showHtmlBody"
|
v-if="hasDescription && descExpanded && descHtml"
|
||||||
class="fc-post-card__desc fc-post-card__desc--html"
|
class="fc-post-card__desc fc-post-card__desc--html"
|
||||||
v-html="descHtml"
|
v-html="descHtml"
|
||||||
/>
|
/>
|
||||||
@@ -90,7 +75,7 @@
|
|||||||
v-else-if="hasDescription" ref="descEl"
|
v-else-if="hasDescription" ref="descEl"
|
||||||
class="fc-post-card__desc"
|
class="fc-post-card__desc"
|
||||||
:class="{ 'fc-post-card__desc--clamped': !descExpanded }"
|
:class="{ 'fc-post-card__desc--clamped': !descExpanded }"
|
||||||
>{{ descTextDisplay }}</p>
|
>{{ descText }}</p>
|
||||||
<p v-else class="fc-post-card__desc fc-post-card__desc--missing">
|
<p v-else class="fc-post-card__desc fc-post-card__desc--missing">
|
||||||
(no description)
|
(no description)
|
||||||
</p>
|
</p>
|
||||||
@@ -144,7 +129,6 @@ import { useModalStore } from '../../stores/modal.js'
|
|||||||
import { usePostsStore } from '../../stores/posts.js'
|
import { usePostsStore } from '../../stores/posts.js'
|
||||||
import { toPlainText } from '../../utils/htmlSanitize.js'
|
import { toPlainText } from '../../utils/htmlSanitize.js'
|
||||||
import PostSeriesMenu from './PostSeriesMenu.vue'
|
import PostSeriesMenu from './PostSeriesMenu.vue'
|
||||||
import PostTranslationControl from './PostTranslationControl.vue'
|
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
post: { type: Object, required: true },
|
post: { type: Object, required: true },
|
||||||
@@ -216,13 +200,13 @@ async function fullImageIds () {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function openModal (imageId) {
|
async function openModal (imageId) {
|
||||||
modal.open(imageId, { playlistIds: await fullImageIds() })
|
modal.open(imageId, { postImageIds: await fullImageIds() })
|
||||||
}
|
}
|
||||||
|
|
||||||
async function openModalAtMore () {
|
async function openModalAtMore () {
|
||||||
const ids = await fullImageIds()
|
const ids = await fullImageIds()
|
||||||
const first = ids[visibleCount.value] ?? ids[0]
|
const first = ids[visibleCount.value] ?? ids[0]
|
||||||
if (first != null) modal.open(first, { playlistIds: ids })
|
if (first != null) modal.open(first, { postImageIds: ids })
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- description "Show more" (text-only, in place, only when truncated) ----
|
// --- description "Show more" (text-only, in place, only when truncated) ----
|
||||||
@@ -247,32 +231,6 @@ const canExpand = computed(
|
|||||||
() => props.post.description_truncated === true || cssOverflow.value,
|
() => props.post.description_truncated === true || cssOverflow.value,
|
||||||
)
|
)
|
||||||
|
|
||||||
// --- translation-forward (#143): the English is shown by default when a
|
|
||||||
// translation exists; a per-card toggle reveals the original. Translations are
|
|
||||||
// plain text, so showTranslated also decides HTML-vs-plain body rendering.
|
|
||||||
const showOriginal = ref(false)
|
|
||||||
const hasTranslation = computed(
|
|
||||||
() => !!(props.post.post_title_translated || props.post.description_translated),
|
|
||||||
)
|
|
||||||
const showTranslated = computed(() => hasTranslation.value && !showOriginal.value)
|
|
||||||
const sourceLangLabel = computed(() =>
|
|
||||||
props.post.translated_source_lang ? ` (${props.post.translated_source_lang})` : '',
|
|
||||||
)
|
|
||||||
const displayTitle = computed(() =>
|
|
||||||
showTranslated.value && props.post.post_title_translated
|
|
||||||
? toPlainText(props.post.post_title_translated)
|
|
||||||
: plainTitle.value,
|
|
||||||
)
|
|
||||||
const descTextDisplay = computed(() => {
|
|
||||||
if (!showTranslated.value) return descText.value
|
|
||||||
return descExpanded.value
|
|
||||||
? (detail.value?.description_translated_full || props.post.description_translated)
|
|
||||||
: props.post.description_translated
|
|
||||||
})
|
|
||||||
const showHtmlBody = computed(
|
|
||||||
() => !showTranslated.value && descExpanded.value && !!descHtml.value,
|
|
||||||
)
|
|
||||||
|
|
||||||
function measureOverflow () {
|
function measureOverflow () {
|
||||||
const el = descEl.value
|
const el = descEl.value
|
||||||
cssOverflow.value = !!el && el.scrollHeight > el.clientHeight + 1
|
cssOverflow.value = !!el && el.scrollHeight > el.clientHeight + 1
|
||||||
@@ -509,19 +467,6 @@ function formatBytes (n) {
|
|||||||
font-size: 0.85rem; font-weight: 600;
|
font-size: 0.85rem; font-weight: 600;
|
||||||
}
|
}
|
||||||
.fc-post-card__more:hover { text-decoration: underline; }
|
.fc-post-card__more:hover { text-decoration: underline; }
|
||||||
/* Translation toggle (#143) — quiet secondary affordance under the title. */
|
|
||||||
.fc-post-card__lang {
|
|
||||||
display: block; margin: 2px 0 6px; padding: 0;
|
|
||||||
background: none; border: 0; cursor: pointer;
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
font-size: 0.75rem; font-weight: 600;
|
|
||||||
}
|
|
||||||
.fc-post-card__lang:hover {
|
|
||||||
text-decoration: underline; color: rgb(var(--v-theme-accent));
|
|
||||||
}
|
|
||||||
.fc-post-card__lang:focus-visible {
|
|
||||||
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 1px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.fc-post-card__atts {
|
.fc-post-card__atts {
|
||||||
display: flex; flex-wrap: wrap; gap: 8px;
|
display: flex; flex-wrap: wrap; gap: 8px;
|
||||||
|
|||||||
@@ -1,102 +0,0 @@
|
|||||||
<template>
|
|
||||||
<!-- Sticky per-post translation override (#155). Quiet inline control under the
|
|
||||||
post title: force a skipped legit-foreign title on, or knock a wrongly
|
|
||||||
mis-translated one back to the original. The choice sticks through a
|
|
||||||
Re-translate-all. -->
|
|
||||||
<div class="fc-post-tx">
|
|
||||||
<span class="fc-post-tx__label">Translation:</span>
|
|
||||||
<v-menu :disabled="busy">
|
|
||||||
<template #activator="{ props: menuProps }">
|
|
||||||
<button
|
|
||||||
type="button" class="fc-post-tx__btn" v-bind="menuProps" :disabled="busy"
|
|
||||||
:aria-label="`Translation handling: ${currentLabel}`"
|
|
||||||
>
|
|
||||||
{{ currentLabel }}
|
|
||||||
<v-icon size="14">mdi-menu-down</v-icon>
|
|
||||||
</button>
|
|
||||||
</template>
|
|
||||||
<v-list density="compact" min-width="240" class="fc-post-tx__list">
|
|
||||||
<v-list-item
|
|
||||||
v-for="opt in OPTIONS" :key="opt.value"
|
|
||||||
:active="opt.value === current" @click="choose(opt.value)"
|
|
||||||
>
|
|
||||||
<template #prepend>
|
|
||||||
<v-icon size="small">{{ opt.icon }}</v-icon>
|
|
||||||
</template>
|
|
||||||
<v-list-item-title>{{ opt.label }}</v-list-item-title>
|
|
||||||
<v-list-item-subtitle>{{ opt.help }}</v-list-item-subtitle>
|
|
||||||
</v-list-item>
|
|
||||||
</v-list>
|
|
||||||
</v-menu>
|
|
||||||
<v-progress-circular v-if="busy" indeterminate size="13" width="2" />
|
|
||||||
</div>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script setup>
|
|
||||||
import { computed, ref } from 'vue'
|
|
||||||
|
|
||||||
import { usePostsStore } from '../../stores/posts.js'
|
|
||||||
import { toast } from '../../utils/toast.js'
|
|
||||||
|
|
||||||
const props = defineProps({ post: { type: Object, required: true } })
|
|
||||||
const posts = usePostsStore()
|
|
||||||
const busy = ref(false)
|
|
||||||
|
|
||||||
const OPTIONS = [
|
|
||||||
{ value: 'auto', label: 'Auto', icon: 'mdi-cog-outline',
|
|
||||||
help: 'Translate only when confident enough' },
|
|
||||||
{ value: 'force', label: 'Force translate', icon: 'mdi-translate',
|
|
||||||
help: 'Always translate, even at low confidence' },
|
|
||||||
{ value: 'original', label: 'Keep original', icon: 'mdi-translate-off',
|
|
||||||
help: 'Never translate — keep the original text' },
|
|
||||||
]
|
|
||||||
|
|
||||||
const current = computed(() => props.post.translation_override || 'auto')
|
|
||||||
const currentLabel = computed(
|
|
||||||
() => (OPTIONS.find((o) => o.value === current.value) || OPTIONS[0]).label,
|
|
||||||
)
|
|
||||||
|
|
||||||
async function choose (value) {
|
|
||||||
if (value === current.value || busy.value) return
|
|
||||||
busy.value = true
|
|
||||||
try {
|
|
||||||
const res = await posts.applyTranslationOverride(props.post.id, value)
|
|
||||||
if (res.applied === 'queued') {
|
|
||||||
toast({
|
|
||||||
text: 'Saved — this post will translate on the next sweep (service offline).',
|
|
||||||
type: 'info',
|
|
||||||
})
|
|
||||||
}
|
|
||||||
} catch (e) {
|
|
||||||
toast({ text: `Couldn't update translation: ${e.message}`, type: 'error' })
|
|
||||||
} finally {
|
|
||||||
busy.value = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<style scoped>
|
|
||||||
.fc-post-tx {
|
|
||||||
display: inline-flex; align-items: center; gap: 6px;
|
|
||||||
margin: 2px 0 6px;
|
|
||||||
}
|
|
||||||
.fc-post-tx__label {
|
|
||||||
font-size: 0.72rem;
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
}
|
|
||||||
.fc-post-tx__btn {
|
|
||||||
display: inline-flex; align-items: center; gap: 1px;
|
|
||||||
padding: 0; background: none; border: 0; cursor: pointer;
|
|
||||||
font-size: 0.72rem; font-weight: 700;
|
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
|
||||||
}
|
|
||||||
.fc-post-tx__btn:hover { color: rgb(var(--v-theme-accent)); }
|
|
||||||
.fc-post-tx__btn:focus-visible {
|
|
||||||
outline: 2px solid rgb(var(--v-theme-accent)); outline-offset: 1px;
|
|
||||||
border-radius: 2px;
|
|
||||||
}
|
|
||||||
.fc-post-tx__btn:disabled { cursor: default; opacity: 0.6; }
|
|
||||||
.fc-post-tx__list :deep(.v-list-item-subtitle) {
|
|
||||||
font-size: 0.7rem;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
@@ -1,137 +0,0 @@
|
|||||||
<template>
|
|
||||||
<MaintenanceTile
|
|
||||||
icon="mdi-crop"
|
|
||||||
title="Crop proposers (where to crop)"
|
|
||||||
blurb="Detectors that decide WHERE to crop each image before embedding — general figures, anime/furry anatomy, and comic panels — so a tag can ground to a region, not just the whole image."
|
|
||||||
:open="false"
|
|
||||||
>
|
|
||||||
<p class="fc-muted text-body-2 mb-4">
|
|
||||||
Each proposer finds regions that the crop → SigLIP bag scores alongside the
|
|
||||||
whole image; the best-scoring region is what grounds a tag. Weights accept
|
|
||||||
an <strong>ultralytics name</strong> (<code>yolo11n.pt</code>), a
|
|
||||||
<strong>URL</strong>, or <strong>hf_repo::file</strong>. Edits reach the GPU
|
|
||||||
agent on its next lease — no restart. After turning a proposer on, re-process
|
|
||||||
the library so existing images gain its regions.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<div v-for="p in proposers" :key="p.key" class="fc-proposer">
|
|
||||||
<SettingToggleRow
|
|
||||||
v-model="p.on" :loading="busy" :icon="p.icon"
|
|
||||||
:icon-color="p.on ? 'accent' : null" :label="p.label"
|
|
||||||
@change="v => saveToggle(p, v)"
|
|
||||||
/>
|
|
||||||
<p class="fc-muted text-body-2 mb-2">{{ p.help }}</p>
|
|
||||||
<div class="d-flex flex-wrap mb-4" style="gap: 12px;">
|
|
||||||
<v-text-field
|
|
||||||
v-model="p.weights" label="Weights" density="compact" hide-details
|
|
||||||
style="min-width: 300px; flex: 1;" :disabled="busy || !p.on"
|
|
||||||
placeholder="name | URL | hf_repo::file"
|
|
||||||
@change="saveField({ [`detector_${p.key}_weights`]: p.weights })"
|
|
||||||
/>
|
|
||||||
<SettingNumberField
|
|
||||||
v-model="p.conf" label="Confidence" :min="0" :max="1" :step="0.05"
|
|
||||||
max-width="140px" :disabled="busy || !p.on"
|
|
||||||
@change="saveField({ [`detector_${p.key}_conf`]: Number(p.conf) })"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="fc-caps">
|
|
||||||
<div class="fc-section-h mb-1">Caps</div>
|
|
||||||
<p class="fc-muted text-body-2 mb-3">
|
|
||||||
Bound how many crops each image can produce — protects GPU time and
|
|
||||||
storage. Dedupe IoU drops near-duplicate crops before embedding.
|
|
||||||
</p>
|
|
||||||
<div class="d-flex flex-wrap" style="gap: 12px;">
|
|
||||||
<SettingNumberField
|
|
||||||
v-for="c in caps" :key="c.key"
|
|
||||||
v-model="c.val" :label="c.label"
|
|
||||||
:min="c.min" :max="c.max" :step="c.step || 1"
|
|
||||||
max-width="165px" :disabled="busy"
|
|
||||||
@change="saveField({ [c.key]: Number(c.val) })"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</MaintenanceTile>
|
|
||||||
</template>
|
|
||||||
|
|
||||||
<script setup>
|
|
||||||
import { onMounted, ref } from 'vue'
|
|
||||||
|
|
||||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
|
||||||
import SettingNumberField from '../common/SettingNumberField.vue'
|
|
||||||
import SettingToggleRow from '../common/SettingToggleRow.vue'
|
|
||||||
import { useSettingSave } from '../../composables/useSettingSave.js'
|
|
||||||
import { useMLStore } from '../../stores/ml.js'
|
|
||||||
|
|
||||||
const mlSettings = useMLStore()
|
|
||||||
const { busy, save } = useSettingSave(mlSettings.patchSettings)
|
|
||||||
const proposers = ref([])
|
|
||||||
const caps = ref([])
|
|
||||||
|
|
||||||
// Static copy per proposer; the mutable state (on/weights/conf) is seeded from
|
|
||||||
// settings on mount so the card reflects the live DB config (rule 26 — it works
|
|
||||||
// untouched).
|
|
||||||
const DEFS = [
|
|
||||||
{
|
|
||||||
key: 'person', label: 'Figures (person)', icon: 'mdi-account-outline',
|
|
||||||
help: 'General figure detector for the Western/realistic art the anime ' +
|
|
||||||
'detector misses → character identity + a whole-figure crop.',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: 'anatomy', label: 'Anatomy (booru)', icon: 'mdi-dog-side',
|
|
||||||
help: 'Anime / furry / NSFW torso components (heads, busts, hips, …) → ' +
|
|
||||||
'localized concept crops.',
|
|
||||||
},
|
|
||||||
{
|
|
||||||
key: 'panel', label: 'Comic panels', icon: 'mdi-view-grid-outline',
|
|
||||||
help: 'Splits a comic page into its panels → one concept crop each.',
|
|
||||||
},
|
|
||||||
]
|
|
||||||
const CAP_DEFS = [
|
|
||||||
{ key: 'detector_max_figures', label: 'Max figures', min: 1 },
|
|
||||||
{ key: 'detector_max_components', label: 'Max components', min: 1 },
|
|
||||||
{ key: 'detector_max_panels', label: 'Max panels', min: 1 },
|
|
||||||
{ key: 'detector_max_regions', label: 'Max regions / job', min: 1 },
|
|
||||||
{ key: 'detector_dedupe_iou', label: 'Dedupe IoU', min: 0, max: 1, step: 0.05 },
|
|
||||||
]
|
|
||||||
|
|
||||||
onMounted(async () => {
|
|
||||||
try { await mlSettings.loadSettings() } catch { /* non-fatal — defaults show */ }
|
|
||||||
const s = mlSettings.settings || {}
|
|
||||||
proposers.value = DEFS.map(d => ({
|
|
||||||
...d,
|
|
||||||
on: !!s[`detector_${d.key}_enabled`],
|
|
||||||
weights: s[`detector_${d.key}_weights`] ?? '',
|
|
||||||
conf: s[`detector_${d.key}_conf`] ?? 0.3,
|
|
||||||
}))
|
|
||||||
caps.value = CAP_DEFS.map(c => ({ ...c, val: s[c.key] ?? 0 }))
|
|
||||||
})
|
|
||||||
|
|
||||||
// Field @change → persist with a "Saved" confirmation. SettingNumberField has
|
|
||||||
// already clamped numeric values to their [min,max] before this fires.
|
|
||||||
function saveField(patch) {
|
|
||||||
save(patch, { successMessage: 'Saved' })
|
|
||||||
}
|
|
||||||
|
|
||||||
async function saveToggle(p, v) {
|
|
||||||
// Revert the switch on failure so it never lies about the persisted state.
|
|
||||||
const ok = await save({ [`detector_${p.key}_enabled`]: !!v }, { successMessage: 'Saved' })
|
|
||||||
if (!ok) p.on = !v
|
|
||||||
}
|
|
||||||
</script>
|
|
||||||
|
|
||||||
<style scoped>
|
|
||||||
.fc-proposer {
|
|
||||||
border-top: 1px solid rgb(var(--v-theme-surface-light)); padding-top: 14px;
|
|
||||||
}
|
|
||||||
.fc-proposer:first-of-type { border-top: none; padding-top: 0; }
|
|
||||||
.fc-caps {
|
|
||||||
border-top: 1px solid rgb(var(--v-theme-surface-light)); padding-top: 16px;
|
|
||||||
}
|
|
||||||
code {
|
|
||||||
font-family: 'JetBrains Mono', monospace; font-size: 0.85em;
|
|
||||||
background: rgb(var(--v-theme-surface-light)); padding: 1px 5px;
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
@@ -112,6 +112,7 @@ async function onCommit() {
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
.fc-code {
|
.fc-code {
|
||||||
background: rgb(var(--v-theme-surface-light));
|
background: rgb(var(--v-theme-surface-light));
|
||||||
border-radius: 4px; padding: 2px 8px;
|
border-radius: 4px; padding: 2px 8px;
|
||||||
|
|||||||
@@ -42,7 +42,7 @@
|
|||||||
</tr>
|
</tr>
|
||||||
</tbody>
|
</tbody>
|
||||||
</v-table>
|
</v-table>
|
||||||
<p v-else class="text-caption mt-3 fc-muted">
|
<p v-else class="text-caption mt-3" style="opacity: 0.6;">
|
||||||
No table statistics yet.
|
No table statistics yet.
|
||||||
</p>
|
</p>
|
||||||
</MaintenanceTile>
|
</MaintenanceTile>
|
||||||
|
|||||||
@@ -72,5 +72,6 @@ onUnmounted(() => { if (pollId) clearInterval(pollId) })
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
.fc-bad { color: rgb(var(--v-theme-error)); }
|
.fc-bad { color: rgb(var(--v-theme-error)); }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -95,6 +95,7 @@ onUnmounted(() => { if (pollId) clearInterval(pollId) })
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
.fc-cells { display: flex; gap: 28px; }
|
.fc-cells { display: flex; gap: 28px; }
|
||||||
.fc-cell__n {
|
.fc-cell__n {
|
||||||
font-size: 20px; font-weight: 700; line-height: 1.1;
|
font-size: 20px; font-weight: 700; line-height: 1.1;
|
||||||
@@ -104,5 +105,6 @@ onUnmounted(() => { if (pollId) clearInterval(pollId) })
|
|||||||
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
}
|
}
|
||||||
|
.fc-good { color: rgb(var(--v-theme-success)); }
|
||||||
.fc-bad { color: rgb(var(--v-theme-error)); }
|
.fc-bad { color: rgb(var(--v-theme-error)); }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -367,6 +367,11 @@ async function onReprocess() {
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-section-h {
|
||||||
|
font-size: 13px; font-weight: 700; letter-spacing: 0.03em;
|
||||||
|
text-transform: uppercase; color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
.fc-token {
|
.fc-token {
|
||||||
display: flex; align-items: center; gap: 4px;
|
display: flex; align-items: center; gap: 4px;
|
||||||
background: rgb(var(--v-theme-surface-light)); border-radius: 6px;
|
background: rgb(var(--v-theme-surface-light)); border-radius: 6px;
|
||||||
@@ -385,4 +390,6 @@ async function onReprocess() {
|
|||||||
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
}
|
}
|
||||||
|
.fc-good { color: rgb(var(--v-theme-success)); }
|
||||||
|
.fc-weak { color: rgb(var(--v-theme-error)); }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -155,6 +155,11 @@ async function onRecover(it) {
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
.fc-section-h {
|
||||||
|
font-size: 13px; font-weight: 700; letter-spacing: 0.03em;
|
||||||
|
text-transform: uppercase; color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
.fc-queue { display: flex; gap: 24px; }
|
.fc-queue { display: flex; gap: 24px; }
|
||||||
.fc-q__n {
|
.fc-q__n {
|
||||||
font-size: 20px; font-weight: 700; line-height: 1.1;
|
font-size: 20px; font-weight: 700; line-height: 1.1;
|
||||||
@@ -164,6 +169,8 @@ async function onRecover(it) {
|
|||||||
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em;
|
||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
}
|
}
|
||||||
|
.fc-good { color: rgb(var(--v-theme-success)); }
|
||||||
|
.fc-weak { color: rgb(var(--v-theme-error)); }
|
||||||
.fc-defect {
|
.fc-defect {
|
||||||
display: flex; align-items: center; gap: 12px;
|
display: flex; align-items: center; gap: 12px;
|
||||||
background: rgb(var(--v-theme-surface-light)); border-radius: 8px;
|
background: rgb(var(--v-theme-surface-light)); border-radius: 8px;
|
||||||
|
|||||||
@@ -95,10 +95,14 @@
|
|||||||
|
|
||||||
<!-- Earned auto-apply -->
|
<!-- Earned auto-apply -->
|
||||||
<div class="fc-auto mt-6">
|
<div class="fc-auto mt-6">
|
||||||
<SettingToggleRow
|
<div class="d-flex align-center mb-1" style="gap: 10px;">
|
||||||
v-model="autoEnabled" :loading="settingBusy"
|
<v-icon size="18" color="accent">mdi-lightning-bolt</v-icon>
|
||||||
icon="mdi-lightning-bolt" label="Auto-apply" @change="onToggleAuto"
|
<span class="fc-section-h">Auto-apply</span>
|
||||||
/>
|
<v-switch
|
||||||
|
v-model="autoEnabled" :loading="settingBusy" hide-details density="compact"
|
||||||
|
color="success" class="ml-auto" @update:model-value="onToggleAuto"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
<p class="fc-muted text-body-2 mb-3">
|
<p class="fc-muted text-body-2 mb-3">
|
||||||
Graduated heads (⚡, with ≥ {{ autoMinPosInput }} examples) apply their tag
|
Graduated heads (⚡, with ≥ {{ autoMinPosInput }} examples) apply their tag
|
||||||
on their own where they clear {{ Math.round((autoPrecisionInput || 0) * 100) }}%
|
on their own where they clear {{ Math.round((autoPrecisionInput || 0) * 100) }}%
|
||||||
@@ -107,14 +111,17 @@
|
|||||||
</p>
|
</p>
|
||||||
|
|
||||||
<div class="d-flex mb-3" style="gap: 12px;">
|
<div class="d-flex mb-3" style="gap: 12px;">
|
||||||
<SettingNumberField
|
<v-text-field
|
||||||
v-model="autoPrecisionInput" label="Precision target"
|
v-model.number="autoPrecisionInput" label="Precision target"
|
||||||
:min="0.5" :max="0.999" :step="0.01" :disabled="settingBusy"
|
type="number" min="0.5" max="0.999" step="0.01" density="compact"
|
||||||
|
hide-details style="max-width: 200px;" :disabled="settingBusy"
|
||||||
@change="onSaveSettings"
|
@change="onSaveSettings"
|
||||||
/>
|
/>
|
||||||
<SettingNumberField
|
<v-text-field
|
||||||
v-model="autoMinPosInput" label="Min examples to fire"
|
v-model.number="autoMinPosInput" label="Min examples to fire"
|
||||||
:min="1" :disabled="settingBusy" @change="onSaveSettings"
|
type="number" min="1" density="compact" hide-details
|
||||||
|
style="max-width: 200px;" :disabled="settingBusy"
|
||||||
|
@change="onSaveSettings"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -152,67 +159,6 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Presentation chrome auto-hide (#141) -->
|
|
||||||
<div class="fc-auto mt-6">
|
|
||||||
<SettingToggleRow
|
|
||||||
v-model="presentationEnabled" :loading="settingBusy"
|
|
||||||
icon="mdi-image-off-outline" label="Hide presentation chrome"
|
|
||||||
@change="onTogglePresentation"
|
|
||||||
/>
|
|
||||||
<p class="fc-muted text-body-2 mb-3">
|
|
||||||
Auto-hide <code>banner</code> chrome from the gallery once a head has
|
|
||||||
learned it (≥ {{ minPositives }} examples) and clears
|
|
||||||
{{ Math.round((presentationThresholdInput || 0) * 100) }}% confidence.
|
|
||||||
(<code>wip</code> and <code>editor screenshot</code> are handled by the
|
|
||||||
process auto-tagger below.) If a hidden image also looks like real content
|
|
||||||
(≥ {{ Math.round((presentationConflictInput || 0) * 100) }}% on a content
|
|
||||||
tag), it's flagged for review instead of buried. Every auto-hide is reversible.
|
|
||||||
</p>
|
|
||||||
<div class="d-flex mb-3" style="gap: 12px;">
|
|
||||||
<SettingNumberField
|
|
||||||
v-model="presentationThresholdInput" label="Hide confidence"
|
|
||||||
:min="0.5" :max="0.999" :step="0.01" :disabled="settingBusy"
|
|
||||||
@change="onSavePresentation"
|
|
||||||
/>
|
|
||||||
<SettingNumberField
|
|
||||||
v-model="presentationConflictInput" label="Flag if content ≥"
|
|
||||||
:min="0" :max="1" :step="0.05" :disabled="settingBusy"
|
|
||||||
@change="onSavePresentation"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- Process auto-tagging (#1464): wip / editor screenshot -->
|
|
||||||
<div class="fc-auto mt-6">
|
|
||||||
<SettingToggleRow
|
|
||||||
v-model="processEnabled" :loading="settingBusy"
|
|
||||||
icon="mdi-progress-wrench" label="Auto-tag work-in-progress"
|
|
||||||
@change="onToggleProcess"
|
|
||||||
/>
|
|
||||||
<p class="fc-muted text-body-2 mb-3">
|
|
||||||
Auto-tag <code>wip</code> and <code>editor screenshot</code> process art
|
|
||||||
once a head has learned them (≥ {{ minPositives }} examples) and clears
|
|
||||||
{{ Math.round((processThresholdInput || 0) * 100) }}% confidence. These stay
|
|
||||||
<strong>visible</strong> in the gallery — the tag just keeps them out of
|
|
||||||
training and the Explore rabbit-hole. Off by default. If a tagged image also
|
|
||||||
looks like real content (≥ {{ Math.round((processConflictInput || 0) * 100) }}%
|
|
||||||
on a content tag), it's flagged for review. Learns only from your titles +
|
|
||||||
manual tags, never its own guesses — so it can't run away. Every tag reversible.
|
|
||||||
</p>
|
|
||||||
<div class="d-flex mb-3" style="gap: 12px;">
|
|
||||||
<SettingNumberField
|
|
||||||
v-model="processThresholdInput" label="Tag confidence"
|
|
||||||
:min="0.5" :max="0.999" :step="0.01" :disabled="settingBusy"
|
|
||||||
@change="onSaveProcess"
|
|
||||||
/>
|
|
||||||
<SettingNumberField
|
|
||||||
v-model="processConflictInput" label="Flag if content ≥"
|
|
||||||
:min="0" :max="1" :step="0.05" :disabled="settingBusy"
|
|
||||||
@change="onSaveProcess"
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<!-- Performance / tuning -->
|
<!-- Performance / tuning -->
|
||||||
<div v-if="metricsConcepts.length" class="mt-5">
|
<div v-if="metricsConcepts.length" class="mt-5">
|
||||||
<div class="fc-section-h mb-1">How auto-apply is landing</div>
|
<div class="fc-section-h mb-1">How auto-apply is landing</div>
|
||||||
@@ -237,7 +183,7 @@
|
|||||||
<td class="fc-r fc-mono">{{ c.n_auto_applied }}</td>
|
<td class="fc-r fc-mono">{{ c.n_auto_applied }}</td>
|
||||||
<td class="fc-r fc-mono">{{ c.n_misfires }}</td>
|
<td class="fc-r fc-mono">{{ c.n_misfires }}</td>
|
||||||
<td class="fc-r fc-mono" :class="rateClass(c.misfire_rate)">
|
<td class="fc-r fc-mono" :class="rateClass(c.misfire_rate)">
|
||||||
{{ pct(c.misfire_rate) }}
|
{{ ratePct(c.misfire_rate) }}
|
||||||
</td>
|
</td>
|
||||||
<td class="fc-r fc-mono">{{ c.n_underfires }}</td>
|
<td class="fc-r fc-mono">{{ c.n_underfires }}</td>
|
||||||
</tr>
|
</tr>
|
||||||
@@ -253,9 +199,6 @@ import { toast } from '../../utils/toast.js'
|
|||||||
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
import { computed, onMounted, onUnmounted, ref } from 'vue'
|
||||||
|
|
||||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
import SettingNumberField from '../common/SettingNumberField.vue'
|
|
||||||
import SettingToggleRow from '../common/SettingToggleRow.vue'
|
|
||||||
import { useSettingSave } from '../../composables/useSettingSave.js'
|
|
||||||
import { useHeadsStore } from '../../stores/heads.js'
|
import { useHeadsStore } from '../../stores/heads.js'
|
||||||
import { useMLStore } from '../../stores/ml.js'
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
|
|
||||||
@@ -269,22 +212,12 @@ let pollTimer = null
|
|||||||
const autoEnabled = ref(false)
|
const autoEnabled = ref(false)
|
||||||
const autoPrecisionInput = ref(0.97)
|
const autoPrecisionInput = ref(0.97)
|
||||||
const autoMinPosInput = ref(30)
|
const autoMinPosInput = ref(30)
|
||||||
// Shared settings-save flow (busy + toast + revert); `settingBusy` gates the
|
const settingBusy = ref(false)
|
||||||
// toggles/fields, `save` returns ok/false for the optimistic-switch revert.
|
|
||||||
const { busy: settingBusy, save } = useSettingSave(mlSettings.patchSettings)
|
|
||||||
const autoBusy = ref(false)
|
const autoBusy = ref(false)
|
||||||
const autoStatus = ref(null)
|
const autoStatus = ref(null)
|
||||||
const metricsData = ref(null)
|
const metricsData = ref(null)
|
||||||
let autoTimer = null
|
let autoTimer = null
|
||||||
|
|
||||||
// --- Presentation chrome auto-hide state (#141) ---
|
|
||||||
const presentationEnabled = ref(true)
|
|
||||||
const presentationThresholdInput = ref(0.90)
|
|
||||||
const presentationConflictInput = ref(0.50)
|
|
||||||
const processEnabled = ref(false)
|
|
||||||
const processThresholdInput = ref(0.90)
|
|
||||||
const processConflictInput = ref(0.50)
|
|
||||||
|
|
||||||
const autoRunning = computed(() => autoStatus.value?.running_id != null)
|
const autoRunning = computed(() => autoStatus.value?.running_id != null)
|
||||||
const lastSweep = computed(() =>
|
const lastSweep = computed(() =>
|
||||||
(autoStatus.value?.runs || []).find(r => r.status !== 'running') || null)
|
(autoStatus.value?.runs || []).find(r => r.status !== 'running') || null)
|
||||||
@@ -315,12 +248,6 @@ onMounted(async () => {
|
|||||||
autoEnabled.value = !!s.head_auto_apply_enabled
|
autoEnabled.value = !!s.head_auto_apply_enabled
|
||||||
autoPrecisionInput.value = s.head_auto_apply_precision ?? 0.97
|
autoPrecisionInput.value = s.head_auto_apply_precision ?? 0.97
|
||||||
autoMinPosInput.value = s.head_auto_apply_min_positives ?? 30
|
autoMinPosInput.value = s.head_auto_apply_min_positives ?? 30
|
||||||
presentationEnabled.value = s.presentation_auto_apply_enabled ?? true
|
|
||||||
presentationThresholdInput.value = s.presentation_auto_apply_threshold ?? 0.90
|
|
||||||
presentationConflictInput.value = s.presentation_conflict_threshold ?? 0.50
|
|
||||||
processEnabled.value = s.process_auto_apply_enabled ?? false
|
|
||||||
processThresholdInput.value = s.process_auto_apply_threshold ?? 0.90
|
|
||||||
processConflictInput.value = s.process_conflict_threshold ?? 0.50
|
|
||||||
} catch { /* non-fatal */ }
|
} catch { /* non-fatal */ }
|
||||||
await refresh()
|
await refresh()
|
||||||
if (running.value) startPoll()
|
if (running.value) startPoll()
|
||||||
@@ -381,39 +308,29 @@ function startAutoPoll() {
|
|||||||
function stopAutoPoll() { if (autoTimer) { clearInterval(autoTimer); autoTimer = null } }
|
function stopAutoPoll() { if (autoTimer) { clearInterval(autoTimer); autoTimer = null } }
|
||||||
|
|
||||||
async function onToggleAuto(val) {
|
async function onToggleAuto(val) {
|
||||||
const ok = await save({ head_auto_apply_enabled: !!val },
|
settingBusy.value = true
|
||||||
{ successMessage: val ? 'Auto-apply on' : 'Auto-apply off', errorPrefix: 'Could not update' })
|
try {
|
||||||
if (!ok) autoEnabled.value = !val // revert the switch
|
await mlSettings.patchSettings({ head_auto_apply_enabled: !!val })
|
||||||
|
toast({ text: val ? 'Auto-apply on' : 'Auto-apply off', type: 'success' })
|
||||||
|
} catch (e) {
|
||||||
|
autoEnabled.value = !val // revert the switch
|
||||||
|
toast({ text: `Could not update: ${e.message}`, type: 'error' })
|
||||||
|
} finally {
|
||||||
|
settingBusy.value = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
async function onSaveSettings() {
|
async function onSaveSettings() {
|
||||||
await save({
|
settingBusy.value = true
|
||||||
head_auto_apply_precision: Number(autoPrecisionInput.value),
|
try {
|
||||||
head_auto_apply_min_positives: Number(autoMinPosInput.value),
|
await mlSettings.patchSettings({
|
||||||
})
|
head_auto_apply_precision: Number(autoPrecisionInput.value),
|
||||||
}
|
head_auto_apply_min_positives: Number(autoMinPosInput.value),
|
||||||
|
})
|
||||||
async function onTogglePresentation(val) {
|
} catch (e) {
|
||||||
const ok = await save({ presentation_auto_apply_enabled: !!val },
|
toast({ text: `Could not save: ${e.message}`, type: 'error' })
|
||||||
{ successMessage: val ? 'Chrome auto-hide on' : 'Chrome auto-hide off', errorPrefix: 'Could not update' })
|
} finally {
|
||||||
if (!ok) presentationEnabled.value = !val // revert the switch
|
settingBusy.value = false
|
||||||
}
|
}
|
||||||
async function onSavePresentation() {
|
|
||||||
await save({
|
|
||||||
presentation_auto_apply_threshold: Number(presentationThresholdInput.value),
|
|
||||||
presentation_conflict_threshold: Number(presentationConflictInput.value),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onToggleProcess(val) {
|
|
||||||
const ok = await save({ process_auto_apply_enabled: !!val },
|
|
||||||
{ successMessage: val ? 'WIP auto-tag on' : 'WIP auto-tag off', errorPrefix: 'Could not update' })
|
|
||||||
if (!ok) processEnabled.value = !val // revert the switch
|
|
||||||
}
|
|
||||||
async function onSaveProcess() {
|
|
||||||
await save({
|
|
||||||
process_auto_apply_threshold: Number(processThresholdInput.value),
|
|
||||||
process_conflict_threshold: Number(processConflictInput.value),
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
function onPreview() { startSweep(true) }
|
function onPreview() { startSweep(true) }
|
||||||
function onApplyNow() { startSweep(false) }
|
function onApplyNow() { startSweep(false) }
|
||||||
@@ -439,6 +356,7 @@ function sweepConcepts(run) {
|
|||||||
.sort((a, b) => b.applied - a.applied)
|
.sort((a, b) => b.applied - a.applied)
|
||||||
}
|
}
|
||||||
function sweepTotal(run) { return run?.n_applied ?? 0 }
|
function sweepTotal(run) { return run?.n_applied ?? 0 }
|
||||||
|
function ratePct(x) { return x == null ? '—' : `${Math.round(x * 100)}%` }
|
||||||
function rateClass(x) {
|
function rateClass(x) {
|
||||||
if (x == null) return ''
|
if (x == null) return ''
|
||||||
if (x <= 0.03) return 'fc-good'
|
if (x <= 0.03) return 'fc-good'
|
||||||
@@ -469,6 +387,12 @@ function relTime(iso) {
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
|
||||||
|
.fc-section-h {
|
||||||
|
font-size: 13px; font-weight: 700; letter-spacing: 0.03em;
|
||||||
|
text-transform: uppercase; color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
.fc-auto {
|
.fc-auto {
|
||||||
border-top: 1px solid rgb(var(--v-theme-surface-light)); padding-top: 16px;
|
border-top: 1px solid rgb(var(--v-theme-surface-light)); padding-top: 16px;
|
||||||
}
|
}
|
||||||
@@ -525,5 +449,7 @@ function relTime(iso) {
|
|||||||
background: rgb(var(--v-theme-surface-light));
|
background: rgb(var(--v-theme-surface-light));
|
||||||
padding: 1px 6px; border-radius: 999px;
|
padding: 1px 6px; border-radius: 999px;
|
||||||
}
|
}
|
||||||
|
.fc-good { color: rgb(var(--v-theme-success)); }
|
||||||
.fc-ok { color: rgb(var(--v-theme-on-surface)); }
|
.fc-ok { color: rgb(var(--v-theme-on-surface)); }
|
||||||
|
.fc-weak { color: rgb(var(--v-theme-error)); }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -15,7 +15,7 @@
|
|||||||
being dropped as duplicates;</strong> raise it to collapse more
|
being dropped as duplicates;</strong> raise it to collapse more
|
||||||
look-alikes. Applies to new imports.
|
look-alikes. Applies to new imports.
|
||||||
</div>
|
</div>
|
||||||
<v-row no-gutters class="align-center">
|
<v-row align="center" no-gutters>
|
||||||
<v-col cols="12" sm="9">
|
<v-col cols="12" sm="9">
|
||||||
<v-slider
|
<v-slider
|
||||||
v-model="local.phash_threshold"
|
v-model="local.phash_threshold"
|
||||||
@@ -79,46 +79,6 @@
|
|||||||
</v-col>
|
</v-col>
|
||||||
</v-row>
|
</v-row>
|
||||||
|
|
||||||
<v-divider class="my-5" />
|
|
||||||
|
|
||||||
<!-- Title-based WIP auto-tagging (task #1458). The switch gates the LIVE
|
|
||||||
import hook; the button runs the one-off back-catalogue scan (an
|
|
||||||
explicit action — it is deliberately not a scheduled sweep so it can't
|
|
||||||
silently re-apply a WIP tag you removed by hand). -->
|
|
||||||
<div class="fc-wip">
|
|
||||||
<div class="fc-wip__title">WIP auto-tagging</div>
|
|
||||||
<v-switch
|
|
||||||
v-model="local.wip_title_tagging_enabled"
|
|
||||||
label="Tag work-in-progress from post titles"
|
|
||||||
density="compact" hide-details color="primary" @change="save"
|
|
||||||
/>
|
|
||||||
<div class="fc-help mb-3">
|
|
||||||
When a post's title says <strong>“WIP”</strong> or
|
|
||||||
<strong>“work in progress”</strong>, new imports get the
|
|
||||||
<code>wip</code> tag automatically — keeping unfinished pieces out of
|
|
||||||
the Explore browse. Applies to new imports; run the scan below to catch
|
|
||||||
posts already in your library.
|
|
||||||
</div>
|
|
||||||
<v-switch
|
|
||||||
v-model="local.wip_soft_title_tagging_enabled"
|
|
||||||
label="Also tag “sketch” / “doodle” titles (lower precision)"
|
|
||||||
density="compact" hide-details color="primary" @change="save"
|
|
||||||
/>
|
|
||||||
<div class="fc-help mb-3">
|
|
||||||
Extends the above to softer cues (<code>sketch</code>, <code>doodle</code>,
|
|
||||||
<code>scribble</code>). These stay <strong>visible</strong> and never train
|
|
||||||
the tagging model — a daily audit flags any that actually look like finished
|
|
||||||
art for review. Off by default.
|
|
||||||
</div>
|
|
||||||
<v-btn
|
|
||||||
variant="tonal" color="primary" size="small"
|
|
||||||
:loading="store.wipScanBusy" prepend-icon="mdi-magnify"
|
|
||||||
@click="store.scanWipTitles()"
|
|
||||||
>
|
|
||||||
Scan existing posts for WIP titles
|
|
||||||
</v-btn>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<v-alert v-if="store.settingsError" type="error" variant="tonal" class="mt-2" closable>
|
<v-alert v-if="store.settingsError" type="error" variant="tonal" class="mt-2" closable>
|
||||||
{{ store.settingsError }}
|
{{ store.settingsError }}
|
||||||
</v-alert>
|
</v-alert>
|
||||||
@@ -149,8 +109,6 @@ const local = reactive({
|
|||||||
skip_transparent: false, transparency_threshold: 0.9,
|
skip_transparent: false, transparency_threshold: 0.9,
|
||||||
skip_single_color: false, single_color_threshold: 0.95,
|
skip_single_color: false, single_color_threshold: 0.95,
|
||||||
phash_threshold: 10,
|
phash_threshold: 10,
|
||||||
wip_title_tagging_enabled: true,
|
|
||||||
wip_soft_title_tagging_enabled: false,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
||||||
@@ -166,18 +124,11 @@ async function save() {
|
|||||||
color: rgb(var(--v-theme-on-surface-variant));
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
margin-top: 2px;
|
margin-top: 2px;
|
||||||
}
|
}
|
||||||
.fc-phash__title,
|
.fc-phash__title {
|
||||||
.fc-wip__title {
|
|
||||||
font-size: 0.95rem;
|
font-size: 0.95rem;
|
||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
color: rgb(var(--v-theme-on-surface));
|
color: rgb(var(--v-theme-on-surface));
|
||||||
}
|
}
|
||||||
.fc-wip code {
|
|
||||||
font-size: 0.85em;
|
|
||||||
padding: 1px 4px;
|
|
||||||
border-radius: 3px;
|
|
||||||
background: rgb(var(--v-theme-surface-variant));
|
|
||||||
}
|
|
||||||
/* Headroom so the tick labels (Exact/Strict/Default/Loose) aren't clipped. */
|
/* Headroom so the tick labels (Exact/Strict/Default/Loose) aren't clipped. */
|
||||||
.fc-phash__slider { margin-bottom: 18px; }
|
.fc-phash__slider { margin-bottom: 18px; }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
@@ -39,14 +39,13 @@
|
|||||||
import { toast } from '../../utils/toast.js'
|
import { toast } from '../../utils/toast.js'
|
||||||
import { onMounted, ref } from 'vue'
|
import { onMounted, ref } from 'vue'
|
||||||
import { useMLStore } from '../../stores/ml.js'
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
import { useSettingSave } from '../../composables/useSettingSave.js'
|
|
||||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
import QueueStatusBar from './QueueStatusBar.vue'
|
import QueueStatusBar from './QueueStatusBar.vue'
|
||||||
const store = useMLStore()
|
const store = useMLStore()
|
||||||
const { busy: saving, save } = useSettingSave(store.patchSettings)
|
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
const done = ref(false)
|
const done = ref(false)
|
||||||
const enabled = ref(true)
|
const enabled = ref(true)
|
||||||
|
const saving = ref(false)
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
try {
|
try {
|
||||||
await store.loadSettings()
|
await store.loadSettings()
|
||||||
@@ -56,12 +55,21 @@ onMounted(async () => {
|
|||||||
} catch { /* non-fatal */ }
|
} catch { /* non-fatal */ }
|
||||||
})
|
})
|
||||||
async function onToggle() {
|
async function onToggle() {
|
||||||
const ok = await save({ cpu_embed_enabled: enabled.value }, {
|
saving.value = true
|
||||||
successMessage: enabled.value
|
try {
|
||||||
? 'CPU embedding on — imports queue embeds for the ml-worker'
|
await store.patchSettings({ cpu_embed_enabled: enabled.value })
|
||||||
: 'CPU embedding off — the GPU embed backfill owns whole-image embeds',
|
toast({
|
||||||
})
|
text: enabled.value
|
||||||
if (!ok) enabled.value = !enabled.value
|
? 'CPU embedding on — imports queue embeds for the ml-worker'
|
||||||
|
: 'CPU embedding off — the GPU embed backfill owns whole-image embeds',
|
||||||
|
type: 'success',
|
||||||
|
})
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Could not save: ${e.message}`, type: 'error' })
|
||||||
|
enabled.value = !enabled.value
|
||||||
|
} finally {
|
||||||
|
saving.value = false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
async function run() {
|
async function run() {
|
||||||
busy.value = true
|
busy.value = true
|
||||||
@@ -72,4 +80,5 @@ async function run() {
|
|||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
.fc-muted { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
+16
-18
@@ -12,17 +12,17 @@
|
|||||||
</div>
|
</div>
|
||||||
<v-row>
|
<v-row>
|
||||||
<v-col cols="12" sm="6">
|
<v-col cols="12" sm="6">
|
||||||
<SettingNumberField
|
<v-text-field
|
||||||
v-model="local.video_frame_interval_seconds"
|
v-model.number="local.video_frame_interval_seconds"
|
||||||
label="Frame interval (s)" :min="0.5" :step="0.5"
|
label="Frame interval (s)" type="number" min="0.5" step="0.5"
|
||||||
density="comfortable" max-width="none" @change="onSave"
|
density="comfortable" hide-details @change="save"
|
||||||
/>
|
/>
|
||||||
</v-col>
|
</v-col>
|
||||||
<v-col cols="12" sm="6">
|
<v-col cols="12" sm="6">
|
||||||
<SettingNumberField
|
<v-text-field
|
||||||
v-model="local.video_max_frames"
|
v-model.number="local.video_max_frames"
|
||||||
label="Max frames" :min="1" :step="1"
|
label="Max frames" type="number" min="1" step="1"
|
||||||
density="comfortable" max-width="none" @change="onSave"
|
density="comfortable" hide-details @change="save"
|
||||||
/>
|
/>
|
||||||
</v-col>
|
</v-col>
|
||||||
</v-row>
|
</v-row>
|
||||||
@@ -32,23 +32,21 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { toast } from '../../utils/toast.js'
|
||||||
import { reactive, watch } from 'vue'
|
import { reactive, watch } from 'vue'
|
||||||
import { useMLStore } from '../../stores/ml.js'
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
import SettingNumberField from '../common/SettingNumberField.vue'
|
|
||||||
import { useSettingSave } from '../../composables/useSettingSave.js'
|
|
||||||
|
|
||||||
const store = useMLStore()
|
const store = useMLStore()
|
||||||
const { save } = useSettingSave(store.patchSettings)
|
|
||||||
const local = reactive({})
|
const local = reactive({})
|
||||||
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
||||||
|
|
||||||
// SettingNumberField clamps interval to ≥0.5 and max-frames to ≥1 before this
|
async function save() {
|
||||||
// fires, so an out-of-range value never reaches the API.
|
const patch = {
|
||||||
function onSave() {
|
video_frame_interval_seconds: local.video_frame_interval_seconds,
|
||||||
save({
|
video_max_frames: local.video_max_frames
|
||||||
video_frame_interval_seconds: Number(local.video_frame_interval_seconds),
|
}
|
||||||
video_max_frames: Number(local.video_max_frames),
|
try { await store.patchSettings(patch) }
|
||||||
})
|
catch (e) { toast({ text: e.message, type: 'error' }) }
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
@@ -13,7 +13,6 @@
|
|||||||
</p>
|
</p>
|
||||||
<div class="fc-tile-stack">
|
<div class="fc-tile-stack">
|
||||||
<ImportFiltersForm />
|
<ImportFiltersForm />
|
||||||
<TranslationCard />
|
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
@@ -24,7 +23,6 @@
|
|||||||
the CPU fallback.
|
the CPU fallback.
|
||||||
</p>
|
</p>
|
||||||
<div class="fc-tile-stack">
|
<div class="fc-tile-stack">
|
||||||
<VideoEmbeddingCard />
|
|
||||||
<GpuAgentCard />
|
<GpuAgentCard />
|
||||||
<GpuTriageCard />
|
<GpuTriageCard />
|
||||||
<MLBackfillCard />
|
<MLBackfillCard />
|
||||||
@@ -37,7 +35,7 @@
|
|||||||
Suggestion thresholds, trained heads and tag aliases.
|
Suggestion thresholds, trained heads and tag aliases.
|
||||||
</p>
|
</p>
|
||||||
<div class="fc-tile-stack">
|
<div class="fc-tile-stack">
|
||||||
<CropProposersCard />
|
<MLThresholdSliders />
|
||||||
<HeadsCard />
|
<HeadsCard />
|
||||||
<AliasTable />
|
<AliasTable />
|
||||||
</div>
|
</div>
|
||||||
@@ -70,15 +68,13 @@
|
|||||||
import { onMounted, onUnmounted } from 'vue'
|
import { onMounted, onUnmounted } from 'vue'
|
||||||
|
|
||||||
import ImportFiltersForm from './ImportFiltersForm.vue'
|
import ImportFiltersForm from './ImportFiltersForm.vue'
|
||||||
import TranslationCard from './TranslationCard.vue'
|
|
||||||
import MLBackfillCard from './MLBackfillCard.vue'
|
import MLBackfillCard from './MLBackfillCard.vue'
|
||||||
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
import ThumbnailBackfillCard from './ThumbnailBackfillCard.vue'
|
||||||
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
import ArchiveReextractCard from './ArchiveReextractCard.vue'
|
||||||
import MissingFileRepairCard from './MissingFileRepairCard.vue'
|
import MissingFileRepairCard from './MissingFileRepairCard.vue'
|
||||||
import GpuTriageCard from './GpuTriageCard.vue'
|
import GpuTriageCard from './GpuTriageCard.vue'
|
||||||
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
||||||
import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
|
import MLThresholdSliders from './MLThresholdSliders.vue'
|
||||||
import CropProposersCard from './CropProposersCard.vue'
|
|
||||||
import HeadsCard from './HeadsCard.vue'
|
import HeadsCard from './HeadsCard.vue'
|
||||||
import GpuAgentCard from './GpuAgentCard.vue'
|
import GpuAgentCard from './GpuAgentCard.vue'
|
||||||
import AliasTable from './AliasTable.vue'
|
import AliasTable from './AliasTable.vue'
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<template>
|
<template>
|
||||||
<v-row density="compact">
|
<v-row dense>
|
||||||
<v-col v-for="card in cards" :key="card.label" cols="12" sm="6" md="4" lg="3" xl="2">
|
<v-col v-for="card in cards" :key="card.label" cols="12" sm="6" md="4" lg="3" xl="2">
|
||||||
<v-card class="fc-stat">
|
<v-card class="fc-stat">
|
||||||
<v-card-text>
|
<v-card-text>
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user