Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 bvandeusen
|
f2f6b6d25e | ||
|
bvandeusen
|
0822240fde | ||
|
bvandeusen
|
27f7f3fd01 | ||
|
bvandeusen
|
c5bf564f53 | ||
|
bvandeusen
|
602c7d275d |
+19
-98
@@ -1,103 +1,24 @@
|
||||
# FabledCurator configuration.
|
||||
#
|
||||
# Copy to `.env` and edit before your first production start:
|
||||
#
|
||||
# cp .env.example .env
|
||||
#
|
||||
# Only the two values under CHANGE THESE actually need your attention. The
|
||||
# rest have working defaults baked into docker-compose.yml and are listed
|
||||
# here so you know they exist, not because you have to set them.
|
||||
#
|
||||
# Almost nothing else lives here on purpose. FabledCurator is configured from
|
||||
# its own Settings UI, backed by the database — no restart, no YAML. If you
|
||||
# are looking for where to set an import path, a download schedule or an ML
|
||||
# threshold, it is in the app, not in this file.
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CHANGE THESE
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# The Postgres password. docker-compose.yml falls back to a published default
|
||||
# (`fabledcurator_dev`) so that `docker compose up` works with no config at
|
||||
# all — which is exactly why you must not leave it at that on a real install.
|
||||
# It is the credential protecting your stored platform session cookies.
|
||||
DB_PASSWORD=
|
||||
|
||||
# Sets Quart's app.secret_key. Today it signs nothing: FabledCurator has no
|
||||
# login and uses no session cookies, so no value here is protecting anything
|
||||
# right now. Set it anyway. It is required at boot rather than defaulted so
|
||||
# that the day something session-backed does land, no instance is already
|
||||
# running on a value published in this file.
|
||||
#
|
||||
# openssl rand -hex 32
|
||||
SECRET_KEY=
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# FIRST BOOT ONLY — then delete this line
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# FabledCurator encrypts your stored platform credentials with a Fernet key it
|
||||
# keeps at /images/secrets/credential_key.b64 — inside the ./images bind mount,
|
||||
# so it outlives the container. On a brand-new install that file does not exist
|
||||
# yet, and the app REFUSES TO START rather than quietly create one:
|
||||
#
|
||||
# MissingCredentialKey: Fernet key file not found at
|
||||
# /images/secrets/credential_key.b64
|
||||
#
|
||||
# That refusal is deliberate. Auto-creating a key is indistinguishable from the
|
||||
# disaster case — a restore that brought the database back but lost
|
||||
# ./images/secrets — and there it would mint a key that cannot decrypt anything,
|
||||
# leaving an instance that looks healthy while every paywalled download fails.
|
||||
# So the choice is yours to make explicitly, once.
|
||||
#
|
||||
# Set this for your first `up`, watch the container come up, then DELETE THE
|
||||
# LINE. Leaving it set disarms the protection permanently, on an instance that
|
||||
# by then has credentials worth protecting.
|
||||
#
|
||||
# BACK UP ./images/secrets/ ALONGSIDE YOUR DATABASE. The key is the only thing
|
||||
# that can read your stored credentials; a database restored without it needs
|
||||
# every credential re-entered by hand.
|
||||
CURATOR_BOOTSTRAP_NEW_KEY=1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Optional — defaults are fine
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Host port the UI is published on. The container always listens on 8080;
|
||||
# this is only the left-hand side of the port mapping.
|
||||
PORT=8080
|
||||
|
||||
# DEBUG | INFO | WARNING | ERROR
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# Postgres identity. Change these only if you are pointing at a database you
|
||||
# manage yourself — the bundled postgres service is created with whatever is
|
||||
# set here, so changing them after the first start will not rename anything.
|
||||
# Database
|
||||
DB_USER=fabledcurator
|
||||
DB_PASSWORD=changeme_use_a_real_password
|
||||
DB_HOST=postgres
|
||||
DB_PORT=5432
|
||||
DB_NAME=fabledcurator
|
||||
|
||||
# Set by docker-compose.yml to reach the bundled services. Override only when
|
||||
# running Postgres or Redis outside this stack.
|
||||
# DB_HOST=postgres
|
||||
# DB_PORT=5432
|
||||
# CELERY_BROKER_URL=redis://redis:6379/0
|
||||
# CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
# Redis / Celery
|
||||
CELERY_BROKER_URL=redis://redis:6379/0
|
||||
CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||
|
||||
# App
|
||||
# Generate with: openssl rand -hex 32
|
||||
SECRET_KEY=changeme_32_byte_hex_secret
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# There is no authentication variable here, and that is not an omission
|
||||
# ---------------------------------------------------------------------------
|
||||
#
|
||||
# FabledCurator has no login, no accounts and no permission model. Anything
|
||||
# that can reach PORT is an administrator and can read the platform session
|
||||
# cookies the app stores for Patreon and SubscribeStar.
|
||||
#
|
||||
# Bind it to a trusted network. See "Before you expose it" in README.md and
|
||||
# the deployment posture section of SECURITY.md.
|
||||
#
|
||||
# The Firefox extension's API key is NOT configured here — it is generated
|
||||
# automatically on first use and shown under Settings → Maintenance, where you
|
||||
# can also rotate it.
|
||||
# Extension API key — used in FC-3, lands later but reserved now
|
||||
# Generate with: openssl rand -hex 32
|
||||
EXTENSION_API_KEY=
|
||||
|
||||
# Logging
|
||||
LOG_LEVEL=INFO
|
||||
|
||||
# Deployment posture: plain HTTP (no TLS in the app; reverse proxy if needed)
|
||||
# See docs/superpowers/specs/2026-05-13-fabledcurator-merge-design.md §2.1
|
||||
|
||||
@@ -1,350 +0,0 @@
|
||||
|
||||
# TEMPORARY — milestone 328. Delete once the baseline has shipped and settled.
|
||||
#
|
||||
# Collapsing 89 alembic revisions into one baseline has exactly one dangerous
|
||||
# failure: the baseline does not reproduce the schema the chain produced, and
|
||||
# the divergence surfaces later, on the operator's live data, in whatever
|
||||
# migration comes next.
|
||||
#
|
||||
# So the comparison happens in CI, against a throwaway pgvector Postgres, where
|
||||
# nothing is at risk. It answers one question: does `upgrade head` on the
|
||||
# collapsed tree produce the same schema as `upgrade head` on the full chain?
|
||||
#
|
||||
# The chain is read out of GIT, not the working tree, which is what lets this
|
||||
# keep working now that the revisions are deleted — `chain_ref` names a commit
|
||||
# that still carries 0001..0089. That is the whole reason this is a workflow
|
||||
# rather than a script someone ran once.
|
||||
#
|
||||
# WHAT THIS CANNOT SEE, and it matters: the comparison is of SCHEMA. Migrations
|
||||
# 0002 and 0003 also INSERTED rows (the import_settings and ml_settings
|
||||
# singletons), and the application reads those with scalar_one(), which raises
|
||||
# on an empty result. A baseline that omitted them would produce an identical
|
||||
# schema, pass this check with a perfect diff, and crash a fresh install on its
|
||||
# first settings access. Only running the app against a new database finds
|
||||
# that class of defect. Do not read a green run here as "the baseline is
|
||||
# correct" — read it as "the schema is correct".
|
||||
#
|
||||
# Autogenerate now emits nearly all of the baseline unaided, which was NOT true
|
||||
# before #3275 put the previously migration-only objects onto the models — the
|
||||
# HNSW index with its opclass, the COALESCE expression index, the partial
|
||||
# unique indexes, 107 server_defaults, the enum CHECKs. An earlier attempt at
|
||||
# this squash was reverted precisely because the generator dropped them all
|
||||
# silently. What still needs hand-adding is only what cannot live in a model:
|
||||
# the two CREATE EXTENSION statements, the two seed rows, and the pgvector
|
||||
# import the generator forgets to write.
|
||||
name: Alembic baseline
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
chain_ref:
|
||||
description: 'Commit/tag carrying the full 0001..0089 chain (pinned: the tree no longer has it)'
|
||||
type: string
|
||||
default: '725bf15'
|
||||
mode:
|
||||
description: 'chain = compare against this tree''s migrations; models = compare against a schema built from the MODELS'
|
||||
type: string
|
||||
default: 'chain'
|
||||
|
||||
jobs:
|
||||
compare:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Full history is the point: `chain_ref` is read out of git, so a
|
||||
# shallow clone would not have the revisions to compare against.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve the Postgres service and install deps
|
||||
run: |
|
||||
set -eux
|
||||
# Same service-IP dance as build.yml's integration job; see the long
|
||||
# comment there for why the job name must stay separator-free.
|
||||
PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
test -n "$PG"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
test -n "$PG_IP"
|
||||
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
||||
echo "DB_HOST=$PG_IP" >> "$GITHUB_ENV"
|
||||
# Socket probe in python, not bash's /dev/tcp — these steps run under
|
||||
# `sh -e`, where that path does not exist. Same fix and same reasoning
|
||||
# as build.yml's integration job; see the comment there.
|
||||
pg_ready=""
|
||||
for i in $(seq 1 60); do
|
||||
if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then
|
||||
pg_ready=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
if [ -z "$pg_ready" ]; then
|
||||
echo "postgres at $PG_IP:5432 did not accept a connection within 120s"
|
||||
exit 1
|
||||
fi
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt
|
||||
else
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
|
||||
# DB 1: the 87-revision chain, read out of git at `chain_ref`.
|
||||
#
|
||||
# A git worktree rather than a checkout, so the current tree — which is
|
||||
# what we are testing — is left completely alone.
|
||||
- name: Build the schema the OLD chain produces
|
||||
env:
|
||||
CHAIN_REF: ${{ github.event.inputs.chain_ref }}
|
||||
THIS_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -eux
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_chain
|
||||
# Blank means "the chain in this ref", which is what you want while
|
||||
# the chain is still intact — comparing the models against a PINNED
|
||||
# older commit reports every migration written since as a difference.
|
||||
# Pin it only after the collapse, when the tree no longer has them.
|
||||
git worktree add /tmp/chain "${CHAIN_REF:-$THIS_SHA}"
|
||||
ls /tmp/chain/alembic/versions/*.py | wc -l
|
||||
cd /tmp/chain
|
||||
DB_NAME=fc_chain alembic upgrade head
|
||||
cd -
|
||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||
--no-owner --no-privileges -d fc_chain > chain.sql
|
||||
wc -l chain.sql
|
||||
# Emit the dump itself, checksummed, for local analysis. Reconciling
|
||||
# the models against the deployed schema (#3275) needs the ACTUAL
|
||||
# schema, not an inference from a diff — parsing table context out of
|
||||
# unified-diff hunks drops every table whose CREATE TABLE line falls
|
||||
# outside a hunk, which silently under-reports.
|
||||
#
|
||||
# base64 + sha256 for the same reason as the candidate: a plain cat
|
||||
# of a file this size was truncated mid-line by the runner with the
|
||||
# step still green (run 4964).
|
||||
set +x
|
||||
B64=$(base64 -w 120 chain.sql)
|
||||
echo "===== BEGIN CHAIN SCHEMA (base64) ====="
|
||||
echo "$B64"
|
||||
echo "===== END CHAIN SCHEMA ====="
|
||||
echo "chain-sha256: $(sha256sum chain.sql | cut -d' ' -f1)"
|
||||
echo "chain-bytes: $(wc -c < chain.sql)"
|
||||
set -x
|
||||
|
||||
# A candidate baseline, autogenerated from the models against an EMPTY
|
||||
# database so every table shows up as a create. Printed for a human to
|
||||
# finish — it will be missing the three raw-SQL items named at the top.
|
||||
#
|
||||
# Gated on the TREE, not on a workflow input. A `type: boolean` input
|
||||
# read back as `github.event.inputs.generate == 'true'` silently
|
||||
# evaluated false on this runner (run 4960 skipped this step entirely
|
||||
# with no diagnostic) — the same `github.event.inputs` typing quirk
|
||||
# build.yml already works around. The file count is the real question
|
||||
# anyway: there is nothing to generate once the chain is collapsed.
|
||||
- name: Autogenerate a candidate baseline
|
||||
run: |
|
||||
set -eux
|
||||
if [ "$(ls alembic/versions/*.py | wc -l)" -le 1 ]; then
|
||||
echo "already collapsed — nothing to generate"
|
||||
exit 0
|
||||
fi
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_gen
|
||||
# Hide the existing revisions so alembic sees an empty history and
|
||||
# emits the whole schema rather than a delta.
|
||||
mkdir -p /tmp/versions_held
|
||||
mv alembic/versions/*.py /tmp/versions_held/ 2>/dev/null || true
|
||||
DB_NAME=fc_gen alembic revision --autogenerate -m "baseline" || true
|
||||
# Printed rather than uploaded: the repo dropped actions/upload-artifact
|
||||
# in 2026-05, when the runner could not run v4+, and the job log is the
|
||||
# retrieval channel this job has proven. (gitea/runner 3.x runs stock
|
||||
# upload-artifact now — Scribe snippet #2271 — so an artifact is an
|
||||
# option if the log ever stops being enough.)
|
||||
#
|
||||
# base64, not the raw file. A plain `cat` of the ~33KB candidate was
|
||||
# TRUNCATED MID-LINE by the runner on run 4964 — it stopped inside
|
||||
# `sa.Column('mime', sa.String(length=128)` and carried straight on
|
||||
# to the next traced command, with the step still green. A silent
|
||||
# cut in the middle of a schema definition is the worst possible
|
||||
# failure here, because the truncated text still looks like a
|
||||
# plausible file.
|
||||
#
|
||||
# base64 at a fixed narrow width gives many short lines instead of
|
||||
# few long ones, and — the actual point — a checksum and a line
|
||||
# count that make truncation DETECTABLE rather than invisible.
|
||||
set +x
|
||||
F=$(ls alembic/versions/*.py | head -1)
|
||||
B64=$(base64 -w 120 "$F")
|
||||
echo "===== BEGIN CANDIDATE BASELINE (base64) ====="
|
||||
echo "$B64"
|
||||
echo "===== END CANDIDATE BASELINE ====="
|
||||
echo "candidate-sha256: $(sha256sum "$F" | cut -d' ' -f1)"
|
||||
echo "candidate-bytes: $(wc -c < "$F")"
|
||||
echo "candidate-b64-lines: $(echo "$B64" | wc -l)"
|
||||
set -x
|
||||
mkdir -p /tmp/candidate
|
||||
cp alembic/versions/*.py /tmp/candidate/
|
||||
# Put the tree back exactly as it was; this job never mutates state.
|
||||
rm -f alembic/versions/*.py
|
||||
mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true
|
||||
|
||||
# DB 2: what the CURRENT tree produces.
|
||||
#
|
||||
# `mode: models` applies the candidate autogenerated from the MODELS
|
||||
# instead, which is what answers "do the models describe the schema?" —
|
||||
# the question #3275 exists because nobody had ever asked it. Under that
|
||||
# mode a clean diff means autogenerate is trustworthy again.
|
||||
#
|
||||
# The two extensions are created by hand first. They are database
|
||||
# objects, not table metadata, so no model can carry them and their
|
||||
# absence is not a model defect — it is simply outside what this
|
||||
# comparison is asking about.
|
||||
- name: Build the schema the CURRENT tree produces
|
||||
env:
|
||||
MODE: ${{ github.event.inputs.mode }}
|
||||
run: |
|
||||
set -eux
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base
|
||||
if [ "${MODE:-chain}" = "models" ]; then
|
||||
docker exec "$PG_CONTAINER" psql -U fabledcurator -d fc_base \
|
||||
-c "CREATE EXTENSION IF NOT EXISTS vector" \
|
||||
-c "CREATE EXTENSION IF NOT EXISTS tsm_system_rows"
|
||||
mkdir -p /tmp/held
|
||||
mv alembic/versions/*.py /tmp/held/
|
||||
cp /tmp/candidate/*.py alembic/versions/
|
||||
# Autogenerate EMITS pgvector.sqlalchemy.vector.VECTOR(...) without
|
||||
# importing it, so the file it writes cannot run:
|
||||
# NameError: name 'pgvector' is not defined
|
||||
# Observed on run 4988, which is the proof rather than the theory.
|
||||
# This is a defect in the GENERATOR, not in the models, so it is
|
||||
# repaired here rather than counted as a schema difference — the
|
||||
# comparison is about whether the models describe the schema.
|
||||
sed -i '0,/^import sqlalchemy as sa$/s//import sqlalchemy as sa\nimport pgvector.sqlalchemy.vector/' alembic/versions/*.py
|
||||
grep -n 'import pgvector' alembic/versions/*.py
|
||||
ls alembic/versions/*.py
|
||||
DB_NAME=fc_base alembic upgrade head
|
||||
rm -f alembic/versions/*.py
|
||||
mv /tmp/held/*.py alembic/versions/
|
||||
else
|
||||
ls alembic/versions/*.py | wc -l
|
||||
DB_NAME=fc_base alembic upgrade head
|
||||
fi
|
||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||
--no-owner --no-privileges -d fc_base > baseline.sql
|
||||
wc -l baseline.sql
|
||||
|
||||
# The verdict.
|
||||
#
|
||||
# pg_dump orders dumpable objects by name within type, not by creation
|
||||
# order, so two schemas built by different routes are directly
|
||||
# comparable. Normalisation is deliberately minimal, because a filter
|
||||
# that hides a real difference is the one way this check passes when it
|
||||
# should fail — blank lines, SQL comments, trailing whitespace, and:
|
||||
#
|
||||
# \restrict / \unrestrict — a per-invocation RANDOM NONCE that newer
|
||||
# pg_dump emits to fence the dump against injection during restore. It
|
||||
# differs on every run by construction, so it is noise by definition,
|
||||
# not a schema difference. Measured on run 4960, the control: two dumps
|
||||
# of the SAME schema came back 1123 lines each and differed on exactly
|
||||
# these two lines and nothing else. That control is what licenses this
|
||||
# filter — it was observed to be the only false positive, rather than
|
||||
# assumed to be one.
|
||||
# Column ORDER inside a CREATE TABLE is compared separately from column
|
||||
# CONTENT, and only content is fatal.
|
||||
#
|
||||
# A table built by 87 migrations has its columns in ADD COLUMN order; the
|
||||
# same table built in one shot has them in declaration order. That is a
|
||||
# real and permanent difference which no baseline can erase — the
|
||||
# operator's existing database keeps chain order forever, a fresh install
|
||||
# gets model order — so a check that fails on it would never pass and
|
||||
# would teach nothing. FC reaches every column through the ORM by name,
|
||||
# and `SELECT *` ordering is not depended on anywhere.
|
||||
#
|
||||
# So the second pass SORTS the column lines within each CREATE TABLE
|
||||
# rather than DELETING them. That distinction is the whole point: sorting
|
||||
# cannot hide a column that exists on one side only, or one whose type,
|
||||
# nullability or default differs — those still land in the diff. A filter
|
||||
# could have hidden all three.
|
||||
#
|
||||
# Both diffs are reported. The ordered one is informational; the
|
||||
# order-insensitive one is the verdict.
|
||||
- name: Diff
|
||||
run: |
|
||||
set -eu
|
||||
norm() {
|
||||
grep -vE '^\s*(--|$)' "$1" \
|
||||
| grep -vE '^\\(un)?restrict ' \
|
||||
| sed 's/[[:space:]]*$//'
|
||||
}
|
||||
norm chain.sql > a.txt
|
||||
norm baseline.sql > b.txt
|
||||
echo "normalised: chain=$(wc -l < a.txt) lines, current=$(wc -l < b.txt) lines"
|
||||
|
||||
sort_table_columns() {
|
||||
python3 - "$1" <<'PYEOF'
|
||||
import re, sys
|
||||
|
||||
lines = open(sys.argv[1]).read().splitlines()
|
||||
out, block = [], None
|
||||
for line in lines:
|
||||
if block is not None:
|
||||
# ');' on its own closes the CREATE TABLE body.
|
||||
if line.strip() == ");":
|
||||
out.extend(sorted(block))
|
||||
out.append(line)
|
||||
block = None
|
||||
else:
|
||||
# Drop the list comma before sorting. Only the LAST
|
||||
# column lacks one, so keeping it would make every
|
||||
# reordering look like a content change as well — the
|
||||
# comma is punctuation, and carries no schema meaning.
|
||||
block.append(line.rstrip().rstrip(","))
|
||||
continue
|
||||
out.append(line)
|
||||
if re.match(r"CREATE TABLE .*\($", line):
|
||||
block = []
|
||||
if block is not None: # unterminated body: emit it rather than drop it
|
||||
out.extend(block)
|
||||
print("\n".join(out))
|
||||
PYEOF
|
||||
}
|
||||
sort_table_columns a.txt > a.sorted.txt
|
||||
sort_table_columns b.txt > b.sorted.txt
|
||||
test "$(wc -l < a.sorted.txt)" = "$(wc -l < a.txt)"
|
||||
test "$(wc -l < b.sorted.txt)" = "$(wc -l < b.txt)"
|
||||
|
||||
if diff -u a.txt b.txt > schema.diff; then
|
||||
echo "ORDERED DIFF: identical, column order included."
|
||||
else
|
||||
echo "ORDERED DIFF: $(grep -cE '^[+-]' schema.diff) changed lines (informational):"
|
||||
cat schema.diff
|
||||
fi
|
||||
echo
|
||||
echo "================================================================"
|
||||
echo
|
||||
if diff -u a.sorted.txt b.sorted.txt > sorted.diff; then
|
||||
echo "SCHEMAS MATCH — every difference above is column ORDER alone."
|
||||
else
|
||||
echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' sorted.diff) changed lines that are NOT ordering:"
|
||||
cat sorted.diff
|
||||
echo
|
||||
echo "The baseline is wrong, not the database. Do not stamp."
|
||||
exit 1
|
||||
fi
|
||||
+24
-2615
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,124 @@
|
||||
name: CI
|
||||
|
||||
# CI lanes per FabledRulebook/forgejo.md "CI philosophy":
|
||||
# - backend-lint-and-test: ruff + `pytest -m "not integration"`, no service containers.
|
||||
# - frontend-build: vitest unit + vite build.
|
||||
# - integration: pgvector + redis service containers; alembic + `pytest -m integration`.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
backend-lint-and-test:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
# DB_PASSWORD and SECRET_KEY are required by config.py at import time
|
||||
# even though unit tests don't actually touch the DB or use the secret.
|
||||
DB_PASSWORD: ci_unit_test_placeholder
|
||||
SECRET_KEY: ci_unit_test_placeholder
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Install Python deps
|
||||
# ruff is pre-installed in the ci-python image (see CI-Runner/CI-python/
|
||||
# Dockerfile's RUFF_VERSION). Per FabledRulebook ci-runners.md, toolchain
|
||||
# versions live on the runner image, not here.
|
||||
run: pip install -r requirements.txt pytest pytest-asyncio
|
||||
|
||||
- name: Ruff lint
|
||||
run: ruff check backend/ tests/ alembic/
|
||||
|
||||
- name: Pytest (unit only — integration runs in the integration job)
|
||||
run: pytest tests/ -v -m "not integration"
|
||||
|
||||
frontend-build:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
defaults:
|
||||
run:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||
# feedback-no-local-runs). Using `npm install` instead of `npm ci`.
|
||||
# If we want strict lockfile-based reproducibility later, commit a
|
||||
# package-lock.json and flip this back to `npm ci`.
|
||||
- run: npm install --no-audit --no-fund
|
||||
# `npm run check` (vue-tsc --noEmit) skipped: the frontend is pure JS
|
||||
# with no .ts files and no JSDoc annotations, so vue-tsc has nothing
|
||||
# to type-check. Re-enable once we add a tsconfig.json and either
|
||||
# convert to TS or add JSDoc.
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
integration:
|
||||
# This act_runner (swarm-runner v0.6.1) puts service containers on the
|
||||
# default bridge with NO service-name DNS, and publishing fixed host
|
||||
# ports collides with the operator's running docker-compose dev stack on
|
||||
# the same shared daemon. Workaround: publish NO host ports, and reach
|
||||
# each service by its bridge IP — discovered at runtime via the mounted
|
||||
# docker socket (the ci-python image ships /usr/bin/docker). Default-bridge
|
||||
# containers can talk by IP (only embedded DNS is missing), so IP
|
||||
# addressing is reliable here. Everything runs in ONE step so resolved
|
||||
# values don't depend on cross-step env passing. Pattern documented in
|
||||
# FabledRulebook/forgejo.md "CI philosophy".
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Integration suite (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
# Scope to THIS job's service containers (act_runner names them
|
||||
# ...JOB-integration...); the operator's compose stack uses the
|
||||
# same images but different names, so it won't match.
|
||||
PG=$(docker ps --filter "name=JOB-integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=JOB-integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
test -n "$PG_IP" && test -n "$RD_IP"
|
||||
export DB_HOST="$PG_IP"
|
||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||
# Wait for Postgres to accept TCP (bash /dev/tcp; no extra tools).
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
pip install -r requirements.txt pytest pytest-asyncio
|
||||
alembic upgrade head
|
||||
pytest tests/ -v -m integration
|
||||
@@ -1,80 +0,0 @@
|
||||
name: Release
|
||||
|
||||
# A `v*` tag publishes a changelog. It does NOT build anything.
|
||||
#
|
||||
# Milestone 318 step 2 removed the tag trigger from build.yml: by the time
|
||||
# anyone tags a commit, `main` has already built and published it, and a
|
||||
# rebuild would re-push `:c-<sha>` — which rule 145 forbids even when the
|
||||
# source matches, since image configs carry timestamps and "same source" does
|
||||
# not mean "same manifest". That left the tag with no consequence at all.
|
||||
#
|
||||
# This is the consequence it has instead. Step 6 put the derived version in the
|
||||
# Settings footer, so an operator can say WHICH build they are running; this
|
||||
# says what is IN it that was not in the one they ran last month. Both halves
|
||||
# of one question (note #3127 §5).
|
||||
#
|
||||
# Nothing here runs on a schedule and nothing auto-tags on merge. Release tags
|
||||
# are bookmarks — cut one when you will want to point at that day by name,
|
||||
# otherwise don't (note #3127 §0). FC went twelve weeks between v26.06.04.0 and
|
||||
# the next one and nothing was wrong. A schedule would turn an optional
|
||||
# bookmark back into ceremony, which is the thing this milestone is removing.
|
||||
#
|
||||
# Cutting the tag is an explicit operator action under rule 2 ("`main` — never
|
||||
# without explicit request", which since 2026-08-28 covers PR, merge and tag
|
||||
# alike). This lane only decides what happens once they do.
|
||||
#
|
||||
# Requires repo secret RELEASE_TOKEN with the `write:release` scope — the same
|
||||
# PAT build.yml uses for the ext-<version> XPI asset cache.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ['v*']
|
||||
# So a release body can be regenerated after the fact — the publisher PATCHes
|
||||
# an existing release rather than falling through on a conflict, so re-running
|
||||
# this on a tag rewrites the body instead of silently keeping the first one
|
||||
# (note #3127 §6.7).
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'Tag to (re)publish notes for'
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
changelog:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Load-bearing twice over: the previous release is found by walking
|
||||
# ancestry back through the tag graph, and the cross-check against
|
||||
# the derived web version calls artifacts.sh, which reads commit
|
||||
# times. A shallow clone would find no previous tag and emit the
|
||||
# entire history as the changelog — plausible-looking and wrong.
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.event.inputs.tag || github.ref }}
|
||||
|
||||
# The `:c-<sha>` rollback refs are only real if `main` built this commit.
|
||||
# The script checks that against origin/main and downgrades the claim to
|
||||
# "unverified" when it cannot resolve one; fetching it here means that
|
||||
# downgrade stays an actual signal instead of firing on every release.
|
||||
- name: Make main's history resolvable
|
||||
run: git fetch --no-tags --quiet origin +main:refs/remotes/origin/main || true
|
||||
|
||||
# TAG goes through the environment, not through `${{ }}` inside the
|
||||
# run block. The value is operator-supplied, and an expression expanded
|
||||
# into a shell line is expanded BEFORE the shell sees it — there is no
|
||||
# quoting that makes that safe. On a tag push it is empty and the script
|
||||
# falls back to GITHUB_REF.
|
||||
- name: Publish the derived changelog
|
||||
env:
|
||||
RELEASE_TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
TAG: ${{ github.event.inputs.tag }}
|
||||
run: |
|
||||
set -eu
|
||||
if [ -n "${TAG:-}" ]; then
|
||||
python3 scripts/release_notes.py "$TAG"
|
||||
else
|
||||
python3 scripts/release_notes.py
|
||||
fi
|
||||
-23
@@ -61,31 +61,8 @@ Thumbs.db
|
||||
|
||||
# Claude Code per-user local overrides (shared .claude/settings.json is OK to commit)
|
||||
.claude/settings.local.json
|
||||
# Transient scheduler lock/state (committed by accident in 3f30327)
|
||||
.claude/scheduled_tasks.lock
|
||||
.claude/scheduled_tasks*.json
|
||||
|
||||
# Alembic / DB scratch
|
||||
alembic/versions/__pycache__/
|
||||
*.sqlite
|
||||
*.sqlite-journal
|
||||
.superpowers/
|
||||
|
||||
# Raw platform captures (milestone 387 C0 and successors). These are real
|
||||
# authenticated API responses taken from the operator's own account, so they
|
||||
# carry account data — creator lists, pledge amounts, and (in Patreon's case)
|
||||
# the account email inside the `card` resources. They are kept locally because
|
||||
# re-capturing means re-authenticating by hand, and they are the ground truth a
|
||||
# characterization gets re-checked against.
|
||||
#
|
||||
# The whole directory is ignored, not one filename, so a future capture is
|
||||
# covered by this rule instead of needing a new line somebody has to remember.
|
||||
#
|
||||
# SANITIZED fixtures derived from these DO belong in git — put them somewhere
|
||||
# else (tests/fixtures/, not here), with the account data stripped.
|
||||
# Ignore the CONTENTS, not the directory: git does not descend into an
|
||||
# excluded directory, so a negation for a file inside one never takes effect.
|
||||
# Writing it this way lets README.md be committed while everything else here
|
||||
# stays out.
|
||||
tests/fixtures/captures/*
|
||||
!tests/fixtures/captures/README.md
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
# Contributing
|
||||
|
||||
FabledCurator is developed by a single maintainer for their own use, and
|
||||
published because it may be useful to others. That shapes what contribution
|
||||
looks like here.
|
||||
|
||||
**Issues are welcome** — bug reports, and questions about running it, are
|
||||
genuinely useful and often the fastest way to find out that something is
|
||||
broken outside the one environment it was built in.
|
||||
|
||||
**Open an issue before writing a pull request.** Not as a formality: the
|
||||
project has opinions that are not obvious from the code, and it is unpleasant
|
||||
for everyone when a finished patch turns out to conflict with one. A short
|
||||
issue first costs you nothing and may save you an evening.
|
||||
|
||||
**Contributions are licensed under the AGPL-3.0**, like the rest of the
|
||||
project. By submitting one you agree it ships under that licence. There is no
|
||||
CLA and no copyright assignment.
|
||||
|
||||
## Running it for development
|
||||
|
||||
```bash
|
||||
docker compose up -d # UI on http://localhost:8080
|
||||
```
|
||||
|
||||
The dev override (`docker-compose.override.yml`) is auto-merged and builds the
|
||||
app images locally from source, so this needs no `.env` and no registry
|
||||
access. Postgres and Redis ports are exposed on the host.
|
||||
|
||||
## What CI checks
|
||||
|
||||
Every push runs these, and they are the definition of done for a change:
|
||||
|
||||
```bash
|
||||
ruff check backend/ tests/ alembic/ agent/ scripts/ # lint (and import order)
|
||||
pytest tests/ -m "not integration" # backend unit tests
|
||||
pytest tests/ -m integration # needs pgvector + redis
|
||||
cd frontend && npm run test:unit && npm run build # frontend
|
||||
```
|
||||
|
||||
The integration lane builds its schema by running the real migrations
|
||||
(`alembic upgrade head`), never from ORM metadata — so a migration that does
|
||||
not apply cleanly fails CI rather than being discovered later.
|
||||
|
||||
Note for the linter: ruff's isort runs with `order-by-type`, which sorts
|
||||
ALL-CAPS names ahead of CamelCase. `from sqlalchemy import JSON, DateTime, ...`
|
||||
is correct; putting `JSON` alphabetically between `Integer` and `String` is
|
||||
not. This catches people out.
|
||||
|
||||
## Database changes
|
||||
|
||||
The ORM models and the migration chain must agree. This is enforced, and it is
|
||||
enforced because they silently diverged for a long time and nobody noticed
|
||||
until they were compared: the models were missing indexes, defaults and
|
||||
uniqueness guarantees that only ever existed inside a migration, which made
|
||||
`alembic revision --autogenerate` actively unsafe to run.
|
||||
|
||||
So: if you change a model, write the migration; if you write a migration,
|
||||
change the model to match. Both, in the same commit.
|
||||
|
||||
Adding a value to a CHECK-constrained column means swapping the constraint in
|
||||
the same change — the constraint is not documentation, and a new value without
|
||||
it fails at insert time.
|
||||
|
||||
## Branch model
|
||||
|
||||
`dev` is where work happens. `main` is production and is only reached by a
|
||||
merge from `dev`, never pushed to directly. If you are sending a pull request,
|
||||
target `dev`.
|
||||
|
||||
## Style
|
||||
|
||||
Match the surrounding code. The one convention worth stating explicitly is
|
||||
that comments here explain *why*, especially where a choice looks wrong at a
|
||||
glance — a comment recording which migration a constraint came from, or why a
|
||||
default is a `text()` rather than a string, is the kind that has repeatedly
|
||||
turned out to be worth its space.
|
||||
+6
-132
@@ -1,6 +1,6 @@
|
||||
# syntax=docker/dockerfile:1.25
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
FROM node:24-alpine AS frontend-builder
|
||||
FROM node:22-alpine AS frontend-builder
|
||||
WORKDIR /build
|
||||
COPY frontend/package.json frontend/package-lock.json* ./
|
||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||
@@ -18,20 +18,13 @@ ENV PYTHONUNBUFFERED=1 \
|
||||
|
||||
# System deps: ffmpeg (transcode + thumbnails, FC-2), unar (archives, FC-2),
|
||||
# libpq for psycopg, postgresql-client + zstd for FC-5 backup/restore
|
||||
# (pg_dump + tar --zstd), image libs, megatools (mega.nz public-link downloads
|
||||
# for off-platform file-host links, #830 — `megatools dl`; Debian-native, no
|
||||
# external MEGA apt repo needed).
|
||||
# (pg_dump + tar --zstd), image libs.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
unar \
|
||||
libpq5 \
|
||||
postgresql-client \
|
||||
zstd \
|
||||
megatools \
|
||||
# PID 1 for every role. See the ENTRYPOINT note at the foot of this file:
|
||||
# without it the image needs `init: true` in whatever runs it, which is a
|
||||
# deployment remembering a flag for the image to behave correctly.
|
||||
tini \
|
||||
libjpeg62-turbo \
|
||||
libwebp7 \
|
||||
libpng16-16 \
|
||||
@@ -40,59 +33,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt requirements-ml.txt ./
|
||||
COPY requirements.txt ./
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
# --- ML, merged from Dockerfile.ml (milestone 422 step 6) --------------------
|
||||
#
|
||||
# ONE image now serves every lane. It was two because the ML lane ran in its
|
||||
# own container; with the single-container layout (step 5) running every lane
|
||||
# in one process tree, a second image would mean the `ml` lane could never be
|
||||
# enabled from the UI — there would be no worker in this container to enable.
|
||||
#
|
||||
# THE COST, MEASURED from run 7273 rather than guessed — and it is far
|
||||
# smaller than the estimate this comment first carried, which said "everyone
|
||||
# pulls ~4GB":
|
||||
#
|
||||
# torch 2.12.1+cpu wheel 192.3 MB
|
||||
# torchvision 0.27.1+cpu 1.8 MB
|
||||
# transformers / onnxruntime / opencv / sklearn and friends (opencv and
|
||||
# onnxruntime since dropped, #1451 — nothing here imported them)
|
||||
# 62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB
|
||||
# largest newly-pushed layer 222.07 MB
|
||||
#
|
||||
# So the ML code adds a few hundred MB to the pull, not gigabytes. The CPU
|
||||
# index is what makes that true: the default PyPI torch wheel bundles the
|
||||
# NVIDIA CUDA runtime and is ~2GB on its own.
|
||||
#
|
||||
# The GIGABYTES are in the MODEL — ~3.5GB of SigLIP weights — and those are
|
||||
# NOT in this image. They arrive only when the operator enables the lane,
|
||||
# which is what lets rule 164 permit a runtime fetch at all ("optional and
|
||||
# clearly off"). That also settles the trade this step was asked to weigh:
|
||||
# baking the weights in would add ~3.5GB to every pull for a feature many
|
||||
# adopters never enable, against ~350MB for the code that makes the switch
|
||||
# available. Off-by-default wins by an order of magnitude, which was NOT
|
||||
# obvious before measuring — the estimate had the two costs within 15% of
|
||||
# each other.
|
||||
#
|
||||
# `--index-url`, not `--extra-index-url`: the latter would let pip resolve a
|
||||
# +cu wheel anyway, and the whole saving above depends on it not doing that.
|
||||
#
|
||||
# CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the
|
||||
# GPU agent is a separate service with its own image.
|
||||
RUN pip install --index-url https://download.pytorch.org/whl/cpu \
|
||||
"torch>=2.14" "torchvision>=0.29"
|
||||
RUN pip install -r requirements-ml.txt
|
||||
|
||||
# Where the model lands. Deliberately NOT a VOLUME instruction: that mints an
|
||||
# anonymous volume when nobody mounts one, which survives `docker rm` and
|
||||
# accumulates 3.5GB copies nobody can find. The compose files mount it
|
||||
# explicitly instead, so an unmounted run simply re-downloads — visible, and
|
||||
# recoverable.
|
||||
ENV HF_HOME=/models/.huggingface \
|
||||
TRANSFORMERS_CACHE=/models/.huggingface \
|
||||
ML_MODEL_DIR=/models
|
||||
|
||||
COPY backend/ ./backend/
|
||||
COPY alembic/ ./alembic/
|
||||
COPY alembic.ini ./
|
||||
@@ -101,76 +44,7 @@ RUN chmod +x entrypoint.sh
|
||||
|
||||
COPY --from=frontend-builder /build/dist ./frontend/dist
|
||||
|
||||
# Which channel this image belongs to — `dev` or `main` (milestone 271 step 7).
|
||||
# build.yml passes it; /api/extension/manifest reports it beside the version so
|
||||
# an operator can tell which channel an install came from without the channel
|
||||
# ever touching the version string.
|
||||
#
|
||||
# Empty by default, deliberately: a locally-built image then reports NO channel
|
||||
# rather than claiming to be one, and the manifest omits the field entirely —
|
||||
# indistinguishable from an image built before the field existed, which is
|
||||
# exactly the shape every reader already has to handle.
|
||||
#
|
||||
# Declared LAST on purpose. An ARG/ENV invalidates every layer below it, and
|
||||
# these are the values that differ between builds of otherwise identical
|
||||
# source — put them any earlier and the two channels could never share a
|
||||
# cached pip install.
|
||||
#
|
||||
# FC_VERSION is what the instance reports about itself in the UI. Since
|
||||
# milestone 318 stopped publishing version image tags, that self-report is
|
||||
# the only answer to "which build is this?" — nothing else names it.
|
||||
ARG FC_CHANNEL=""
|
||||
ENV FC_CHANNEL=${FC_CHANNEL}
|
||||
ARG FC_VERSION=""
|
||||
ENV FC_VERSION=${FC_VERSION}
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# ONE healthcheck for every role, because the image knows which role it is
|
||||
# running and a deployment should not have to repeat it. `healthcheck` reads
|
||||
# the role entrypoint.sh recorded and asks the right question: HTTP for web,
|
||||
# a self-addressed celery ping for a worker lane, both-for-every-lane for the
|
||||
# consolidated `all`.
|
||||
#
|
||||
# start-period covers the SLOWEST role, which is `all`: alembic, then
|
||||
# hypercorn, then four celery workers registering with the broker. A web-only
|
||||
# container is ready long before this; the cost of the shared number is that
|
||||
# a broken one takes a little longer to be called broken.
|
||||
#
|
||||
# A service may still declare its own healthcheck and docker will prefer it —
|
||||
# the escape hatch for a deployment that wants something different.
|
||||
HEALTHCHECK --interval=30s --timeout=15s --start-period=90s --retries=3 \
|
||||
CMD ["python", "-m", "backend.app.scripts.healthcheck"]
|
||||
|
||||
# tini is PID 1, and the image brings its own rather than asking the
|
||||
# deployment for one.
|
||||
#
|
||||
# PID 1 carries a duty no other process has: every orphaned process in the
|
||||
# container reparents to it and must be reaped, or it stays a zombie holding
|
||||
# a PID slot. This app makes orphans in normal operation — six service
|
||||
# modules shell out (gallery-dl, ffmpeg, pg_dump, the external fetchers) and
|
||||
# celery's prefork pool forks children that spawn them.
|
||||
#
|
||||
# Whatever the role, something that is not an init ends up as PID 1:
|
||||
# supervisord for `all`, hypercorn for `web`, celery for a worker. The fix
|
||||
# was `init: true` in the compose/stack file, which is out of the norm and
|
||||
# put correct process handling in the hands of whoever deploys the image —
|
||||
# the same mistake as declaring the healthcheck per service. A flag that is
|
||||
# silently dropped (an older Swarm, a `docker run` without it) costs reaping
|
||||
# with no signal at all.
|
||||
#
|
||||
# So the image owns it. `docker run <image>` is correct on its own, and
|
||||
# nothing downstream has to know. The smoke asserts /proc/1/comm is tini.
|
||||
ENTRYPOINT ["/usr/bin/tini", "--", "./entrypoint.sh"]
|
||||
# The DEFAULT is the whole application, not one lane of it.
|
||||
#
|
||||
# `docker run fabledcurator` with no command starts hypercorn plus every
|
||||
# worker lane under supervisord — the shape an adopter wants and the shape the
|
||||
# consolidated stack runs. It was `web`, which meant the single-container
|
||||
# layout only worked if you knew to ask for it by name, and a compose file
|
||||
# that forgot `command:` got a web server with nothing processing its queues:
|
||||
# a gallery that loads, accepts an import, and never finishes one.
|
||||
#
|
||||
# The multi-service stack is unaffected — every service there names its role
|
||||
# explicitly, which is exactly what makes it the multi-service stack.
|
||||
CMD ["all"]
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
CMD ["web"]
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
FROM python:3.14-slim
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
HF_HOME=/models/.huggingface \
|
||||
TRANSFORMERS_CACHE=/models/.huggingface \
|
||||
ML_MODEL_DIR=/models
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libpq5 \
|
||||
libjpeg62-turbo \
|
||||
libwebp7 \
|
||||
libpng16-16 \
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements-ml.txt requirements.txt ./
|
||||
# CPU-only torch: the default PyPI wheel bundles the CUDA runtime (~5.6GB
|
||||
# layer); this pipeline never uses a GPU. --index-url (not --extra-index-url)
|
||||
# guarantees only +cpu wheels are considered, so no nvidia-*-cu12 deps.
|
||||
RUN pip install --index-url https://download.pytorch.org/whl/cpu \
|
||||
"torch>=2.12,<3.0" "torchvision>=0.27,<0.28"
|
||||
RUN pip install -r requirements-ml.txt
|
||||
|
||||
COPY backend/ ./backend/
|
||||
COPY alembic/ ./alembic/
|
||||
COPY alembic.ini ./
|
||||
COPY entrypoint.sh ./
|
||||
RUN chmod +x entrypoint.sh
|
||||
|
||||
# Models self-heal into /models on first start (FC-2 implements this)
|
||||
VOLUME ["/models"]
|
||||
|
||||
ENTRYPOINT ["./entrypoint.sh"]
|
||||
CMD ["ml-worker"]
|
||||
@@ -1,661 +0,0 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
@@ -1,337 +1,50 @@
|
||||
<img src="frontend/public/logo.svg" alt="" width="132" align="right" />
|
||||
|
||||
# FabledCurator
|
||||
|
||||
<!-- overview:start -->
|
||||
Self-hosted media curation — a gallery, ML auto-tagging, and subscription-driven
|
||||
downloading in one application. Part of the FabledSword family.
|
||||
Self-hosted media curation — gallery, ML tagging, and subscription-driven downloading in one app. Part of the FabledSword family.
|
||||
|
||||
## What it does
|
||||
Combines what was [ImageRepo](https://git.fabledsword.com/bvandeusen/ImageRepo) (gallery, ML, importer) and [GallerySubscriber](https://git.fabledsword.com/bvandeusen/GallerySubscriber) (gallery-dl wrapper, subscriptions, credential capture) into a single product.
|
||||
|
||||
You point it at creators you follow. It downloads what they post, files it,
|
||||
tags it, and gives you something better than a folder full of images to look
|
||||
through afterwards.
|
||||
## Status
|
||||
|
||||
- **Gallery and browsing.** Images, videos and multi-page works, organised by
|
||||
artist, tag, post and series. A newest-first feed of what just arrived as the
|
||||
front page, a random Showcase, a filterable gallery, a similarity-driven
|
||||
Explore view, and a page-turning reader for series.
|
||||
- **Subscriptions.** Follows creators on Patreon, SubscribeStar, Discord and
|
||||
HentaiFoundry, on a schedule. Handles paywalled posts using your own
|
||||
logged-in session.
|
||||
- **ML tagging.** Runs image models in-container to suggest tags, group
|
||||
characters, find near-duplicates and power similarity search. Suggestions are
|
||||
reviewable — it proposes, you confirm, and it learns which proposals you keep
|
||||
rejecting.
|
||||
- **Deduplication and provenance.** Everything that arrives is hashed and
|
||||
deduplicated by content, metadata sidecars are read wherever the source
|
||||
writes them, and every file keeps a record of where it came from.
|
||||
- **Maintenance.** Backups, library audits, thumbnail and embedding backfills,
|
||||
orphan cleanup — all from the UI, all as background jobs you can watch.
|
||||
Pre-v1. Not yet functional.
|
||||
|
||||
Everything is configured from the Settings UI and stored in the database. There
|
||||
is no config file to edit beyond a handful of bootstrap environment variables.
|
||||
<!-- overview:end -->
|
||||
## Quick start
|
||||
|
||||
## Before you expose it
|
||||
|
||||
**FabledCurator has no login.** There are no user accounts, no passwords and no
|
||||
permission model. Anything that can reach the port is an administrator.
|
||||
|
||||
That matters more here than it would in most self-hosted apps, because of what
|
||||
this one stores: **live platform session cookies for Patreon and
|
||||
SubscribeStar** — accounts that usually have a payment method attached. Whoever reaches
|
||||
the port can read them, alongside your entire library.
|
||||
|
||||
So:
|
||||
|
||||
- Bind it to a LAN, a VPN, or a tunnel you control.
|
||||
- Do not port-forward it. Do not put it on a public hostname.
|
||||
- A reverse proxy that adds TLS but no authentication **does not help**. If you
|
||||
want it reachable from outside, put an authenticating proxy in front of it —
|
||||
a forward-auth provider, HTTP basic auth, an identity-aware tunnel — and treat
|
||||
that layer as the only thing standing between the internet and your accounts.
|
||||
|
||||
This is a deliberate design decision for a single-operator tool on a trusted
|
||||
network, not a bug and not an oversight. It is stated here because it decides
|
||||
how you are allowed to deploy it. [SECURITY.md](SECURITY.md) covers the rest of
|
||||
the threat model.
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Docker** with Compose v2.
|
||||
- **~4 GB RAM** for the app, plus whatever Postgres needs for your library size.
|
||||
- **Disk** for your media, plus several GB for ML model weights.
|
||||
- **No GPU required.** The ML worker runs on CPU — tagging and embedding are
|
||||
slower, and that is the whole difference. A GPU is only involved if you
|
||||
separately run the optional agent (below), which is a different machine's job.
|
||||
|
||||
## Install
|
||||
For local development and testing, just:
|
||||
|
||||
```bash
|
||||
git clone https://git.fabledsword.com/bvandeusen/FabledCurator.git
|
||||
cd FabledCurator
|
||||
|
||||
cp .env.example .env
|
||||
$EDITOR .env # set DB_PASSWORD and SECRET_KEY
|
||||
|
||||
docker compose -f docker-compose.yml up -d
|
||||
docker compose up -d
|
||||
# UI: http://localhost:8080
|
||||
```
|
||||
|
||||
Then open <http://localhost:8080>.
|
||||
That uses sane dev defaults baked into `docker-compose.yml` and the dev
|
||||
override (`docker-compose.override.yml`, auto-merged) — local builds, DEBUG
|
||||
logging, exposed Postgres + Redis ports on the host. No `.env` required.
|
||||
|
||||
**The `-f docker-compose.yml` is required, not decoration.** Compose
|
||||
auto-merges `docker-compose.override.yml` when you leave it off, and that
|
||||
override builds the images locally from source — the contributor path, not
|
||||
yours. Naming the file explicitly skips the override and pulls the published
|
||||
`:latest` images, which is the stable channel built from `main`.
|
||||
|
||||
If you forget it, the symptom is a long build instead of a quick pull.
|
||||
|
||||
## First run
|
||||
|
||||
The database schema is created automatically on first start — the web container
|
||||
runs its migrations before serving. Nothing to initialise by hand.
|
||||
|
||||
**One thing does need a deliberate act, and the app will not start without it.**
|
||||
FabledCurator encrypts your stored platform credentials with a key it keeps at
|
||||
`./images/secrets/credential_key.b64`. On a brand-new install that file does not
|
||||
exist, and rather than quietly creating one the app stops:
|
||||
|
||||
```
|
||||
MissingCredentialKey: Fernet key file not found at /images/secrets/credential_key.b64
|
||||
```
|
||||
|
||||
Set `CURATOR_BOOTSTRAP_NEW_KEY=1` in your `.env` for the first `up`, then delete
|
||||
the line once the container is running. `.env.example` ships it with that
|
||||
instruction attached.
|
||||
|
||||
The refusal is deliberate, and worth understanding rather than working around:
|
||||
auto-creating a key is indistinguishable from the disaster case — a restore that
|
||||
brought the database back but lost `./images/secrets` — where it would mint a key
|
||||
that cannot decrypt anything, leaving an instance that looks healthy while every
|
||||
paywalled download fails. Making you say so once, on an empty install, is the
|
||||
price of that not happening silently later.
|
||||
|
||||
**Which means: back up `./images/secrets/` alongside your database.** It is the
|
||||
only thing that can read your stored credentials. A database restored without it
|
||||
needs every credential entered again by hand.
|
||||
|
||||
A few other things are worth knowing about the first few minutes:
|
||||
|
||||
- **The ML worker downloads its model weights on first boot**, several GB from
|
||||
HuggingFace into `./models`. Until that finishes, tagging is queued rather
|
||||
than broken. It is idempotent — a restart resumes rather than refetches.
|
||||
- **The gallery starts empty**, and that is the expected state. Add a creator
|
||||
under **Subscriptions** and it fills as posts come down.
|
||||
- **If you already have a library on disk**, there is no screen that imports
|
||||
it, and there is not going to be one. Folder ingestion had a UI until July
|
||||
2026; it was retired once posts began arriving entirely through
|
||||
subscriptions and the browser extension, and the decision to leave it
|
||||
retired is deliberate — the folder path carries complexity the product does
|
||||
not need in order to do its job. The supported way to fill a new install is
|
||||
to add the creators you follow under **Subscriptions** and let it pull.
|
||||
|
||||
The `/api/import/trigger` endpoint is still wired up for anyone who wants to
|
||||
script a one-off against a folder mounted at `./import`, and its progress
|
||||
shows under **Settings → Activity**. Treat it as an unsupported escape
|
||||
hatch rather than a feature: nothing in the UI drives it and nothing else
|
||||
in this README depends on it.
|
||||
- **To download from a paywalled account**, FabledCurator needs that account's
|
||||
session — see the browser extension below. Without one it can still fetch
|
||||
public posts.
|
||||
- **Check Settings → Overview** to confirm the workers are alive. Every long
|
||||
operation in FabledCurator is a background job, so if the queues are not
|
||||
running, the UI will look like it is ignoring you rather than like it is
|
||||
broken.
|
||||
|
||||
## The browser extension
|
||||
|
||||
A Firefox extension does two jobs: it hands your logged-in platform sessions to
|
||||
FabledCurator so it can download on your behalf, and it adds a creator as a
|
||||
subscription in one click from their page.
|
||||
|
||||
It ships **inside the web image** — there is no add-on store listing to find.
|
||||
Go to **Subscriptions → Settings**, find the *Browser extension* card, and click
|
||||
**Install Firefox extension**. The XPI is Mozilla-signed, so Firefox installs it
|
||||
like any other add-on; the button serves it directly rather than making you
|
||||
download and side-load a file.
|
||||
|
||||
It pairs with your instance using an API key generated automatically on first
|
||||
use. The bar directly under that card shows the key and can rotate it.
|
||||
|
||||
See [extension/README.md](extension/README.md) for what it does in detail.
|
||||
|
||||
## The GPU agent
|
||||
|
||||
Optional, and separate. If you have a desktop with a graphics card, you can run
|
||||
an agent on it that leases ML jobs from FabledCurator over HTTP, does them on
|
||||
the GPU, and hands the results back. It never touches the database or Redis, so
|
||||
it is safe to run somewhere the rest of the stack is not.
|
||||
|
||||
Run it for a burst of tagging, stop it to get your card back. It deploys from
|
||||
`agent/docker-compose.yml`, not the main stack — see
|
||||
[agent/README.md](agent/README.md).
|
||||
|
||||
## Upgrading
|
||||
For a production-like deployment, override the dev defaults via shell env
|
||||
or a `.env` file (see `.env.example` for the variable names) and use:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml pull
|
||||
docker compose -f docker-compose.yml up -d
|
||||
# (skips the override so containers pull registry images)
|
||||
```
|
||||
|
||||
Migrations run automatically on start. Take a database backup first — Settings →
|
||||
Maintenance has one — because the schema moves forward and does not move back.
|
||||
|
||||
## Deployment posture
|
||||
|
||||
FabledCurator is built to run inside a homelab over plain HTTP. It does not
|
||||
generate certificates, redirect to HTTPS, or set HSTS. If you want TLS,
|
||||
terminate it at your reverse proxy. See [Before you expose it](#before-you-expose-it)
|
||||
for why TLS alone is not enough.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**The UI loads but nothing ever finishes.** The web container is up and the
|
||||
workers are not. `docker compose -f docker-compose.yml ps` — check `worker`,
|
||||
`scheduler` and `ml-worker` are healthy, not restarting.
|
||||
|
||||
**`docker compose up` started building instead of pulling.** You left off
|
||||
`-f docker-compose.yml`, so the dev override took over. See [Install](#install).
|
||||
|
||||
**Downloads fail with an auth error.** The stored session for that platform has
|
||||
expired. Re-capture it with the extension; sessions do not last forever.
|
||||
|
||||
**Which build am I running?** The foot of Settings shows a version and a
|
||||
channel, and `/api/health` returns the same two fields. There are no version
|
||||
tags on the images, so this is the authoritative answer.
|
||||
|
||||
---
|
||||
|
||||
# Developing FabledCurator
|
||||
|
||||
Everything below is about working on FabledCurator rather than running it. If
|
||||
you are installing it, you are done — see [CONTRIBUTING.md](CONTRIBUTING.md) if
|
||||
you want to send a patch.
|
||||
|
||||
## Status and channels
|
||||
|
||||
In production. `main` is continuously deployed — every merge builds and
|
||||
publishes `:latest`, so whatever is on `main` is what is running. Day-to-day
|
||||
work happens on `dev`, which publishes `:dev`.
|
||||
|
||||
For local development, the dev override handles everything:
|
||||
|
||||
```bash
|
||||
docker compose up -d # note: no -f, so the override applies
|
||||
```
|
||||
|
||||
That builds the images from source, turns on DEBUG logging, and exposes
|
||||
Postgres and Redis on the host. No `.env` required.
|
||||
|
||||
## Versions and tags
|
||||
|
||||
Three image tags exist, and no others:
|
||||
|
||||
| Tag | Branch | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `:latest` | `main` | Production. Moves on every merge. |
|
||||
| `:c-<sha>` | `main` | Immutable — the rollback unit, all three images together. |
|
||||
| `:dev` | `dev` | The rolling test channel. Moves on every push. |
|
||||
|
||||
There are deliberately **no version tags**. Nothing pins one, and a per-build
|
||||
name nobody reads is upkeep for a model FC does not run (family rule 145; the
|
||||
reasoning is note #3127 §5). Rolling back is `docker pull …:c-<sha>`.
|
||||
|
||||
Each artifact still has a version, derived rather than chosen: the commit time
|
||||
of the newest change to that artifact's *own* shipped files, as
|
||||
`YYYY.MM.DD.HHMM` UTC (rule 148). Three artifacts, three independent versions
|
||||
— a push touching only `agent/` re-versions the agent and leaves web and the
|
||||
extension alone, and CI skips the builds whose content did not move.
|
||||
|
||||
Because no registry name carries it, the running instance's own report is the
|
||||
only answer to "which build is this?". The foot of Settings shows
|
||||
`FabledCurator 2026.08.29.0201 · dev`, and `/api/health` returns the same two
|
||||
fields.
|
||||
|
||||
Release tags are optional bookmarks — FC went twelve weeks without one and
|
||||
nothing was wrong. Pushing `v<version>` publishes a Forgejo release listing the
|
||||
commits since the previous tag; it builds no image.
|
||||
|
||||
## What's in here
|
||||
|
||||
Four deployable pieces, built by `.forgejo/workflows/build.yml`:
|
||||
|
||||
| Piece | Built from | Image | Role |
|
||||
| --- | --- | --- | --- |
|
||||
| **Web / workers** | `Dockerfile` | `fabledcurator` | Quart API + the built Vue SPA in one image. `entrypoint.sh` picks the role: `web`, `worker`, `scheduler`, `ml-worker`, or `all` (every lane under supervisord, the single-container layout). The `maintenance-long` service is a second `worker` pinned to the long-running maintenance queue. |
|
||||
| **GPU agent** | `agent/Dockerfile` | `fabledcurator-agent` | Optional desktop-GPU worker (`agent/`). Leases jobs over **HTTP only** — never touches the database or Redis. See `agent/README.md`. |
|
||||
| **Firefox extension** | `extension/` | signed XPI | MV3 extension: pushes platform session cookies into FC and adds a creator as a Source in one click. AMO-signed on both `dev` and `main` (one signature per extension change, shared by the two channels), bundled into that channel's web image and served from Settings → Maintenance. See `extension/README.md`. |
|
||||
| **Data** | — | `pgvector/pgvector:pg16`, `redis:7-alpine` | Postgres with pgvector for embeddings; Redis as the Celery broker. |
|
||||
FabledCurator is designed to run inside a self-hosted homelab environment over plain HTTP. If you want TLS, terminate it at your reverse proxy. The app does not generate certificates, redirect to HTTPS, or set HSTS.
|
||||
|
||||
## CI / Forgejo setup
|
||||
|
||||
Two workflows that matter here: `build.yml` (the six verification lanes — lint,
|
||||
extension-version check, backend unit tests, frontend build, extension lint +
|
||||
vitest + XPI content check, integration — and then sign + publish), and
|
||||
`release.yml`, which runs only on a `v*` tag and publishes a changelog without
|
||||
building anything. The extension lane was its own `extension.yml` until
|
||||
milestone 429, which let a red extension suite sign and ship the XPI anyway.
|
||||
The repo's workflows expect:
|
||||
|
||||
**The lanes and the publish are one workflow on purpose.** They were two
|
||||
(`ci.yml` and `build.yml`) until 2026-09-23, on the same push trigger, which
|
||||
meant the build could not see the tests' verdict and published whatever it
|
||||
built — a red unit lane and a fresh `:dev` image, in the same minute. A
|
||||
`needs:` edge only exists inside one workflow graph, so the two are one graph
|
||||
and the gate is that edge: a lane that fails, **or that merely skips**, leaves
|
||||
the publishing jobs unrun. Pull-request runs (Renovate bumps into `dev`) are
|
||||
the lanes and nothing else.
|
||||
|
||||
**The toolchain each job runs in is its `container.image`, not its `runs-on`
|
||||
label.** `runs-on: python-ci` only schedules the job onto a runner; every job
|
||||
then names the image it actually wants. `ci-requirements.md` is the current,
|
||||
authoritative list of images and per-job installs — read that rather than a
|
||||
copy here, so the two can't drift.
|
||||
|
||||
The repo expects one secret:
|
||||
|
||||
- **`RELEASE_TOKEN`** — a Forgejo PAT with:
|
||||
- **Runner label `python-ci`** — a Forgejo runner with Python 3.14, ruff, and Node 22 pre-installed. Both `ci.yml` and `build.yml` use this label. The runner image (`runner-base:python-ci`) is built from `CI-Runner/CI-python/` in the operator's workspace; `make push` from that directory builds and pushes a new image when toolchain pins change.
|
||||
- **Repo secret `RELEASE_TOKEN`** — a Forgejo PAT with the following scopes:
|
||||
- `write:package` + `read:package` — for `docker push` to `git.fabledsword.com`
|
||||
- `write:release` — for the `ext-<version>` releases that cache the signed XPI
|
||||
- `write:issue` — for issue-management automation
|
||||
- `write:release` — for future release-cutting workflows
|
||||
- `write:issue` — for future issue-management automation
|
||||
|
||||
Generate at https://git.fabledsword.com/user/settings/applications. The injected `GITHUB_TOKEN` cannot be used because it lacks `write:package`.
|
||||
|
||||
AMO signing additionally needs `MOZILLA_AMO_JWT_KEY` / `MOZILLA_AMO_JWT_SECRET`.
|
||||
It runs on **both** channels and is cached per version: because the version is
|
||||
derived from commit time, `dev` and `main` derive the same number for the same
|
||||
source, so `main` finds `dev`'s signature already cached and makes no second AMO
|
||||
call. That cache is why signing must be one-shot — AMO rejects a re-signed
|
||||
version.
|
||||
|
||||
## History
|
||||
|
||||
FabledCurator combines what was
|
||||
[ImageRepo](https://git.fabledsword.com/bvandeusen/ImageRepo) (gallery, ML,
|
||||
importer) and
|
||||
[GallerySubscriber](https://git.fabledsword.com/bvandeusen/GallerySubscriber)
|
||||
(gallery-dl wrapper, subscriptions, credential capture) into a single product.
|
||||
Both are superseded; neither is maintained.
|
||||
|
||||
## License
|
||||
|
||||
**GNU Affero General Public License v3.0** — see [LICENSE](LICENSE).
|
||||
|
||||
You may run, study, modify and redistribute this software. The condition is
|
||||
reciprocity: if you distribute a modified version, or **run one as a network
|
||||
service that other people use**, you must offer those users the corresponding
|
||||
source under the same licence. That second clause (AGPL §13) is the reason this
|
||||
licence rather than the GPL — for a self-hosted web application, "distribution"
|
||||
otherwise never happens, and the obligation would never bite.
|
||||
|
||||
Running an unmodified copy for yourself, your household or your organisation
|
||||
carries no obligation at all. Neither does modifying it privately. The licence
|
||||
asks something of you only when you hand your modified version to others.
|
||||
|
||||
Contributions ship under the same licence — see [CONTRIBUTING](CONTRIBUTING.md).
|
||||
Security reports: [SECURITY.md](SECURITY.md).
|
||||
Personal project; use at your own discretion.
|
||||
|
||||
-77
@@ -1,77 +0,0 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a vulnerability
|
||||
|
||||
**Please do not put vulnerability details in a public issue.**
|
||||
|
||||
This project has no private disclosure channel yet. Until it does, open an
|
||||
issue on the repository that says only that you have a security report — no
|
||||
reproduction steps, no affected endpoint, no payload — and a maintainer will
|
||||
reply with a private contact to send the details to.
|
||||
|
||||
That is a deliberately awkward first step, and it exists because the
|
||||
alternative is worse: an issue tracker is public the moment it is written to,
|
||||
and every self-hosted instance stays vulnerable until its operator has had a
|
||||
chance to update.
|
||||
|
||||
Please include, once you have a private channel:
|
||||
|
||||
- what an attacker can do, and what access they need to start
|
||||
- the version or commit you tested
|
||||
- reproduction steps
|
||||
|
||||
## Scope — what this software actually handles
|
||||
|
||||
FabledCurator is self-hosted and holds things worth stating plainly, because
|
||||
they shape what counts as a serious bug here:
|
||||
|
||||
- **Platform credentials.** The app captures and stores session cookies for
|
||||
third-party subscription sites (Patreon, SubscribeStar) so it can
|
||||
download on the operator's behalf. These are live credentials for accounts
|
||||
that usually carry a payment method. Anything that discloses them, decrypts
|
||||
them, or lets one user of a shared instance read another's is high severity.
|
||||
- **An extension API key.** The Firefox extension authenticates to the backend
|
||||
with a shared key. Anything that leaks it or lets it be bypassed is a way in.
|
||||
- **No authentication of its own.** This is the most important thing on this
|
||||
page. FabledCurator has no login, no user accounts and no permission model —
|
||||
there is no `User` table and no session auth anywhere in the backend. Every
|
||||
HTTP client that can reach the port is the administrator, with full read and
|
||||
write access to everything above, including the stored platform credentials.
|
||||
Access control is entirely the operator's job, done at the network layer.
|
||||
Reports that an unauthenticated caller can reach an endpoint are therefore
|
||||
describing the design; reports that something *crosses the network boundary
|
||||
the operator drew* — an SSRF, a request forgery that rides a browser the
|
||||
operator already has open, a path that leaks state to an origin the operator
|
||||
did not authorise — are in scope and are serious.
|
||||
- **Arbitrary media from the internet.** Downloaded files are decoded, hashed,
|
||||
thumbnailed and fed to ML models. Anything that turns a hostile file into
|
||||
code execution is in scope.
|
||||
|
||||
## Deployment posture — read this before reporting
|
||||
|
||||
FabledCurator is designed to run **inside a private network, over plain HTTP,
|
||||
reachable only by its operator**. It does not terminate TLS, redirect to
|
||||
HTTPS, or set HSTS; if you want transport security, terminate it at your
|
||||
reverse proxy. It also does not authenticate anyone — see above. These are
|
||||
documented design decisions, not oversights.
|
||||
|
||||
Putting this on the public internet, with or without TLS, hands whoever finds
|
||||
it your Patreon and SubscribeStar sessions. A reverse proxy that adds
|
||||
TLS but not an authentication layer does not change that.
|
||||
|
||||
Reports that reduce to "the application is served over HTTP", "there is no
|
||||
HSTS header", or "the API needs no credentials" describe those decisions
|
||||
rather than vulnerabilities. Reports that the operator can cause the software
|
||||
to do something destructive are usually also by design — the operator is the
|
||||
administrator of their own instance.
|
||||
|
||||
What remains in scope is everything that crosses a boundary the software is
|
||||
actually supposed to hold: between untrusted downloaded content and the host,
|
||||
between a third-party origin and an operator's open browser session, and
|
||||
between the credentials at rest and anything that is not the operator.
|
||||
|
||||
## Supported versions
|
||||
|
||||
Fixes land on the `main` branch and reach the `:latest` image. There are no
|
||||
maintained release branches — the supported version is the current one, and
|
||||
the remedy for a security issue is to update.
|
||||
@@ -1,63 +0,0 @@
|
||||
# FabledCurator GPU agent — runs on the desktop with the GPU.
|
||||
#
|
||||
# The `base` flavour, not `cudnn-runtime`: CUDA and cuDNN arrive as the
|
||||
# `nvidia-*` pip packages torch and onnxruntime-gpu depend on, so the base only
|
||||
# has to hand the container the driver (it sets NVIDIA_VISIBLE_DEVICES /
|
||||
# NVIDIA_DRIVER_CAPABILITIES for the Container Toolkit). Until #1451 this was
|
||||
# `12.9.2-cudnn-runtime` under a `torch==2.6.0+cu124` — and requirements.txt then
|
||||
# REPLACED that torch with PyPI's CUDA-13 build (ultralytics pulls torchvision,
|
||||
# which pulls its matching torch), beside a CUDA-13 onnxruntime-gpu. The image
|
||||
# ran CUDA 13 on a CUDA-12 base, carrying ~3 GB of base libraries and a ~3 GB
|
||||
# torch nothing loaded: 10 GB compressed.
|
||||
#
|
||||
# 13.0 because that is the line both wheels are built for (torch's cu130 index,
|
||||
# onnxruntime-gpu's `nvidia-cuda-runtime~=13.0`). Needs an NVIDIA driver that
|
||||
# supports CUDA 13 (580+); fc_agent/accel.py logs at startup whether torch and
|
||||
# onnxruntime actually got the GPU, since both fall back to the CPU silently.
|
||||
# ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
|
||||
FROM nvidia/cuda:13.0.3-base-ubuntu24.04
|
||||
|
||||
# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
|
||||
# managed (PEP 668), so a global `pip install` errors without this. It's a
|
||||
# single-purpose container — we own the whole environment, so installing into
|
||||
# the system site-packages is fine (and simplest — no venv on PATH to manage).
|
||||
ENV DEBIAN_FRONTEND=noninteractive PYTHONUNBUFFERED=1 PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends python3 python3-pip ffmpeg \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
# torch AND torchvision from the cu130 index, together and first. Installing
|
||||
# torch alone is what let the next step swap it out: ultralytics needs
|
||||
# torchvision, PyPI's torchvision pins its own torch, and pip replaced ours to
|
||||
# match. With both present, requirements.txt finds them satisfied.
|
||||
RUN pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu130 \
|
||||
torch torchvision
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install --no-cache-dir -r requirements.txt
|
||||
COPY fc_agent ./fc_agent
|
||||
|
||||
# imgutils ONNX models + the transformers SigLIP weights both cache here; mount
|
||||
# a volume to persist them across restarts (the SigLIP download is ~3.5 GB once).
|
||||
ENV HF_HOME=/models
|
||||
|
||||
# Declared LAST on purpose, exactly as the web Dockerfile does: an ARG/ENV
|
||||
# invalidates every layer below it, and these are the only values that differ
|
||||
# between builds of otherwise identical source. Any earlier and the ~6.3 GB
|
||||
# CUDA + torch layers could never be shared between the dev and main builds of
|
||||
# one commit — which is the cost #3114 measured at 9m26s cold.
|
||||
#
|
||||
# Three values, never folded together (rule 149) — the NAME a person reads, the
|
||||
# CHANNEL it came from, and the REVISION that identifies the content. See
|
||||
# fc_agent/build_info.py; CI derives all three from scripts/artifacts.sh.
|
||||
ARG FC_CHANNEL=""
|
||||
ENV FC_CHANNEL=${FC_CHANNEL}
|
||||
ARG FC_VERSION=""
|
||||
ENV FC_VERSION=${FC_VERSION}
|
||||
ARG FC_REVISION=""
|
||||
ENV FC_REVISION=${FC_REVISION}
|
||||
|
||||
EXPOSE 8770
|
||||
|
||||
# The control UI; the worker is started from it (or POST /start).
|
||||
CMD ["uvicorn", "fc_agent.app:app", "--host", "0.0.0.0", "--port", "8770"]
|
||||
@@ -1,86 +0,0 @@
|
||||
# FabledCurator GPU agent
|
||||
|
||||
A desktop-GPU worker that embeds characters (CCIP) + figure crops for
|
||||
FabledCurator. It talks to FC **only over HTTP** — it leases jobs, fetches image
|
||||
pixels, runs the models on your GPU, and posts results back. Your FC database and
|
||||
Redis stay private; the agent never touches them.
|
||||
|
||||
You run it when you want a burst and stop it to reclaim the card.
|
||||
|
||||
## 0. Host prerequisite — NVIDIA Container Toolkit
|
||||
Docker needs the toolkit to hand the GPU to a container (else: *"could not select
|
||||
device driver nvidia with capabilities [[gpu]]"*). On Arch/CachyOS:
|
||||
```sh
|
||||
sudo pacman -S nvidia-container-toolkit
|
||||
sudo nvidia-ctk runtime configure --runtime=docker
|
||||
sudo systemctl restart docker
|
||||
# verify:
|
||||
docker run --rm --gpus all nvidia/cuda:13.0.3-base-ubuntu24.04 nvidia-smi
|
||||
# the header's CUDA version must be 13.0 or later (driver 580+)
|
||||
```
|
||||
|
||||
### After a driver update: regenerate the CDI spec
|
||||
If the agent's first log lines say `accel: torch is NOT on the GPU` or report
|
||||
`cudaGetDeviceCount: unknown error (999)` while `nvidia-smi` still works, the
|
||||
toolkit's saved device list (`/etc/cdi/nvidia.yaml`) is out of date. The
|
||||
`nvidia-uvm` device number changes between driver versions, and a spec
|
||||
generated before the update hands the container a device node that no longer
|
||||
exists (2026-09-24: host `511,0`, container `235,0`). Compare
|
||||
`ls -l /dev/nvidia-uvm` on the host with the same inside the container, then:
|
||||
```sh
|
||||
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
|
||||
# if your toolkit ships it, this keeps it current on every driver update:
|
||||
sudo systemctl enable --now nvidia-cdi-refresh.path
|
||||
```
|
||||
|
||||
## 1. Get a token
|
||||
In FC: **Settings → Tagging → GPU agent → Generate token** (or Rotate). Copy it.
|
||||
|
||||
## 2. Pull (CI publishes it alongside the web image)
|
||||
```sh
|
||||
docker pull git.fabledsword.com/bvandeusen/fabledcurator-agent:latest
|
||||
```
|
||||
> Local build for development instead: `docker build -t fc-gpu-agent agent/`
|
||||
|
||||
## 3. Run (on the machine with the GPU)
|
||||
```sh
|
||||
docker run --rm --gpus all -p 8770:8770 \
|
||||
-e FC_URL=http://curator.traefik.internal \
|
||||
-e FC_TOKEN=<paste-the-token> \
|
||||
-v fc-agent-models:/models \
|
||||
git.fabledsword.com/bvandeusen/fabledcurator-agent:latest
|
||||
```
|
||||
Then open <http://localhost:8770> — the control page. Click **Start** to begin
|
||||
draining the queue; **Pause**/**Stop** to yield the GPU. The `-v fc-agent-models`
|
||||
volume caches the downloaded ONNX models so restarts are fast.
|
||||
|
||||
Kick off a backfill from FC (**GPU agent card → Queue character embedding**), then
|
||||
watch the queue counts on the control page (or FC's card) drain.
|
||||
|
||||
## Config (env)
|
||||
| var | default | meaning |
|
||||
|---|---|---|
|
||||
| `FC_URL` | `http://localhost:8000` | FC base URL |
|
||||
| `FC_TOKEN` | — | the bearer token (required) |
|
||||
| `AGENT_ID` | `desktop-agent` | identifies this agent's leases |
|
||||
| `BATCH_SIZE` | `4` | jobs leased per round (still processed one at a time) |
|
||||
| `CCIP_MODEL` | imgutils default | CCIP model name |
|
||||
| `DETECTOR_LEVEL` | `m` | person-detector size: `n` < `s` < `m` < `x` |
|
||||
| `POLL_IDLE_SECONDS` | `10` | wait between empty leases |
|
||||
|
||||
## ⚠️ Verify on first run
|
||||
This part can't be CI-tested (no GPU/models in CI), so confirm against your
|
||||
installed `dghs-imgutils` (`pip show dghs-imgutils`) — see `fc_agent/models.py`:
|
||||
- `imgutils.detect.detect_person(image, level=...)` returns
|
||||
`[((x0,y0,x1,y1), label, score), ...]`.
|
||||
- `imgutils.metrics.ccip_extract_feature(image, model=...)` returns a vector
|
||||
(768-d for caformer). If you want the F1-0.94 variant, set
|
||||
`CCIP_MODEL=ccip-caformer_b36-24` (verify the exact string in imgutils).
|
||||
|
||||
If FC's matcher under/over-fires, tune the cosine threshold in
|
||||
`backend/app/services/ml/ccip.py` (`DEFAULT_SIM_THRESHOLD`) and use
|
||||
`GET /api/ccip/overview` + `/api/ccip/images/<id>` to spot-check.
|
||||
|
||||
## CPU fallback
|
||||
Swap `onnxruntime-gpu` → `onnxruntime` in `requirements.txt` and drop `--gpus all`
|
||||
to grind it slowly on the server instead. Same agent, no card.
|
||||
@@ -1,73 +0,0 @@
|
||||
# FabledCurator GPU agent — desktop run via docker compose.
|
||||
#
|
||||
# Usage:
|
||||
# 1. Generate a token: FC → Settings → Tagging → GPU agent → Generate token.
|
||||
# 2. Create a .env next to this file:
|
||||
# FC_URL=http://curator.traefik.internal
|
||||
# FC_TOKEN=<paste-the-token>
|
||||
# # optional: CCIP_MODEL=ccip-caformer_b36-24 (the F1-0.94 variant)
|
||||
# 3. docker compose up -d (pulls the published image)
|
||||
# 4. Open http://localhost:8770 → Start. Pause/Stop hands the GPU back.
|
||||
# docker compose down to stop the container entirely.
|
||||
#
|
||||
# Surviving a curator redeploy (you're away, can't touch the agent):
|
||||
# - A running agent rides out curator being unreachable on its own — it retries
|
||||
# leasing with capped backoff and resumes when the server is back. In-flight
|
||||
# work is handed back (not failed), so a redeploy never poisons good jobs.
|
||||
# - AUTO_START=1 (below) also resumes the worker if the AGENT container itself
|
||||
# restarts (host reboot / crash via `restart: unless-stopped`) — no click.
|
||||
#
|
||||
# Needs the NVIDIA Container Toolkit installed on the host for --gpus.
|
||||
|
||||
services:
|
||||
fc-gpu-agent:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator-agent:latest
|
||||
pull_policy: always
|
||||
ports:
|
||||
- "8770:8770"
|
||||
environment:
|
||||
FC_URL: ${FC_URL:-http://curator.traefik.internal}
|
||||
FC_TOKEN: ${FC_TOKEN:?set FC_TOKEN in .env (FC → GPU agent → Generate token)}
|
||||
CCIP_MODEL: ${CCIP_MODEL:-}
|
||||
DETECTOR_LEVEL: ${DETECTOR_LEVEL:-m}
|
||||
BATCH_SIZE: ${BATCH_SIZE:-4}
|
||||
# Resume the worker automatically on container start (survive a reboot /
|
||||
# crash-restart while you're away). Set to 0 to require a manual Start.
|
||||
AUTO_START: ${AUTO_START:-1}
|
||||
# Autoscale the worker count (throughput hill-climb that finds the sweet
|
||||
# spot + backs off under VRAM pressure). On by default; toggle live in the
|
||||
# control UI. Set to 0 to start in manual mode.
|
||||
AUTO_SCALE: ${AUTO_SCALE:-1}
|
||||
# Aggregate download cap in MB/s (stills + video streams combined), so the
|
||||
# agent can't saturate the desktop's network and wreck browsing — WiFi
|
||||
# especially. 0 = unlimited; tunable live in the control UI.
|
||||
BANDWIDTH_LIMIT_MB_S: ${BANDWIDTH_LIMIT_MB_S:-8}
|
||||
# Crop embedder (SigLIP concept bag): float16 keeps VRAM low on a shared
|
||||
# desktop GPU; the model itself is announced by the server.
|
||||
SIGLIP_DTYPE: ${SIGLIP_DTYPE:-float16}
|
||||
# Crop PROPOSERS (extra YOLO detectors → more/better concept crops). Each
|
||||
# downloads its weights once (cached on the models volume) and self-disables
|
||||
# if the download/load fails. Blank any one to turn it off.
|
||||
# PERSON_WEIGHTS: general COCO person detector (Western/realistic figures),
|
||||
# merged with the anime detector. yolo11n.pt (~6 MB, auto-downloaded).
|
||||
# ANATOMY_WEIGHTS: booru_yolo anime/furry/NSFW components (~40 MB). NB the
|
||||
# repo states no license — fine for private use. yolov8n_as01.pt is the
|
||||
# 6 MB nano if you want lighter than yolov11m_aa22.pt.
|
||||
# PANEL_WEIGHTS: mosesb comic-panel detector (Apache-2.0), "hf_repo::file".
|
||||
PERSON_WEIGHTS: ${PERSON_WEIGHTS:-yolo11n.pt}
|
||||
ANATOMY_WEIGHTS: ${ANATOMY_WEIGHTS:-https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt}
|
||||
PANEL_WEIGHTS: ${PANEL_WEIGHTS:-mosesb/best-comic-panel-detection::best.pt}
|
||||
volumes:
|
||||
# Persist the downloaded ONNX models so restarts are fast.
|
||||
- fc-agent-models:/models
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
volumes:
|
||||
fc-agent-models:
|
||||
@@ -1,124 +0,0 @@
|
||||
"""Which accelerator each runtime actually got — reported once, at startup.
|
||||
|
||||
The agent has two GPU runtimes and both fall back to the CPU without raising:
|
||||
torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils
|
||||
detector + CCIP models) when its CUDA provider cannot load its libraries. A
|
||||
fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the
|
||||
image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base
|
||||
(#1451), and whether the ONNX half was on the GPU could not be answered from
|
||||
anything the agent had ever logged.
|
||||
|
||||
Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA
|
||||
provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image
|
||||
they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT
|
||||
1.21+) loads them from there, so the provider resolves them by soname.
|
||||
|
||||
Stdlib-only at import, so the unit suite can import it — torch and onnxruntime
|
||||
are imported inside the functions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import importlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("fc_agent.accel")
|
||||
|
||||
# Filled by report(); /status carries it so the page can show it too.
|
||||
LAST: dict = {}
|
||||
|
||||
|
||||
def torch_status(imp=importlib.import_module) -> dict:
|
||||
try:
|
||||
torch = imp("torch")
|
||||
except Exception as e:
|
||||
return {"device": "unavailable", "error": str(e)}
|
||||
out = {"version": torch.__version__, "cuda_build": torch.version.cuda}
|
||||
if torch.cuda.is_available():
|
||||
out["device"] = "cuda"
|
||||
out["gpu"] = torch.cuda.get_device_name(0)
|
||||
else:
|
||||
out["device"] = "cpu"
|
||||
return out
|
||||
|
||||
|
||||
def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict:
|
||||
try:
|
||||
ort = imp("onnxruntime")
|
||||
except Exception as e:
|
||||
return {"device": "unavailable", "error": str(e)}
|
||||
out = {"version": ort.__version__, "providers": list(ort.get_available_providers())}
|
||||
if "CUDAExecutionProvider" not in out["providers"]:
|
||||
out["device"] = "cpu"
|
||||
return out
|
||||
preload = getattr(ort, "preload_dlls", None)
|
||||
if preload is not None:
|
||||
try:
|
||||
preload()
|
||||
except Exception as e:
|
||||
out["preload_error"] = str(e)
|
||||
# "Available" only means the build HAS the provider. Loading its library is
|
||||
# what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are
|
||||
# missing, and the one a session would otherwise fail silently on.
|
||||
capi = Path(ort.__file__).parent / "capi"
|
||||
try:
|
||||
load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL)
|
||||
load(str(capi / "libonnxruntime_providers_cuda.so"))
|
||||
except OSError as e:
|
||||
out["device"] = "cpu"
|
||||
out["error"] = str(e)
|
||||
return out
|
||||
# Loading proves the libraries resolve, NOT that a GPU can be used: on
|
||||
# 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with
|
||||
# "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA
|
||||
# runtime for a device initialises the driver the provider would use.
|
||||
error = _cuda_device_error(load)
|
||||
out["device"] = "cpu" if error else "cuda"
|
||||
if error:
|
||||
out["error"] = error
|
||||
return out
|
||||
|
||||
|
||||
def _cuda_device_error(load=ctypes.CDLL) -> str | None:
|
||||
"""None when the CUDA runtime can reach a device, else why it cannot."""
|
||||
try:
|
||||
cudart = load("libcudart.so.13")
|
||||
except OSError as e:
|
||||
return str(e)
|
||||
count = ctypes.c_int(0)
|
||||
rc = cudart.cudaGetDeviceCount(ctypes.byref(count))
|
||||
if rc != 0:
|
||||
cudart.cudaGetErrorString.restype = ctypes.c_char_p
|
||||
return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})"
|
||||
return None if count.value > 0 else "no CUDA device visible"
|
||||
|
||||
|
||||
def summary() -> dict | None:
|
||||
"""The report as FabledCurator stores it: each runtime's device, and why
|
||||
when it is not the GPU. Sent on every lease and heartbeat, so the System
|
||||
view can call a running agent that fell back to the CPU "degraded" rather
|
||||
than "running" — the 2026-09-24 fallback went unseen for weeks because
|
||||
only this agent's own log said so. None before report() has run."""
|
||||
if not LAST:
|
||||
return None
|
||||
out = {}
|
||||
for name, s in LAST.items():
|
||||
entry = {"device": s.get("device")}
|
||||
if s.get("error"):
|
||||
entry["error"] = str(s["error"])[:200]
|
||||
out[name] = entry
|
||||
return out
|
||||
|
||||
|
||||
def report() -> dict:
|
||||
"""Check both runtimes, log the result, and keep it for /status."""
|
||||
LAST.clear()
|
||||
LAST.update(torch=torch_status(), onnx=onnx_status())
|
||||
for name, s in LAST.items():
|
||||
if s.get("device") == "cuda":
|
||||
log.info("accel: %s on GPU (%s)", name, s)
|
||||
else:
|
||||
log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s)
|
||||
return dict(LAST)
|
||||
@@ -1,451 +0,0 @@
|
||||
"""FastAPI control surface for the agent (served on localhost).
|
||||
|
||||
Start / stop the download→GPU pipeline, tune the downloader count live (the
|
||||
workload is download-bound, so downloaders are the dial that trades desktop
|
||||
bandwidth for throughput), and watch GPU load + buffer occupancy + progress +
|
||||
the server-side queue. Config is env-seeded; the downloader count is adjustable
|
||||
here on the fly (GPU consumers autoscale between 1 and 2 on their own).
|
||||
"""
|
||||
import logging
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from . import accel, logbuf
|
||||
from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version
|
||||
from .config import Config
|
||||
from .gpu import read_gpu
|
||||
from .worker import Worker
|
||||
|
||||
log = logging.getLogger("fc_agent.app")
|
||||
|
||||
# DERIVED at image build time, not hand-maintained — see build_info. This was a
|
||||
# literal an author was asked to bump on every agent change, and the September
|
||||
# image printed the same "2026-07-17.1" as the July one, so the surface meant to
|
||||
# answer "did my pull work?" answered the same either way.
|
||||
#
|
||||
# Two values with two jobs, kept apart (rule 149): the page SHOWS the version
|
||||
# and COMPARES the build id. /status reports both, plus the raw fields, so a
|
||||
# reader never has to take a formatted string apart to get at one of them.
|
||||
|
||||
logbuf.install()
|
||||
cfg = Config.from_env()
|
||||
worker = Worker(cfg)
|
||||
app = FastAPI(title="FabledCurator GPU agent")
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def _no_store(request, call_next):
|
||||
# The control page is a static string and the status/gpu/logs polls are
|
||||
# live data — never let the browser cache either, or a freshly-pulled agent
|
||||
# image still shows the OLD UI until a hard refresh (operator-flagged
|
||||
# 2026-06-30).
|
||||
resp = await call_next(request)
|
||||
resp.headers["Cache-Control"] = "no-store"
|
||||
return resp
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def _maybe_autostart() -> None:
|
||||
# Before the worker: the report also preloads the CUDA libraries the ONNX
|
||||
# models need, and it says in the log which runtimes landed on the GPU.
|
||||
accel.report()
|
||||
# With AUTO_START set, a container restart (host reboot, or `restart:
|
||||
# unless-stopped` after a crash) resumes the worker on its own — the slots
|
||||
# then ride out a still-down curator via lease backoff. Lets the agent
|
||||
# survive a redeploy with nobody at the desktop to click Start.
|
||||
if cfg.auto_start and cfg.token:
|
||||
worker.start()
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def index() -> str:
|
||||
# Two substitutions, not one: `__VERSION__` is what a person reads in the
|
||||
# meta line, `__BUILD_ID__` is what the script compares against /status to
|
||||
# notice the page is a cached copy from a previous build.
|
||||
return (
|
||||
_PAGE
|
||||
.replace("__VERSION__", display_version())
|
||||
.replace("__BUILD_ID__", build_id())
|
||||
)
|
||||
|
||||
|
||||
@app.post("/start")
|
||||
def start():
|
||||
log.info("UI: Start button pressed") # the press; worker logs the transition
|
||||
worker.start()
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/stop")
|
||||
def stop():
|
||||
log.info("UI: Stop button pressed")
|
||||
worker.stop()
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/concurrency")
|
||||
async def concurrency(request: Request):
|
||||
body = await request.json()
|
||||
worker.set_concurrency(int(body.get("value", 1)))
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/auto")
|
||||
async def auto(request: Request):
|
||||
body = await request.json()
|
||||
worker.set_auto(bool(body.get("value", True)))
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.post("/bandwidth")
|
||||
async def bandwidth(request: Request):
|
||||
body = await request.json()
|
||||
worker.set_bandwidth(float(body.get("value", 0)))
|
||||
return JSONResponse(worker.status())
|
||||
|
||||
|
||||
@app.get("/gpu")
|
||||
def gpu():
|
||||
# GPU meters poll this on their own fast cadence. It only reads local
|
||||
# nvidia-smi — no curator round-trip — so the util/VRAM bars stay live even
|
||||
# when /status is slow waiting on the (sometimes busy) curator queue call.
|
||||
g = read_gpu() or {}
|
||||
us = worker.util_smooth()
|
||||
if us is not None:
|
||||
g["util_smooth"] = round(us, 1) # autoscaler's EWMA — the UI bar tracks this
|
||||
return JSONResponse(g)
|
||||
|
||||
|
||||
@app.get("/logs")
|
||||
def logs():
|
||||
return JSONResponse({"lines": list(logbuf.LINES)})
|
||||
|
||||
|
||||
@app.get("/status")
|
||||
def status():
|
||||
# Pure in-memory read: worker.status() is lock-free and the queue snapshot is
|
||||
# kept fresh by a background poller — NO inline curator call, so this can't
|
||||
# stall the status view when curator is buried under a big backlog.
|
||||
worker.note_ui() # a browser is watching → keep the queue snapshot warm
|
||||
s = worker.status()
|
||||
s["fc_url"] = cfg.fc_url
|
||||
s["configured"] = bool(cfg.token)
|
||||
s["queue"] = worker.latest_queue()
|
||||
# `build` is the comparison token the page checks — see build_info.
|
||||
# `version`/`channel`/`revision` ride BESIDE it rather than inside it, so a
|
||||
# reader wanting the version never has to parse it back out of something
|
||||
# else. Absent rather than empty when the image carries no stamp.
|
||||
s["build"] = build_id()
|
||||
s["version"] = FC_VERSION or None
|
||||
s["channel"] = FC_CHANNEL or None
|
||||
s["revision"] = FC_REVISION or None
|
||||
s["accel"] = accel.LAST or None
|
||||
return JSONResponse(s)
|
||||
|
||||
|
||||
_PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
<meta name=viewport content="width=device-width,initial-scale=1">
|
||||
<title>FabledCurator · GPU agent</title>
|
||||
<style>
|
||||
:root{--bg:#0f1216;--panel:#181c22;--panel2:#1e232b;--bd:#2a313b;--fg:#e9edf2;
|
||||
--mut:#8b97a6;--acc:#e8923a;--grn:#46c46a;--red:#e8584d;--amb:#e8b23a}
|
||||
*{box-sizing:border-box}
|
||||
body{font:14px/1.5 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;margin:0;
|
||||
background:radial-gradient(1200px 600px at 50% -10%,#1a2029,#0f1216);color:var(--fg)}
|
||||
.wrap{max-width:820px;margin:0 auto;padding:28px 20px 28px;height:100vh;
|
||||
box-sizing:border-box;overflow:hidden;display:flex;flex-direction:column}
|
||||
header{display:flex;align-items:center;justify-content:space-between;margin-bottom:4px}
|
||||
.brand{display:flex;align-items:center;gap:10px;font-size:19px;font-weight:700;letter-spacing:.2px}
|
||||
.logo{color:var(--acc);font-size:20px}
|
||||
.brand .sub{color:var(--mut);font-weight:600;font-size:13px;text-transform:uppercase;letter-spacing:.12em}
|
||||
.conn{display:flex;align-items:center;gap:8px;color:var(--mut);font-size:13px;font-weight:600}
|
||||
.dot{width:9px;height:9px;border-radius:50%;background:var(--mut);box-shadow:0 0 0 0 rgba(0,0,0,0)}
|
||||
.dot.green{background:var(--grn);box-shadow:0 0 10px 1px rgba(70,196,106,.5)}
|
||||
.dot.amber{background:var(--amb)} .dot.red{background:var(--red)}
|
||||
.meta{color:var(--mut);margin:0 0 18px;font-size:13px}
|
||||
code{background:#11151a;border:1px solid var(--bd);padding:2px 7px;border-radius:6px;
|
||||
font:12px ui-monospace,SFMono-Regular,Menlo,monospace;color:#cdd6e0}
|
||||
.card{background:linear-gradient(180deg,var(--panel),var(--panel2));border:1px solid var(--bd);
|
||||
border-radius:14px;padding:16px 18px;margin-bottom:14px;box-shadow:0 1px 0 rgba(255,255,255,.02) inset}
|
||||
.card-h{font-size:11px;font-weight:800;letter-spacing:.12em;text-transform:uppercase;
|
||||
color:var(--mut);margin-bottom:14px}
|
||||
.controls{display:flex;align-items:center;gap:10px;flex-wrap:wrap}
|
||||
.spacer{flex:1}
|
||||
.btn{font:600 14px system-ui;padding:.5rem 1rem;border:1px solid transparent;border-radius:9px;
|
||||
cursor:pointer;color:#fff;transition:.12s}
|
||||
.btn:hover{transform:translateY(-1px)}
|
||||
.btn[disabled]{opacity:.45;pointer-events:none;transform:none}
|
||||
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
|
||||
.tile .n.busy{color:var(--acc);animation:pulse 1s ease-in-out infinite}
|
||||
.btn.start{background:linear-gradient(180deg,#2f9c4c,#247a3c)}
|
||||
.btn.stop{background:linear-gradient(180deg,#3a3f48,#2a2f37);color:#e9edf2;border-color:var(--bd)}
|
||||
.switch{display:inline-flex;align-items:center;gap:8px;cursor:pointer;font-weight:600;user-select:none}
|
||||
.switch input{display:none}
|
||||
.switch .track{width:38px;height:22px;border-radius:11px;background:#2a313b;position:relative;transition:.15s}
|
||||
.switch .track:after{content:"";position:absolute;top:2px;left:2px;width:18px;height:18px;border-radius:50%;
|
||||
background:#cdd6e0;transition:.15s}
|
||||
.switch input:checked+.track{background:var(--acc)}
|
||||
.switch input:checked+.track:after{transform:translateX(16px);background:#fff}
|
||||
.stepper{display:inline-flex;align-items:center;gap:6px}
|
||||
.step{background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:8px;
|
||||
width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
|
||||
.step:hover{border-color:var(--acc)}
|
||||
#conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
|
||||
color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
|
||||
/* The browser's own spin arrows, hidden: the − / + beside each field are the
|
||||
control, styled like the rest of the page (operator, 2026-09-24). */
|
||||
#conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
|
||||
#bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
|
||||
.unit{color:var(--mut);font-size:12px;font-weight:600}
|
||||
.hint{color:var(--mut);font-size:12px;margin-top:12px}
|
||||
.tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
|
||||
.tile{background:#13171d;border:1px solid var(--bd);border-radius:10px;padding:12px 8px;text-align:center}
|
||||
.tile .n{font:800 22px ui-monospace,monospace;line-height:1.1}
|
||||
.tile .n.warn{color:var(--red)} .tile .n.ok{color:var(--grn)}
|
||||
.tile .l{font-size:10px;text-transform:uppercase;letter-spacing:.06em;color:var(--mut);margin-top:4px}
|
||||
.meters{display:flex;flex-direction:column;gap:10px;margin-bottom:14px}
|
||||
.meter-h{display:flex;justify-content:space-between;font-size:12px;color:var(--mut);margin-bottom:4px}
|
||||
.meter-h b{color:var(--fg);font-variant-numeric:tabular-nums}
|
||||
.bar{height:9px;border-radius:5px;background:#11151a;border:1px solid var(--bd);overflow:hidden}
|
||||
.bar>i{display:block;height:100%;width:0;background:linear-gradient(90deg,#3a7d57,var(--grn));transition:width .4s}
|
||||
#utilbar{background:linear-gradient(90deg,#9a5a1f,var(--acc))}
|
||||
#bufbar{background:linear-gradient(90deg,#2f5a9a,#4a86d8)}
|
||||
.queue{font:13px ui-monospace,monospace;color:var(--mut)}
|
||||
.banner{margin:0 0 14px;padding:.7rem .9rem;border-radius:10px;background:#3a2f12;
|
||||
border:1px solid #5a4a17;color:#ffd98a;font-size:13px}
|
||||
.logs-h{display:flex;align-items:center;justify-content:space-between}
|
||||
.grow{flex:1;display:flex;flex-direction:column;min-height:0}
|
||||
.grow .logs{flex:1;min-height:0}
|
||||
.copybtn{font:600 11px system-ui;letter-spacing:.04em;text-transform:uppercase;
|
||||
background:#262c34;color:var(--fg);border:1px solid var(--bd);border-radius:7px;
|
||||
padding:5px 11px;cursor:pointer}
|
||||
.copybtn:hover{border-color:var(--acc)}
|
||||
.logs{margin:0;background:#0b0e12;border:1px solid var(--bd);border-radius:10px;padding:12px;
|
||||
overflow:auto;font:12px/1.55 ui-monospace,SFMono-Regular,Menlo,monospace;
|
||||
color:#b9c4d0;white-space:pre-wrap;word-break:break-word}
|
||||
</style></head><body>
|
||||
<div class=wrap>
|
||||
<header>
|
||||
<div class=brand><span class=logo>◆</span> FabledCurator <span class=sub>GPU agent</span></div>
|
||||
<div class=conn><span class="dot" id=dot></span><span id=connlbl>—</span></div>
|
||||
</header>
|
||||
<p class=meta>Server <code id=fc>—</code> · token <code id=cfg>—</code> · build <code id=build>__VERSION__</code></p>
|
||||
|
||||
<div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
|
||||
a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
|
||||
</div>
|
||||
<div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
|
||||
<div id=banner class=banner style=display:none>
|
||||
curator unreachable — holding work + retrying, resumes on its own (no restart needed)
|
||||
</div>
|
||||
|
||||
<section class=card>
|
||||
<div class=card-h>Control</div>
|
||||
<div class=controls>
|
||||
<button class="btn start" id=startbtn onclick=act('start')>▶ Start</button>
|
||||
<button class="btn stop" id=stopbtn onclick=act('stop')>■ Stop</button>
|
||||
<div class=spacer></div>
|
||||
<label class=switch><input type=checkbox id=autochk onchange="setauto(this.checked)"><span class=track></span>Auto</label>
|
||||
<div class=stepper>
|
||||
<button class=step onclick=setc(-1)>−</button>
|
||||
<input id=conc type=number min=1 value=1 onchange="setv(this.value)">
|
||||
<button class=step onclick=setc(1)>+</button>
|
||||
</div>
|
||||
<div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
|
||||
<button class=step onclick=stepbw(-1)>−</button>
|
||||
<input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
|
||||
<button class=step onclick=stepbw(1)>+</button>
|
||||
<span class=unit>MB/s</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class=hint id=conchint>auto-tuning downloaders to keep the GPU fed · max 8</div>
|
||||
</section>
|
||||
|
||||
<section class=card>
|
||||
<div class=card-h>Status</div>
|
||||
<div class=tiles>
|
||||
<div class=tile><div class=n id=state>—</div><div class=l>state</div></div>
|
||||
<div class=tile><div class=n id=jpm>—</div><div class=l>jobs / min</div></div>
|
||||
<div class=tile><div class=n id=dpm>—</div><div class=l>downloads / min</div></div>
|
||||
<div class=tile><div class="n ok" id=done>0</div><div class=l>processed</div></div>
|
||||
<div class=tile><div class=n id=err>0</div><div class=l>errors</div></div>
|
||||
<div class=tile><div class=n id=waited>0</div><div class=l>waited out</div></div>
|
||||
</div>
|
||||
<div class=meters>
|
||||
<div class=meter><div class=meter-h><span>GPU util</span><b id=utillbl>—</b></div>
|
||||
<div class=bar><i id=utilbar></i></div></div>
|
||||
<div class=meter><div class=meter-h><span>VRAM</span><b id=vramlbl>—</b></div>
|
||||
<div class=bar><i id=gpubar></i></div></div>
|
||||
<div class=meter><div class=meter-h><span>buffer occupancy</span><b id=buflbl>—</b></div>
|
||||
<div class=bar><i id=bufbar></i></div></div>
|
||||
</div>
|
||||
<div class=queue id=pipe>downloaders — · consumers — · on GPU 0</div>
|
||||
<div class=queue id=queue>queue —</div>
|
||||
</section>
|
||||
|
||||
<section class="card grow">
|
||||
<div class="card-h logs-h">Logs
|
||||
<button class=copybtn id=copybtn onclick=copyLogs()>Copy</button>
|
||||
</div>
|
||||
<pre class=logs id=logs>waiting for activity…</pre>
|
||||
</section>
|
||||
</div>
|
||||
<script>
|
||||
const PAGE_BUILD="__BUILD_ID__"
|
||||
let CAP=8
|
||||
// Optimistic transitional state on click, then apply the POST's own status
|
||||
// response (it returns worker.status()) for instant feedback — don't wait on the
|
||||
// separate /status poll, which can lag behind the curator queue call.
|
||||
async function act(p){
|
||||
pending(p==='start'?'starting':'stopping')
|
||||
// Abort a slow POST after 8s so the buttons never stay stuck — the periodic
|
||||
// /status refresh (now always fast) recovers the true state either way.
|
||||
const ac=new AbortController(); const to=setTimeout(()=>ac.abort(),8000)
|
||||
try{ applyStatus(await (await fetch('/'+p,{method:'POST',signal:ac.signal})).json()) }
|
||||
catch{ refresh() /* on abort/error, repaint the real state from /status */ }
|
||||
finally{ clearTimeout(to) }
|
||||
}
|
||||
function pending(label){
|
||||
// Instant optimistic feedback on click; applyStatus (POST response, then the
|
||||
// periodic poll) then owns the real state + which buttons are enabled.
|
||||
state.textContent=label; state.className='n busy'
|
||||
dot.className='dot amber'
|
||||
startbtn.disabled=true; stopbtn.disabled=true
|
||||
}
|
||||
function setc(d){ if(conc.disabled)return; setv((parseInt(conc.value||'1'))+d) }
|
||||
async function setv(v){
|
||||
v=Math.max(1,Math.min(CAP,parseInt(v)||1)); conc.value=v
|
||||
await fetch('/concurrency',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({value:v})});refresh()
|
||||
}
|
||||
async function setauto(on){
|
||||
await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({value:on})});refresh()
|
||||
}
|
||||
function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
|
||||
// Runtimes that did NOT get the GPU, from the startup report. Both fall back
|
||||
// to the CPU without raising, so this banner and the pill are the only place
|
||||
// on this page a slow, CPU-bound agent announces itself.
|
||||
function cpuOnly(s){
|
||||
const a=s.accel||{}
|
||||
return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
|
||||
}
|
||||
async function setbw(v){
|
||||
v=Math.max(0,parseFloat(v)||0); bw.value=v
|
||||
await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({value:v})});refresh()
|
||||
}
|
||||
async function refresh(){
|
||||
let s; try{ s=await (await fetch('/status')).json() }catch{ return }
|
||||
applyStatus(s)
|
||||
}
|
||||
function applyStatus(s){
|
||||
// NB: don't write a separate `capn` element here — conchint.textContent below
|
||||
// rewrites the whole hint (incl. the max), and any child element nested in it
|
||||
// would be destroyed by that write, breaking the NEXT applyStatus call.
|
||||
CAP=s.max_concurrency||8
|
||||
// The backend owns the state now (stopped|starting|running|stopping) and drives
|
||||
// every transition, so the pill is always truthful — no client-side guessing
|
||||
// from active>0, which used to wedge on "stopping" forever.
|
||||
const st=s.state||'stopped'
|
||||
const running=st==='running'
|
||||
const busy=(st==='starting'||st==='stopping')
|
||||
// Stale-page guard: if the server is a newer build than this page, the cached
|
||||
// controls may misbehave — tell the operator to reload.
|
||||
if(s.build && s.build!==PAGE_BUILD) verbanner.style.display='block'
|
||||
state.textContent=st
|
||||
state.className='n'+(busy?' busy':'')
|
||||
// Buttons follow the real state so you can't fight a transition: Start only
|
||||
// from stopped; Stop only while up; both disabled through "stopping" until the
|
||||
// backend truthfully lands on "stopped".
|
||||
startbtn.disabled=(st!=='stopped')
|
||||
stopbtn.disabled=!(running||st==='starting')
|
||||
// Throughput rates arrive READY from the backend (jobs/min ≈ GPU throughput,
|
||||
// dl/min ≈ fetch throughput), computed there on a fixed cadence — so they show
|
||||
// a real number no matter how often this tab polls (a backgrounded tab throttles
|
||||
// its timers, which used to leave a client-side delta-rate blank forever).
|
||||
jpm.textContent=(s.jobs_per_min!=null)?Math.round(s.jobs_per_min):'—'
|
||||
dpm.textContent=(s.downloads_per_min!=null)?Math.round(s.downloads_per_min):'—'
|
||||
done.textContent=s.processed
|
||||
err.textContent=s.errors; err.className='n'+(s.errors>0?' warn':'')
|
||||
waited.textContent=s.transient||0
|
||||
// Instantaneous pool state → demoted to the sub-line, where its jumpiness reads
|
||||
// as live churn rather than a "broken" headline metric.
|
||||
// '=== false' (not falsy) so a stale page that doesn't send models_loaded shows
|
||||
// nothing; when the idle monitor unloads, the VRAM meter drops alongside this.
|
||||
pipe.textContent='downloaders '+(s.downloaders!=null?s.downloaders:'—')+' · consumers '+(s.consumers!=null?s.consumers:'—')+' · on GPU '+(s.active||0)
|
||||
+' · net '+(s.net_mb_s!=null?s.net_mb_s.toFixed(1):'—')+' MB/s'
|
||||
+(s.bandwidth_limit_mb_s>0?(' / cap '+s.bandwidth_limit_mb_s):'')
|
||||
+(s.models_loaded===false?' · GPU models unloaded (idle — reload on next job)':'')
|
||||
if(document.activeElement!==bw && s.bandwidth_limit_mb_s!=null) bw.value=s.bandwidth_limit_mb_s
|
||||
// Buffer occupancy bar (also driven here so it tracks the /status cadence).
|
||||
if(s.buffer!=null && s.buffer_max){ const p=Math.round(100*s.buffer/s.buffer_max)
|
||||
buflbl.textContent=s.buffer+' / '+s.buffer_max; bufbar.style.width=p+'%' }
|
||||
// Auto on → dial reflects the auto-chosen count (read-only); off → manual.
|
||||
if(document.activeElement!==autochk) autochk.checked=!!s.auto
|
||||
conc.disabled=!!s.auto; conc.style.opacity=s.auto?0.55:1
|
||||
conchint.textContent=(s.auto?('auto-tuning downloaders to keep the GPU fed · max '+CAP):('manual downloaders · max '+CAP))
|
||||
+(s.idle?' · idle — queue empty, lease poll backed off (new work noticed within ~15 min)'
|
||||
:(s.bw_capped?' · holding at the bandwidth cap (more downloaders would not go faster)':''))
|
||||
if(document.activeElement!==conc) conc.value=s.concurrency
|
||||
conc.max=CAP
|
||||
// Connection pill + queue come only from the /status poll (the Start/Stop POST
|
||||
// responses skip the slow curator call to stay snappy) — guard so an action
|
||||
// response doesn't blank them.
|
||||
if('configured' in s){
|
||||
const ok=s.configured
|
||||
fc.textContent=s.fc_url; cfg.textContent=ok?'set':'MISSING'
|
||||
// Pill colour + label track the real state: green only when running AND
|
||||
// curator is answering; amber for the transient states + a running-but-
|
||||
// unreachable curator; grey when stopped; red with no token.
|
||||
let dc='dot', lbl='stopped'
|
||||
if(!ok){ dc='dot red'; lbl='no token' }
|
||||
else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
|
||||
if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
|
||||
else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
|
||||
else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
|
||||
dot.className=dc; connlbl.textContent=lbl
|
||||
banner.style.display=(st==='running' && !s.queue)?'block':'none'
|
||||
const slow=cpuOnly(s)
|
||||
accelbanner.style.display=slow.length?'block':'none'
|
||||
accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
|
||||
+slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
|
||||
+'. After a driver update, regenerate the CDI spec (agent README).'):''
|
||||
queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
|
||||
}
|
||||
}
|
||||
// GPU meters poll their OWN endpoint on a fast cadence — kept off /status so a
|
||||
// slow curator queue call can't freeze the bars (they only stale on refresh).
|
||||
let UAVG=null // smoothed util for the bar (raw util swings 0↔99; show the trend)
|
||||
async function refreshGpu(){
|
||||
let g; try{ g=await (await fetch('/gpu')).json() }catch{ return }
|
||||
if(g && g.util_pct!=null){
|
||||
// Prefer the agent's own EWMA (util_smooth) when running; otherwise smooth
|
||||
// the raw reading here so a stopped agent's bar still glides, not jumps.
|
||||
const raw=g.util_pct
|
||||
UAVG = (g.util_smooth!=null) ? g.util_smooth
|
||||
: (UAVG==null ? raw : 0.25*raw + 0.75*UAVG)
|
||||
const used=g.mem_used_mb, tot=g.mem_total_mb||1
|
||||
utillbl.textContent=Math.round(UAVG)+'% · '+g.temp_c+'°C'; utilbar.style.width=Math.round(UAVG)+'%'
|
||||
vramlbl.textContent=used+' / '+tot+' MB'; gpubar.style.width=Math.round(100*used/tot)+'%'
|
||||
} else { UAVG=null; utillbl.textContent='n/a'; vramlbl.textContent='n/a (CPU?)'; utilbar.style.width='0%'; gpubar.style.width='0%' }
|
||||
}
|
||||
async function refreshLogs(){
|
||||
try{
|
||||
const r=await (await fetch('/logs')).json()
|
||||
const el=logs, atBottom=el.scrollHeight-el.scrollTop-el.clientHeight<40
|
||||
el.textContent=(r.lines&&r.lines.length)?r.lines.join('\\n'):'waiting for activity…'
|
||||
if(atBottom) el.scrollTop=el.scrollHeight
|
||||
}catch{}
|
||||
}
|
||||
async function copyLogs(){
|
||||
const txt=logs.textContent||''
|
||||
try{ await navigator.clipboard.writeText(txt) }
|
||||
catch{ const t=document.createElement('textarea'); t.value=txt; document.body.appendChild(t);
|
||||
t.select(); try{document.execCommand('copy')}catch{}; t.remove() }
|
||||
copybtn.textContent='Copied'; setTimeout(()=>{copybtn.textContent='Copy'},1200)
|
||||
}
|
||||
refresh(); refreshGpu(); refreshLogs()
|
||||
setInterval(refresh,3000); setInterval(refreshGpu,1500); setInterval(refreshLogs,2500)
|
||||
</script></body></html>"""
|
||||
@@ -1,79 +0,0 @@
|
||||
"""What this agent build IS — stamped at image build time, not configurable.
|
||||
|
||||
The mirror of `backend/app/build_info.py`, for the same reasons and with the
|
||||
same posture. Kept as its own module rather than as constants in `app.py`
|
||||
because it is stdlib-only and therefore importable by the test suite, which
|
||||
cannot import `app` (torch, transformers and ultralytics are not in the CI
|
||||
image — see build.yml's "Agent syntax check").
|
||||
|
||||
## Why this replaced a hand-written string
|
||||
|
||||
`app.VERSION` used to be a literal an author was asked to bump, carrying a
|
||||
version AND a changelog in one string:
|
||||
|
||||
VERSION = "2026-07-17.1 · idle model-unload: after ~5 min idle ..."
|
||||
|
||||
Nobody bumped it. The September image printed the identical string to the July
|
||||
one, so the one surface that was supposed to answer *"did my pull work?"*
|
||||
answered *"2026-07-17.1"* either way. An artifact that cannot identify itself
|
||||
is worse than one that says nothing, because the stale value reads as an
|
||||
answer.
|
||||
|
||||
The values are now derived by `scripts/artifacts.sh` from the commit its
|
||||
shipped files last changed in — the same derivation the web image has used
|
||||
since milestone 313, and the same one the reuse check already ran for the
|
||||
agent and discarded.
|
||||
|
||||
## Three values, never folded together (rule 149)
|
||||
|
||||
* `FC_VERSION` — the NAME, `YYYY.MM.DD.HHMM` UTC, derived from COMMIT time.
|
||||
For people to read and quote. Identical on `dev` and `main` for the same
|
||||
source, which is the property that makes "am I running the same code as
|
||||
production?" answerable at a glance.
|
||||
* `FC_CHANNEL` — a SIBLING field, never a suffix inside the name.
|
||||
* `FC_REVISION` — the 12-char commit sha, the artifact's IDENTITY. This is
|
||||
what the reuse check already keys on as the `fc.revision` image label.
|
||||
|
||||
**Absent rather than empty when unknown.** A locally built image has no
|
||||
stamp, and neither does any image predating this module. One spelling of
|
||||
"cannot say" instead of two.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
FC_VERSION = os.environ.get("FC_VERSION", "").strip()
|
||||
FC_CHANNEL = os.environ.get("FC_CHANNEL", "").strip()
|
||||
FC_REVISION = os.environ.get("FC_REVISION", "").strip()
|
||||
|
||||
|
||||
def display_version() -> str:
|
||||
"""The build, as a line for a human: `2026.09.24.1052 (dev)`.
|
||||
|
||||
`unknown` rather than a blank when unstamped — an empty slot in the meta
|
||||
line reads as "no version", which is a different and false claim from "this
|
||||
build does not carry one".
|
||||
"""
|
||||
if not FC_VERSION:
|
||||
return "unknown"
|
||||
return f"{FC_VERSION} ({FC_CHANNEL})" if FC_CHANNEL else FC_VERSION
|
||||
|
||||
|
||||
def build_id() -> str:
|
||||
"""The token the control page compares against `/status` to notice it is
|
||||
showing a CACHED page from a previous build.
|
||||
|
||||
Deliberately NOT `display_version()`. That is the value for reading; this
|
||||
is the value for deciding, and folding the two is what rule 149 is about.
|
||||
The revision is the better discriminator of the two — two builds of the
|
||||
same commit ARE the same agent and should not prompt a reload, and two
|
||||
different commits always differ here even when they land in the same
|
||||
minute and derive one version name.
|
||||
|
||||
A locally built image falls through to a constant, so the reload banner
|
||||
cannot fire for it. That is honest rather than a gap: nothing in an
|
||||
unstamped image knows what source it was built from, and a per-process
|
||||
nonce would make every ordinary container RESTART claim a new version had
|
||||
arrived — a false positive on the exact surface the banner exists to keep
|
||||
trustworthy.
|
||||
"""
|
||||
return FC_REVISION or FC_VERSION or "local"
|
||||
@@ -1,147 +0,0 @@
|
||||
"""HTTP client for the FabledCurator GPU-job API.
|
||||
|
||||
The agent's ONLY contact with FC — lease/submit/heartbeat/fail + fetch image
|
||||
bytes, all over HTTP with the bearer token. No DB/Redis.
|
||||
"""
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from . import accel
|
||||
|
||||
|
||||
class FcClient:
|
||||
def __init__(self, base_url: str, token: str, agent_id: str):
|
||||
self.base = base_url.rstrip("/")
|
||||
self.agent_id = agent_id
|
||||
# Main session: NO in-request retry — lease/fetch are cheap to redo and
|
||||
# the worker loop already backs off + re-leases on failure. (Auto-retrying
|
||||
# a lease could double-claim a batch if a response is lost.)
|
||||
self.s = self._session(token)
|
||||
# Submit session: retry in-place, because by submit time the GPU work is
|
||||
# already DONE — a momentary blip (dropped connection, gateway 5xx during
|
||||
# a curator redeploy) must not throw that work away and force a full
|
||||
# re-download + recompute on another agent. A duplicate submit after a
|
||||
# lost response is harmless: the job is already closed, so it just returns
|
||||
# 409 lease_invalid (a no-op). Idempotent enough to retry POST safely.
|
||||
retry = Retry(
|
||||
total=3, connect=3, read=3, status=3,
|
||||
backoff_factor=0.5, # ~0.5s, 1s, 2s between tries
|
||||
status_forcelist=(500, 502, 503, 504), # transient server/gateway
|
||||
allowed_methods=frozenset({"POST"}),
|
||||
raise_on_status=False, # let raise_for_status decide
|
||||
)
|
||||
self._submit_s = self._session(token, retry)
|
||||
|
||||
@staticmethod
|
||||
def _session(token: str, retry: Retry | None = None) -> requests.Session:
|
||||
s = requests.Session()
|
||||
s.headers["Authorization"] = f"Bearer {token}"
|
||||
# Many worker threads share a Session; the default pool (10) would
|
||||
# throttle them + spam "connection pool is full". Size it for the cap.
|
||||
adapter = HTTPAdapter(
|
||||
pool_connections=64, pool_maxsize=64, max_retries=retry or 0
|
||||
)
|
||||
s.mount("http://", adapter)
|
||||
s.mount("https://", adapter)
|
||||
return s
|
||||
|
||||
def _submit(self, path: str, payload: dict) -> dict:
|
||||
"""POST to a submit endpoint on the RETRYING session (by submit time the
|
||||
GPU work is done — a blip must not throw it away), raise on a hard error,
|
||||
and return the parsed JSON. `agent_id` is added to every body."""
|
||||
r = self._submit_s.post(
|
||||
f"{self.base}{path}",
|
||||
json={"agent_id": self.agent_id, **payload},
|
||||
timeout=120,
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
|
||||
def _post_quiet(self, path: str, payload: dict) -> None:
|
||||
"""Fire-and-forget POST on the main session — heartbeat/fail/release are
|
||||
best-effort, so a transport error is swallowed (the worker's own retry and
|
||||
the server's orphan-recovery cover a lost call). `agent_id` is added."""
|
||||
try:
|
||||
self.s.post(
|
||||
f"{self.base}{path}",
|
||||
json={"agent_id": self.agent_id, **payload},
|
||||
timeout=30,
|
||||
)
|
||||
except requests.RequestException:
|
||||
pass
|
||||
|
||||
def lease(self, batch_size: int) -> list[dict]:
|
||||
r = self.s.post(
|
||||
f"{self.base}/api/gpu/jobs/lease",
|
||||
json={
|
||||
"agent_id": self.agent_id, "batch_size": batch_size,
|
||||
"accel": accel.summary(),
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.json().get("jobs", [])
|
||||
|
||||
def submit(self, job_id: int, regions: list[dict], replace_kinds: list[str]) -> dict:
|
||||
return self._submit("/api/gpu/jobs/submit", {
|
||||
"job_id": job_id, "regions": regions, "replace_kinds": replace_kinds,
|
||||
})
|
||||
|
||||
def submit_embedding(self, job_id: int, embedding: list, version: str) -> dict:
|
||||
"""Post a whole-image SigLIP embedding (the 'embed' task) → image_record."""
|
||||
return self._submit("/api/gpu/jobs/submit_embedding", {
|
||||
"job_id": job_id, "embedding": embedding, "embedding_version": version,
|
||||
})
|
||||
|
||||
def heartbeat(self, job_ids: list[int]) -> None:
|
||||
self._post_quiet(
|
||||
"/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
|
||||
)
|
||||
|
||||
def fail(self, job_id: int, error: str) -> None:
|
||||
self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
|
||||
|
||||
def release(self, job_ids: list[int]) -> None:
|
||||
# Graceful hand-back on stop so orphaned work is re-leased at once.
|
||||
if not job_ids:
|
||||
return
|
||||
self._post_quiet("/api/gpu/jobs/release", {"job_ids": job_ids})
|
||||
|
||||
def fetch_image(self, image_url: str, throttle=None) -> bytes:
|
||||
# image_url is a server-relative path ("/images/...").
|
||||
# timeout=(connect, read): the read timeout is BETWEEN-BYTES, not total,
|
||||
# so a large-but-flowing download still completes — but a stuck/dead
|
||||
# connection (curator overloaded) fails in 60s instead of hanging a
|
||||
# downloader for 180s and piling up concurrent stuck requests on curator.
|
||||
# With a throttle (the worker's shared TokenBucket), the body is streamed
|
||||
# in chunks and each chunk is charged to the global bandwidth budget —
|
||||
# pausing between reads lets TCP flow control pace curator's send side.
|
||||
with self.s.get(
|
||||
f"{self.base}{image_url}", timeout=(10, 60), stream=throttle is not None
|
||||
) as r:
|
||||
r.raise_for_status()
|
||||
if throttle is None:
|
||||
return r.content
|
||||
buf = bytearray()
|
||||
for chunk in r.iter_content(chunk_size=262_144):
|
||||
throttle.take(len(chunk))
|
||||
buf.extend(chunk)
|
||||
return bytes(buf)
|
||||
|
||||
def is_reachable(self) -> bool:
|
||||
"""Cheap 'is curator responding at all right now?' check. Used to decide,
|
||||
when a video can't be sampled, between a transient outage (keep retrying —
|
||||
survives a redeploy) and an unprocessable file (fail it, don't loop)."""
|
||||
try:
|
||||
r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
|
||||
return r.status_code < 500
|
||||
except requests.RequestException:
|
||||
return False
|
||||
|
||||
def queue_status(self) -> dict:
|
||||
# Short timeout: this backs the UI /status poll, so a busy curator must
|
||||
# not hang the page for long (the GPU meters poll /gpu separately).
|
||||
r = self.s.get(f"{self.base}/api/gpu/status", timeout=5)
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
@@ -1,100 +0,0 @@
|
||||
"""Agent config, all from env (the control container is configured at run)."""
|
||||
# Lazy annotations so the `from_env(cls) -> Config` self-reference is a string,
|
||||
# not evaluated at class-definition time — otherwise it NameErrors on the agent's
|
||||
# Python 3.10 (CI lints on 3.14, where PEP 649 hides this).
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
def _bool_env(name: str, default: str = "") -> bool:
|
||||
"""A boolean env var — present + truthy ('1'/'true'/'yes') → True."""
|
||||
return os.environ.get(name, default).lower() in ("1", "true", "yes")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
fc_url: str # base URL of the FabledCurator web service
|
||||
token: str # the bearer token from Settings → Tagging → GPU agent
|
||||
agent_id: str # identifies this agent's leases
|
||||
batch_size: int # jobs a worker leases per round
|
||||
concurrency: int # INITIAL parallel workers (tunable live from the UI)
|
||||
ccip_model: str # imgutils CCIP model name ("" → imgutils default)
|
||||
detector_level: str # imgutils person-detector level: n|s|m|x
|
||||
poll_idle_seconds: float # wait between empty leases
|
||||
embed_dtype: str # torch dtype for the crop embedder: float16|float32
|
||||
embed_model_override: str # force a SigLIP-family model ("" → use the one
|
||||
# the server announces in the lease)
|
||||
auto_start: bool # start the worker pool on boot (so a container restart
|
||||
# resumes processing without anyone clicking Start)
|
||||
auto_scale: bool # autoscale the worker count (throughput hill-climb)
|
||||
# Crop PROPOSERS (extra YOLO detectors that say where to crop). Each weight
|
||||
# spec is an ultralytics name | http(s) URL | "hf_repo::file" ("" = off).
|
||||
person_weights: str # general COCO person detector (Western/realistic figs)
|
||||
person_conf: float
|
||||
anatomy_weights: str # booru_yolo anime/furry/NSFW components
|
||||
anatomy_conf: float
|
||||
panel_weights: str # comic-panel detector
|
||||
panel_conf: float
|
||||
max_components: int # cap anatomy component crops per frame
|
||||
max_panels: int # cap panel crops per frame
|
||||
max_figures: int # cap figure boxes per frame (each = a CCIP call + crop)
|
||||
max_regions: int # hard cap on total regions per JOB (submit-size backstop)
|
||||
dedupe_iou: float # crops overlapping >= this (same kind) are near-dupes,
|
||||
# dropped before the embed; >=1.0 disables it
|
||||
frame_dedupe_distance: int # video frames whose dHash differs by < this many
|
||||
# bits are near-duplicates, dropped before detect;
|
||||
# higher keeps more frames, 0 disables
|
||||
ffmpeg_timeout: float # hard ceiling (s) for ffmpeg-from-URL video sampling;
|
||||
# generous so a SLOW media link still completes
|
||||
bandwidth_limit_mb_s: float # aggregate download cap in MEGABYTES/s across
|
||||
# all downloaders + video streams (0 = unlimited);
|
||||
# tunable live from the agent UI
|
||||
idle_unload_seconds: float # after this long with the GPU idle (nothing in
|
||||
# flight, queue empty or Stopped), unload the
|
||||
# SigLIP embedder + YOLO proposers to free their
|
||||
# VRAM; they reload lazily on the next job. A
|
||||
# 24/7 agent otherwise squats on ~5GB doing
|
||||
# nothing. 0 disables (keep models warm forever).
|
||||
|
||||
@classmethod
|
||||
def from_env(cls) -> Config:
|
||||
return cls(
|
||||
fc_url=os.environ.get("FC_URL", "http://localhost:8000").rstrip("/"),
|
||||
token=os.environ.get("FC_TOKEN", ""),
|
||||
agent_id=os.environ.get("AGENT_ID", "desktop-agent"),
|
||||
batch_size=int(os.environ.get("BATCH_SIZE", "4")),
|
||||
concurrency=int(os.environ.get("CONCURRENCY", "1")),
|
||||
ccip_model=os.environ.get("CCIP_MODEL", ""),
|
||||
detector_level=os.environ.get("DETECTOR_LEVEL", "m"),
|
||||
poll_idle_seconds=float(os.environ.get("POLL_IDLE_SECONDS", "10")),
|
||||
embed_dtype=os.environ.get("SIGLIP_DTYPE", "float16"),
|
||||
embed_model_override=os.environ.get("EMBED_MODEL_NAME", ""),
|
||||
auto_start=_bool_env("AUTO_START"),
|
||||
auto_scale=_bool_env("AUTO_SCALE", "true"),
|
||||
person_weights=os.environ.get("PERSON_WEIGHTS", "yolo11n.pt"),
|
||||
person_conf=float(os.environ.get("PERSON_CONF", "0.35")),
|
||||
anatomy_weights=os.environ.get("ANATOMY_WEIGHTS", ""),
|
||||
anatomy_conf=float(os.environ.get("ANATOMY_CONF", "0.30")),
|
||||
panel_weights=os.environ.get("PANEL_WEIGHTS", ""),
|
||||
panel_conf=float(os.environ.get("PANEL_CONF", "0.30")),
|
||||
max_components=int(os.environ.get("MAX_COMPONENTS", "8")),
|
||||
max_panels=int(os.environ.get("MAX_PANELS", "8")),
|
||||
max_figures=int(os.environ.get("MAX_FIGURES", "8")),
|
||||
max_regions=int(os.environ.get("MAX_REGIONS", "128")),
|
||||
dedupe_iou=float(os.environ.get("DEDUPE_IOU", "0.85")),
|
||||
frame_dedupe_distance=int(os.environ.get("FRAME_DEDUPE_DISTANCE", "8")),
|
||||
ffmpeg_timeout=float(os.environ.get("FFMPEG_TIMEOUT", "1200")),
|
||||
# Default 8 MB/s (~64 Mbit/s): ~20% of the measured ~300 Mbit/s home
|
||||
# WiFi, so browsing stays snappy while the agent works — yet MORE
|
||||
# sweep throughput than the self-inflicted congestion collapse this
|
||||
# replaces (2026-07-02: 8 unthrottled downloaders bufferbloated the
|
||||
# link to ~1-1.5 MB/s per stream, browser included). Raise it (or 0)
|
||||
# from the agent UI on wired/faster networks.
|
||||
bandwidth_limit_mb_s=float(os.environ.get("BANDWIDTH_LIMIT_MB_S", "8")),
|
||||
# 5 min: long enough that a lull between job bursts doesn't thrash the
|
||||
# (few-second) reload, short enough that an agent left running with an
|
||||
# empty queue hands its VRAM back promptly.
|
||||
idle_unload_seconds=float(os.environ.get("IDLE_UNLOAD_SECONDS", "300")),
|
||||
)
|
||||
@@ -1,36 +0,0 @@
|
||||
"""Crop primitive — vendored from backend/app/services/ml/crops.py so the agent
|
||||
is self-contained. Keep in sync if the floor logic changes."""
|
||||
from PIL import Image
|
||||
|
||||
MIN_CROP_FRACTION = 0.10
|
||||
MIN_CROP_PX = 64
|
||||
|
||||
|
||||
def crop_region(
|
||||
img: Image.Image,
|
||||
bbox: tuple[float, float, float, float],
|
||||
*,
|
||||
pad: float = 0.0,
|
||||
min_fraction: float = MIN_CROP_FRACTION,
|
||||
min_px: int = MIN_CROP_PX,
|
||||
) -> Image.Image | None:
|
||||
"""Crop a NORMALIZED bbox (x, y, w, h in [0,1]); None if below the size
|
||||
floor (max of a fraction-of-short-side and an absolute pixel floor)."""
|
||||
iw, ih = img.size
|
||||
x, y, w, h = bbox
|
||||
px, py, pw, ph = x * iw, y * ih, w * iw, h * ih
|
||||
if pad:
|
||||
px -= pw * pad / 2.0
|
||||
py -= ph * pad / 2.0
|
||||
pw *= (1.0 + pad)
|
||||
ph *= (1.0 + pad)
|
||||
left = max(0, int(round(px)))
|
||||
top = max(0, int(round(py)))
|
||||
right = min(iw, int(round(px + pw)))
|
||||
bottom = min(ih, int(round(py + ph)))
|
||||
if right <= left or bottom <= top:
|
||||
return None
|
||||
floor = max(min_px, int(min_fraction * min(iw, ih)))
|
||||
if min(right - left, bottom - top) < floor:
|
||||
return None
|
||||
return img.crop((left, top, right, bottom)).convert("RGB")
|
||||
@@ -1,233 +0,0 @@
|
||||
"""Region PROPOSERS — small YOLO detectors that decide WHERE to crop. They run
|
||||
on the agent GPU and their boxes feed the crop → SigLIP → max-over-bag pipeline:
|
||||
|
||||
- person (general COCO yolo11n): full-figure boxes for realistic / Western art
|
||||
the anime person-detector misses; NMS-merged with imgutils detect_person and
|
||||
fed to CCIP (identity) + a concept crop.
|
||||
- anatomy (booru_yolo): anime / furry / NSFW torso components (head, cat-head,
|
||||
boob, hip, …) — concept crops aligned to the operator's tag vocabulary.
|
||||
- panel (mosesb): a comic page → panel regions → concept crops.
|
||||
|
||||
Each proposer is INDEPENDENTLY optional + guarded: a bad weight path or an
|
||||
inference error disables just that proposer (logged) and never breaks the
|
||||
worker, which still falls back to imgutils detection. Weights resolve from an
|
||||
ultralytics builtin name ("yolo11n.pt"), an http(s) URL, or "hf_repo::file" —
|
||||
cached under HF_HOME so the download happens once.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("fc_agent.detectors")
|
||||
_CACHE = Path(os.environ.get("HF_HOME", "/models")) / "yolo"
|
||||
|
||||
|
||||
def _resolve(spec: str) -> str | None:
|
||||
"""A local weights path (downloading if needed) or an ultralytics builtin
|
||||
name. None if the spec is empty/unresolvable."""
|
||||
if not spec:
|
||||
return None
|
||||
if "::" in spec: # hf_repo::filename
|
||||
repo, _, fname = spec.partition("::")
|
||||
from huggingface_hub import hf_hub_download
|
||||
return hf_hub_download(
|
||||
repo_id=repo, filename=fname, cache_dir=str(_CACHE)
|
||||
)
|
||||
if spec.startswith(("http://", "https://")):
|
||||
_CACHE.mkdir(parents=True, exist_ok=True)
|
||||
dest = _CACHE / spec.rsplit("/", 1)[-1]
|
||||
if not dest.is_file():
|
||||
import requests
|
||||
r = requests.get(spec, timeout=300)
|
||||
r.raise_for_status()
|
||||
dest.write_bytes(r.content)
|
||||
return str(dest)
|
||||
return spec # ultralytics builtin name
|
||||
|
||||
|
||||
def _iou(a, b) -> float:
|
||||
ax, ay, aw, ah = a
|
||||
bx, by, bw, bh = b
|
||||
ix = max(0.0, min(ax + aw, bx + bw) - max(ax, bx))
|
||||
iy = max(0.0, min(ay + ah, by + bh) - max(ay, by))
|
||||
inter = ix * iy
|
||||
union = aw * ah + bw * bh - inter
|
||||
return inter / union if union > 0 else 0.0
|
||||
|
||||
|
||||
def nms_merge(boxes, iou_thresh: float = 0.6):
|
||||
"""Greedy NMS over (bbox_norm, score, label) from possibly several detectors,
|
||||
so the same figure found by two of them collapses to one (higher-score) box."""
|
||||
kept = []
|
||||
for bb, sc, lb in sorted(boxes, key=lambda b: b[1], reverse=True):
|
||||
if all(_iou(bb, k[0]) < iou_thresh for k in kept):
|
||||
kept.append((bb, sc, lb))
|
||||
return kept
|
||||
|
||||
|
||||
def dedupe_crops(pending, iou_thresh: float = 0.85):
|
||||
"""Greedy high-IoU dedupe over a list of (crop, region_template) pairs, run
|
||||
just before the batched SigLIP embed so we never embed the same region twice.
|
||||
|
||||
Figure boxes are already NMS-merged and each YOLO self-NMSes, but the combined
|
||||
per-frame pile (figure→concept ∪ anatomy component→concept ∪ panel) can still
|
||||
carry genuine near-duplicates across proposers — e.g. a figure box that nearly
|
||||
coincides with an anatomy component on a solo bust, or overlapping booru head
|
||||
classes on one head. Those embed the same region twice, wasting GPU and a slot
|
||||
against max_regions.
|
||||
|
||||
Boxes are compared ONLY within the same output kind and dropped when they
|
||||
overlap at >= iou_thresh, keeping the highest-scoring one. The HIGH default
|
||||
threshold is deliberate: it collapses only true near-identical boxes while
|
||||
preserving intentional nested crops across scopes (a whole figure vs a small
|
||||
head component sit well below it) and distinct kinds (concept vs panel). A
|
||||
value >= 1.0 effectively disables it (nothing but an exact box matches)."""
|
||||
kept = []
|
||||
kept_boxes: dict = {} # kind -> [bbox, ...] already kept
|
||||
for crop, tmpl in sorted(
|
||||
pending, key=lambda p: p[1].get("score") or 0.0, reverse=True
|
||||
):
|
||||
bb = tmpl.get("bbox")
|
||||
prior = kept_boxes.setdefault(tmpl.get("kind"), [])
|
||||
if bb is not None and any(_iou(bb, kb) >= iou_thresh for kb in prior):
|
||||
continue
|
||||
prior.append(bb)
|
||||
kept.append((crop, tmpl))
|
||||
return kept
|
||||
|
||||
|
||||
class YoloProposer:
|
||||
"""One lazily-loaded ultralytics YOLO. detect(image) → [(bbox_norm, score,
|
||||
label)] with bbox normalized (x, y, w, h) in [0,1]. Self-disables on any
|
||||
load/inference failure."""
|
||||
|
||||
def __init__(self, name, weights, conf=0.25, keep_labels=None):
|
||||
self.name = name
|
||||
self._spec = weights
|
||||
self._conf = conf
|
||||
self._keep = [k.lower() for k in keep_labels] if keep_labels else None
|
||||
self._model = None
|
||||
self._ok = True
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _load(self):
|
||||
if self._model is not None or not self._ok:
|
||||
return
|
||||
with self._lock:
|
||||
if self._model is not None or not self._ok:
|
||||
return
|
||||
try:
|
||||
from ultralytics import YOLO
|
||||
path = _resolve(self._spec)
|
||||
if path is None:
|
||||
self._ok = False
|
||||
return
|
||||
self._model = YOLO(path)
|
||||
# Disable ultralytics' load-time Conv+BN fusion. AutoBackend fuses
|
||||
# the graph on the first predict; some checkpoints (yolo11n, the
|
||||
# comic-panel model) crash that step with "'Conv' object has no
|
||||
# attribute 'bn'" (a partially-fused / version-mismatched graph),
|
||||
# which silently disabled those proposers (operator-flagged
|
||||
# 2026-07-01). Unfused inference is correct — only marginally
|
||||
# slower — and this is robust across ultralytics versions; if a
|
||||
# future version ignores the override, the detect() guard below
|
||||
# still self-disables the proposer instead of spamming per image.
|
||||
inner = getattr(self._model, "model", None)
|
||||
if inner is not None:
|
||||
inner.fuse = types.MethodType(lambda self, *a, **k: self, inner)
|
||||
log.info("detector %s loaded (%s)", self.name, path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("detector %s disabled (load failed): %s", self.name, exc)
|
||||
self._ok = False
|
||||
|
||||
def detect(self, image):
|
||||
self._load()
|
||||
if self._model is None:
|
||||
return []
|
||||
try:
|
||||
res = self._model.predict(image, conf=self._conf, verbose=False)[0]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Permanently self-disable on the FIRST inference failure rather than
|
||||
# re-throwing (and re-logging) on every image forever — an unfixable
|
||||
# model fault degrades to "this proposer is off", logged once.
|
||||
log.warning("detector %s disabled (inference failed): %s", self.name, exc)
|
||||
self._ok = False
|
||||
self._model = None
|
||||
return []
|
||||
iw, ih = image.size
|
||||
names = getattr(res, "names", None) or {}
|
||||
out = []
|
||||
for b in res.boxes:
|
||||
label = str(names.get(int(b.cls), int(b.cls))).lower()
|
||||
if self._keep is not None and not any(k in label for k in self._keep):
|
||||
continue
|
||||
x0, y0, x1, y1 = (float(v) for v in b.xyxy[0].tolist())
|
||||
out.append((
|
||||
(x0 / iw, y0 / ih, (x1 - x0) / iw, (y1 - y0) / ih),
|
||||
float(b.conf), label,
|
||||
))
|
||||
return out
|
||||
|
||||
def unload(self) -> None:
|
||||
"""Drop the loaded YOLO so its VRAM can be reclaimed; detect() reloads it
|
||||
lazily on the next job. Leaves _ok untouched — a healthy proposer comes
|
||||
back, but one that self-disabled on a fault stays off."""
|
||||
with self._lock:
|
||||
self._model = None
|
||||
|
||||
|
||||
class Proposers:
|
||||
"""The agent's proposer set, built from config. Each detector is optional —
|
||||
an empty weight spec leaves that proposer off."""
|
||||
|
||||
def __init__(self, cfg):
|
||||
self.cfg = cfg
|
||||
self._person = (
|
||||
YoloProposer("person-coco", cfg.person_weights,
|
||||
conf=cfg.person_conf, keep_labels=["person"])
|
||||
if cfg.person_weights else None
|
||||
)
|
||||
self._anatomy = (
|
||||
YoloProposer("anatomy", cfg.anatomy_weights, conf=cfg.anatomy_conf)
|
||||
if cfg.anatomy_weights else None
|
||||
)
|
||||
self._panel = (
|
||||
YoloProposer("panel", cfg.panel_weights, conf=cfg.panel_conf)
|
||||
if cfg.panel_weights else None
|
||||
)
|
||||
|
||||
def figures(self, image, base_boxes):
|
||||
"""Merge imgutils person boxes (base_boxes: [(bbox, score)]) with the
|
||||
general COCO person detector → NMS'd figure boxes [(bbox, score, label)],
|
||||
capped to the highest-scoring max_figures. Uncapped, a busy/huge image
|
||||
(many characters) yields hundreds of boxes → hundreds of per-figure CCIP
|
||||
calls + crops → a 30s+ job and an oversized submit (operator-flagged)."""
|
||||
boxes = [(bb, sc if sc is not None else 1.0, "person") for bb, sc in base_boxes]
|
||||
if self._person is not None:
|
||||
boxes += self._person.detect(image)
|
||||
return nms_merge(boxes)[: self.cfg.max_figures] # nms_merge is score-desc
|
||||
|
||||
@staticmethod
|
||||
def _top(detector, image, cap: int):
|
||||
"""Top-`cap` detections by score from an optional proposer (None → the
|
||||
proposer is off → []). Shared by the anatomy + panel proposers, which
|
||||
differ only in which detector and which cap."""
|
||||
if detector is None:
|
||||
return []
|
||||
return sorted(detector.detect(image), key=lambda b: b[1], reverse=True)[:cap]
|
||||
|
||||
def components(self, image):
|
||||
return self._top(self._anatomy, image, self.cfg.max_components)
|
||||
|
||||
def panels(self, image):
|
||||
return self._top(self._panel, image, self.cfg.max_panels)
|
||||
|
||||
def unload(self) -> None:
|
||||
"""Release every loaded proposer's YOLO (idle VRAM reclaim). The worker
|
||||
also drops its reference to this Proposers and rebuilds a fresh one via
|
||||
_proposers_for on the next job, so this is belt-and-braces."""
|
||||
for p in (self._person, self._anatomy, self._panel):
|
||||
if p is not None:
|
||||
p.unload()
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Crop EMBEDDER for the concept bag — model-agnostic (CLIP/SigLIP-family).
|
||||
|
||||
The server trains its per-concept heads in the embedding space of whatever model
|
||||
its `embedder_model_version` names; a crop must be embedded with the SAME model
|
||||
or its vector lands in a different coordinate system and every head misfires. So
|
||||
the model identity (HF name + version) is ANNOUNCED BY THE SERVER in the lease —
|
||||
nothing here is hardcoded to SigLIP. Whatever name the server sends is loaded via
|
||||
transformers `get_image_features` (the CLIP/SigLIP-family image-tower call); a
|
||||
non-CLIP backbone (e.g. a DINO encoder) would need its own pooling adapter.
|
||||
|
||||
torch on CUDA, fp16 by default to keep VRAM low on a shared desktop GPU — the
|
||||
tiny fp16-vs-fp32 difference is negligible for the linear heads (cosine ~0.999).
|
||||
A single inference lock serializes the forward pass: the pipeline is I/O-bound,
|
||||
so the GPU isn't the bottleneck, and one model shared across worker threads is
|
||||
safest behind a lock.
|
||||
"""
|
||||
import threading
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
|
||||
class CropEmbedder:
|
||||
def __init__(self, model_name: str, dtype: str = "float16"):
|
||||
self._name = model_name
|
||||
self._dtype_name = dtype
|
||||
self._model = None
|
||||
self._processor = None
|
||||
self._torch = None
|
||||
self._device = None
|
||||
self._dt = None
|
||||
self._load_lock = threading.Lock()
|
||||
self._infer_lock = threading.Lock()
|
||||
|
||||
@property
|
||||
def model_name(self) -> str:
|
||||
return self._name
|
||||
|
||||
def load(self) -> None:
|
||||
if self._model is not None:
|
||||
return
|
||||
with self._load_lock:
|
||||
if self._model is not None:
|
||||
return
|
||||
import torch
|
||||
from transformers import AutoImageProcessor, AutoModel
|
||||
|
||||
self._torch = torch
|
||||
self._device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
dt = getattr(torch, self._dtype_name, torch.float16)
|
||||
if self._device == "cpu":
|
||||
dt = torch.float32 # fp16 matmul is unsupported/slow on CPU
|
||||
self._dt = dt
|
||||
self._processor = AutoImageProcessor.from_pretrained(self._name)
|
||||
model = AutoModel.from_pretrained(self._name, torch_dtype=dt)
|
||||
model.eval().to(self._device)
|
||||
self._model = model
|
||||
|
||||
def embed(self, image: Image.Image) -> list[float]:
|
||||
"""A crop → its embedding as a plain float list, ready to POST."""
|
||||
return self.embed_batch([image])[0]
|
||||
|
||||
def embed_batch(self, images: list) -> list[list[float]]:
|
||||
"""Embed many crops in ONE forward pass — far better GPU utilisation +
|
||||
only one lock acquisition than embedding each crop separately (which
|
||||
starved the GPU and serialised the whole pool)."""
|
||||
if not images:
|
||||
return []
|
||||
self.load()
|
||||
torch = self._torch
|
||||
enc = self._processor(images=images, return_tensors="pt")
|
||||
pixel_values = enc["pixel_values"].to(self._device, self._dt)
|
||||
with self._infer_lock, torch.no_grad():
|
||||
out = self._model.get_image_features(pixel_values=pixel_values)
|
||||
pooled = out.pooler_output if hasattr(out, "pooler_output") else out
|
||||
arr = pooled.float().cpu().numpy().astype(np.float32)
|
||||
return [row.reshape(-1).tolist() for row in arr]
|
||||
|
||||
def unload(self) -> bool:
|
||||
"""Drop the loaded model so its VRAM can be reclaimed — the idle monitor
|
||||
calls this after a spell with no work so an idle agent doesn't squat on
|
||||
the card; the next embed() reloads it lazily (a few seconds). Held under
|
||||
BOTH the load and inference locks so it can never race a concurrent load
|
||||
or an in-flight forward pass. Returns True if a model was actually
|
||||
released (the caller then runs one empty_cache() to hand the freed blocks
|
||||
back to the driver)."""
|
||||
with self._load_lock, self._infer_lock:
|
||||
if self._model is None:
|
||||
return False
|
||||
self._model = None
|
||||
self._processor = None
|
||||
return True
|
||||
@@ -1,65 +0,0 @@
|
||||
"""GPU load readout via nvidia-smi (present in the container thanks to the
|
||||
NVIDIA Container Toolkit's `utility` capability). Returns None if unavailable —
|
||||
the UI just shows n/a (e.g. CPU-fallback run).
|
||||
|
||||
Reads are CACHED and de-duplicated: the UI meter polls fast, /status reads it,
|
||||
and the autoscaler samples it — if each spawned its own `nvidia-smi` (slow on a
|
||||
busy GPU) those blocking subprocesses would pile up in the server's thread pool
|
||||
and make the Start/Stop buttons feel dead. So a short TTL serves recent callers
|
||||
from cache, and only ONE probe runs at a time (others get the last value)."""
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
_TTL = 1.0 # seconds a sample is reused before re-probing
|
||||
_lock = threading.Lock()
|
||||
_cache: dict | None = None
|
||||
_cache_t = 0.0
|
||||
_probing = False
|
||||
|
||||
|
||||
def _probe() -> dict | None:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[
|
||||
"nvidia-smi",
|
||||
"--query-gpu=utilization.gpu,memory.used,memory.total,temperature.gpu",
|
||||
"--format=csv,noheader,nounits",
|
||||
],
|
||||
capture_output=True, text=True, timeout=5, check=True,
|
||||
).stdout.strip().splitlines()
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
if not out:
|
||||
return None
|
||||
parts = [p.strip() for p in out[0].split(",")]
|
||||
try:
|
||||
return {
|
||||
"util_pct": int(float(parts[0])),
|
||||
"mem_used_mb": int(float(parts[1])),
|
||||
"mem_total_mb": int(float(parts[2])),
|
||||
"temp_c": int(float(parts[3])),
|
||||
}
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def read_gpu(max_age: float = _TTL) -> dict | None:
|
||||
"""Latest GPU reading, cached. Serves from cache when fresh; when stale,
|
||||
exactly one caller re-probes while the rest get the last value — so request
|
||||
threads never block behind more than one `nvidia-smi`."""
|
||||
global _cache, _cache_t, _probing
|
||||
now = time.monotonic()
|
||||
with _lock:
|
||||
fresh = _cache is not None and (now - _cache_t) < max_age
|
||||
if fresh or _probing: # fresh, or a probe is already running
|
||||
return _cache
|
||||
_probing = True
|
||||
try:
|
||||
val = _probe()
|
||||
finally:
|
||||
with _lock:
|
||||
_cache = val
|
||||
_cache_t = time.monotonic()
|
||||
_probing = False
|
||||
return val
|
||||
@@ -1,44 +0,0 @@
|
||||
"""In-memory log ring buffer so the control UI can show recent agent logs
|
||||
(detector loads, job errors, autoscaler decisions, outage back-offs) without
|
||||
needing `docker logs`. A bounded deque holds the last N formatted lines; a
|
||||
logging.Handler appends to it; the UI polls /logs."""
|
||||
import logging
|
||||
from collections import deque
|
||||
|
||||
LINES: deque[str] = deque(maxlen=400)
|
||||
|
||||
|
||||
class RingHandler(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
try:
|
||||
LINES.append(self.format(record))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
_installed = False
|
||||
|
||||
|
||||
def install(level: int = logging.INFO) -> None:
|
||||
"""Attach the ring handler to the root logger once. fc_agent module loggers
|
||||
propagate to root, so their records land here."""
|
||||
global _installed
|
||||
if _installed:
|
||||
return
|
||||
_installed = True
|
||||
h = RingHandler()
|
||||
h.setFormatter(
|
||||
logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s", "%H:%M:%S")
|
||||
)
|
||||
root = logging.getLogger()
|
||||
root.addHandler(h)
|
||||
if root.level == logging.NOTSET or root.level > level:
|
||||
root.setLevel(level)
|
||||
# Keep the buffer signal-rich: silence the chatty HTTP/download libs (every
|
||||
# HF model fetch logs per-request) so the console shows agent activity —
|
||||
# detector loads, job errors, autoscale moves — not request spam.
|
||||
for noisy in (
|
||||
"uvicorn.access", "ultralytics", "httpx", "httpcore",
|
||||
"huggingface_hub", "urllib3", "filelock",
|
||||
):
|
||||
logging.getLogger(noisy).setLevel(logging.WARNING)
|
||||
@@ -1,253 +0,0 @@
|
||||
"""Image + video handling. Stills load directly; videos are sampled into frames
|
||||
(ffmpeg) at the cadence FC sends — so a video becomes a bag of per-frame
|
||||
instances, each with a timestamp."""
|
||||
import io
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from PIL import Image, ImageFile
|
||||
|
||||
from .throttle import PidReadMeter
|
||||
|
||||
log = logging.getLogger("fc_agent.media")
|
||||
|
||||
# Load slightly-truncated images (a few missing trailing bytes) instead of
|
||||
# raising — matches the server embedder. These are common in scraped libraries
|
||||
# and would otherwise fail the job 3× then error (operator-flagged 2026-06-30).
|
||||
ImageFile.LOAD_TRUNCATED_IMAGES = True
|
||||
|
||||
# Disable PIL's decompression-bomb guard: this is a TRUSTED local library, not an
|
||||
# untrusted upload surface, so a legitimately huge image (high-res scans/prints,
|
||||
# 90M+ pixels) must load. The default 89M-pixel limit only WARNS, but PIL raises
|
||||
# DecompressionBombError at 2× (~179M px) — which would fail those jobs outright
|
||||
# (operator-flagged 2026-06-30, images of 90–95M px).
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
|
||||
|
||||
def is_video(mime: str) -> bool:
|
||||
return bool(mime) and (mime.startswith("video/") or mime in {"image/gif"})
|
||||
|
||||
|
||||
def _dhash(img: Image.Image, size: int = 8) -> int:
|
||||
"""Difference hash: compare adjacent pixels of a (size+1 × size) grayscale
|
||||
thumbnail → a `size*size`-bit fingerprint. Cheap (64 comparisons on a 72-px
|
||||
thumbnail) and robust to scaling/compression noise — near-identical frames
|
||||
hash within a few bits, a real scene change moves many."""
|
||||
small = img.convert("L").resize((size + 1, size))
|
||||
px = list(small.getdata())
|
||||
bits = 0
|
||||
for row in range(size):
|
||||
base = row * (size + 1)
|
||||
for col in range(size):
|
||||
bits = (bits << 1) | int(px[base + col] > px[base + col + 1])
|
||||
return bits
|
||||
|
||||
|
||||
def dedupe_frames(
|
||||
frames: list[tuple[float, Image.Image]], min_distance: int
|
||||
) -> list[tuple[float, Image.Image]]:
|
||||
"""Drop visually near-duplicate frames. A near-static video sampled into many
|
||||
frames re-runs the WHOLE detect→CCIP→SigLIP chain on ~identical frames — the
|
||||
dominant video load. Greedy perceptual-hash dedup: keep a frame only if its
|
||||
dHash differs from every already-kept frame by >= min_distance bits (Hamming),
|
||||
so a static run collapses to one frame while genuinely distinct scenes all
|
||||
survive. Order + timestamps preserved. CPU-only (64-bit int XORs), so it runs
|
||||
in the decode stage and spares the GPU the skipped frames entirely.
|
||||
|
||||
min_distance is the coarseness dial: higher keeps more frames (safer for brief
|
||||
localized changes an 8×8 hash can miss), 0 disables. The first frame is always
|
||||
kept (nothing to compare against)."""
|
||||
if min_distance <= 0 or len(frames) <= 1:
|
||||
return frames
|
||||
kept: list[tuple[float, Image.Image]] = []
|
||||
hashes: list[int] = []
|
||||
for t, frame in frames:
|
||||
h = _dhash(frame)
|
||||
if all(bin(h ^ k).count("1") >= min_distance for k in hashes):
|
||||
hashes.append(h)
|
||||
kept.append((t, frame))
|
||||
return kept
|
||||
|
||||
|
||||
def to_rgb(img: Image.Image) -> Image.Image:
|
||||
"""RGB, flattening any transparency onto white first. A naive convert('RGB')
|
||||
on a palette-with-transparency image (common for character PNGs on a clear
|
||||
background) lets PIL guess the transparent pixels — usually black artifacts
|
||||
that bleed into the crop + the embedding (and the "should be converted to
|
||||
RGBA" warning). Compositing over white gives a clean, consistent background."""
|
||||
if img.mode in ("RGBA", "LA", "PA") or (
|
||||
img.mode == "P" and "transparency" in img.info
|
||||
):
|
||||
img = img.convert("RGBA")
|
||||
bg = Image.new("RGBA", img.size, (255, 255, 255, 255))
|
||||
return Image.alpha_composite(bg, img).convert("RGB")
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def load_image(data: bytes) -> Image.Image:
|
||||
return to_rgb(Image.open(io.BytesIO(data)))
|
||||
|
||||
|
||||
# ffmpeg reconnect flags — resume a dropped HTTP transfer (a slow/contended media
|
||||
# store can cut a long stream) instead of failing the whole job. Relies only on
|
||||
# HTTP + Range, which every FC deployment serves → environment-agnostic.
|
||||
_RECONNECT = [
|
||||
"-reconnect", "1", "-reconnect_streamed", "1",
|
||||
"-reconnect_on_network_error", "1", "-reconnect_delay_max", "5",
|
||||
]
|
||||
|
||||
|
||||
def _collect_frames(
|
||||
tmp: str, interval: float, cap: int
|
||||
) -> list[tuple[float, Image.Image]]:
|
||||
out: list[tuple[float, Image.Image]] = []
|
||||
names = sorted(n for n in os.listdir(tmp) if n.startswith("f_"))
|
||||
for i, name in enumerate(names[:cap]):
|
||||
with Image.open(os.path.join(tmp, name)) as im:
|
||||
out.append((round(i * interval, 2), to_rgb(im)))
|
||||
return out
|
||||
|
||||
|
||||
def _terminate(proc: subprocess.Popen) -> None:
|
||||
"""Stop an ffmpeg cleanly, then hard-kill if it ignores SIGTERM."""
|
||||
try:
|
||||
# A bandwidth-paused (SIGSTOPped) process can't receive SIGTERM until it
|
||||
# resumes — always CONT first so termination is prompt, not queued.
|
||||
proc.send_signal(signal.SIGCONT)
|
||||
except OSError:
|
||||
pass
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
try:
|
||||
proc.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
|
||||
|
||||
def _pause(proc: subprocess.Popen, seconds: float, should_stop) -> bool:
|
||||
"""SIGSTOP ffmpeg for ~`seconds` of bandwidth debt, staying responsive to
|
||||
Stop. While paused, the kernel socket buffer fills and TCP flow control
|
||||
stalls curator's send side — that's the throttle. SIGCONT is ALWAYS sent
|
||||
before returning. False = a Stop arrived mid-pause."""
|
||||
try:
|
||||
proc.send_signal(signal.SIGSTOP)
|
||||
except OSError:
|
||||
return True # already exited — nothing to pause
|
||||
try:
|
||||
end = time.monotonic() + seconds
|
||||
while (left := end - time.monotonic()) > 0:
|
||||
if should_stop and should_stop():
|
||||
return False
|
||||
time.sleep(min(0.5, left))
|
||||
return True
|
||||
finally:
|
||||
try:
|
||||
proc.send_signal(signal.SIGCONT)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def sample_frames_from_url(
|
||||
url: str, interval_seconds: float, max_frames: int,
|
||||
*, headers: str = "", timeout: float = 1200.0, should_stop=None,
|
||||
governor=None,
|
||||
) -> tuple[list[tuple[float, Image.Image]], str | None]:
|
||||
"""Sample frames by pointing ffmpeg STRAIGHT at the media URL — it Range-reads
|
||||
only the video index + up to max_frames worth of content, so the agent never
|
||||
downloads the whole file (VR/4K originals run 800MB+ and would buffer ~1GB in
|
||||
RAM and get cut off mid-download). Reconnect flags resume a dropped transfer;
|
||||
the timeout is the per-video ceiling (a slow/reconnecting stream can otherwise
|
||||
run for minutes). `should_stop` is polled while ffmpeg runs so a Stop KILLS the
|
||||
subprocess at once — otherwise a downloader stuck in a long decode keeps the
|
||||
agent "working" long after Stop. `governor` (the worker's shared TokenBucket)
|
||||
meters ffmpeg's network reads from outside via /proc/<pid>/io and SIGSTOPs
|
||||
the process into budget, so video streaming honors the same aggregate
|
||||
bandwidth cap as still downloads.
|
||||
|
||||
Returns (frames, reason): frames is empty on failure/stop/timeout, and
|
||||
`reason` then carries the SPECIFIC cause (ffmpeg's stderr tail / timeout) so
|
||||
the caller can put it in the job's error — a bare "no frames" hid a filter
|
||||
bug as "unprocessable" for weeks. None reason on success."""
|
||||
interval = max(0.5, float(interval_seconds or 4.0))
|
||||
cap = max(1, int(max_frames or 64))
|
||||
hdr = ["-headers", headers] if headers else []
|
||||
# select (NOT the fps filter): always keep the FIRST frame, then one per
|
||||
# `interval` seconds of timestamp. fps=1/N emits round(duration/N) frames,
|
||||
# which is ZERO for any clip shorter than ~N/2 seconds — a whole class of
|
||||
# short animation loops failed as "unprocessable" that way (operator-flagged
|
||||
# 2026-07-02: 0.5s/1.75s clips). scale=out_range=full converts limited-range
|
||||
# yuv420p to full range so the mjpeg (jpg) encoder accepts it at default
|
||||
# strictness instead of erroring on "non full-range YUV".
|
||||
vf = (
|
||||
f"select='isnan(prev_selected_t)+gte(t-prev_selected_t\\,{interval})',"
|
||||
"scale=out_range=full"
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
pattern = os.path.join(tmp, "f_%05d.jpg")
|
||||
cmd = ["ffmpeg", "-nostdin", "-loglevel", "error", *_RECONNECT, *hdr,
|
||||
"-i", url, "-vf", vf, "-fps_mode", "vfr",
|
||||
"-frames:v", str(cap), "-q:v", "3", pattern]
|
||||
# ffmpeg's stderr goes to a file (not a PIPE, which could fill and
|
||||
# deadlock; not DEVNULL, which is how a filter bug hid as "unprocessable"
|
||||
# for weeks) — on failure its tail is logged so the operator can see WHY.
|
||||
errpath = os.path.join(tmp, "stderr.txt")
|
||||
try:
|
||||
with open(errpath, "wb") as errf:
|
||||
proc = subprocess.Popen(
|
||||
cmd, stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL, stderr=errf,
|
||||
)
|
||||
meter = PidReadMeter(proc.pid) if governor is not None else None
|
||||
# Poll rather than block, so a Stop (or the per-video timeout) can
|
||||
# kill a slow/wedged ffmpeg promptly instead of waiting it out.
|
||||
start = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
proc.wait(timeout=0.5)
|
||||
break
|
||||
except subprocess.TimeoutExpired:
|
||||
stopped = should_stop and should_stop()
|
||||
if stopped or (time.monotonic() - start > timeout):
|
||||
_terminate(proc)
|
||||
if stopped:
|
||||
return [], "stopped"
|
||||
log.warning("ffmpeg timed out after %.0fs: %s",
|
||||
timeout, url)
|
||||
return [], f"ffmpeg timed out after {timeout:.0f}s"
|
||||
if meter is not None:
|
||||
read = meter.delta()
|
||||
if read is None: # /proc gone → stop governing
|
||||
meter = None
|
||||
elif (debt := governor.charge(read)) > 0:
|
||||
# Over budget: pause ffmpeg until the bucket
|
||||
# recovers. Pause time counts toward `timeout`
|
||||
# (it stays the wedge backstop either way).
|
||||
if not _pause(proc, debt, should_stop):
|
||||
_terminate(proc)
|
||||
return [], "stopped"
|
||||
except (OSError, ValueError) as exc:
|
||||
return [], f"ffmpeg not runnable: {exc}"
|
||||
frames = _collect_frames(tmp, interval, cap)
|
||||
if not frames:
|
||||
reason = f"ffmpeg exit {proc.returncode}: {_tail(errpath)}"
|
||||
log.warning("ffmpeg produced no frames for %s — %s", url, reason)
|
||||
return [], reason
|
||||
return frames, None
|
||||
|
||||
|
||||
def _tail(path: str, limit: int = 300) -> str:
|
||||
"""Last `limit` chars of a (stderr) file, flattened — for failure logs."""
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
f.seek(0, os.SEEK_END)
|
||||
f.seek(max(0, f.tell() - limit))
|
||||
return f.read().decode("utf-8", "replace").replace("\n", " ").strip()
|
||||
except OSError:
|
||||
return "?"
|
||||
@@ -1,39 +0,0 @@
|
||||
"""imgutils model wrappers — the figure DETECTOR + the CCIP EMBEDDER.
|
||||
|
||||
⚠️ VERIFY ON FIRST RUN: the exact imgutils function names/signatures + the CCIP
|
||||
model string can drift between dghs-imgutils releases. These are the two seams to
|
||||
check against your installed version (`pip show dghs-imgutils`):
|
||||
- detect_person(image, level=...) -> [((x0,y0,x1,y1), label, score), ...]
|
||||
- ccip_extract_feature(image, model=...) -> a vector (768-d for caformer)
|
||||
imgutils auto-downloads the ONNX models from HuggingFace on first use; GPU is
|
||||
used when onnxruntime-gpu is installed.
|
||||
"""
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def detect_figures(image: Image.Image, level: str = "m") -> list[tuple[tuple, float | None]]:
|
||||
"""Person/figure bounding boxes, NORMALIZED (x, y, w, h in [0,1]) + score.
|
||||
Returns [] if detection finds nothing (caller falls back to whole-image)."""
|
||||
from imgutils.detect import detect_person
|
||||
|
||||
iw, ih = image.size
|
||||
out = []
|
||||
for (x0, y0, x1, y1), _label, score in detect_person(image, level=level):
|
||||
out.append((
|
||||
(x0 / iw, y0 / ih, (x1 - x0) / iw, (y1 - y0) / ih),
|
||||
float(score),
|
||||
))
|
||||
return out
|
||||
|
||||
|
||||
def ccip_vector(image: Image.Image, model: str | None = None) -> list[float]:
|
||||
"""The CCIP identity embedding of a (cropped) character image, as a plain
|
||||
float list ready to POST."""
|
||||
from imgutils.metrics import ccip_extract_feature
|
||||
|
||||
feat = (
|
||||
ccip_extract_feature(image, model=model)
|
||||
if model else ccip_extract_feature(image)
|
||||
)
|
||||
return np.asarray(feat, dtype=np.float32).reshape(-1).tolist()
|
||||
@@ -1,111 +0,0 @@
|
||||
"""Global download-bandwidth governor (one token bucket for the whole agent).
|
||||
|
||||
The agent lives on someone's desktop and shares that desktop's network —
|
||||
typically WiFi, where saturating the link doesn't just slow other apps: it
|
||||
bufferbloats the airtime (RTT 21→45ms) and collapses EVERY connection,
|
||||
the operator's browser included. Measured 2026-07-02: the idle link moved
|
||||
~38 MB/s single-stream, but under the 8-downloader sweep every stream on the
|
||||
machine crawled at ~1-1.5 MB/s. So the cap is on the AGGREGATE, not per
|
||||
stream: still downloads pump their chunks through take(), and ffmpeg video
|
||||
streams — whose sockets live in a subprocess we can't wrap — are metered from
|
||||
outside via /proc/<pid>/io and paused (SIGSTOP) into budget using charge()'s
|
||||
debt signal; TCP flow control then stalls the sender while ffmpeg sleeps.
|
||||
|
||||
Accounting is post-paid (charge the bytes first, then wait out any debt): the
|
||||
bytes have already crossed the network by the time we count them, and it means
|
||||
a chunk larger than one second of budget can never deadlock the bucket.
|
||||
Stdlib-only on purpose — unit-tested in CI, where the agent's ML deps
|
||||
don't exist.
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
class TokenBucket:
|
||||
"""Thread-safe token bucket in bytes/second. rate 0 = unlimited.
|
||||
|
||||
`consumed` is the monotonic total of bytes charged (throttled or not) —
|
||||
the worker's rate loop derives the UI's "net MB/s" readout from it.
|
||||
"""
|
||||
|
||||
def __init__(self, rate_bytes_per_s: float = 0.0):
|
||||
self._cond = threading.Condition()
|
||||
self._rate = max(0.0, float(rate_bytes_per_s))
|
||||
# Burst = one second of budget: enough that chunked reads stay smooth,
|
||||
# small enough that a burst can't meaningfully lift the average.
|
||||
self._level = self._rate
|
||||
self._stamp = time.monotonic()
|
||||
self.consumed = 0
|
||||
|
||||
@property
|
||||
def rate(self) -> float:
|
||||
return self._rate
|
||||
|
||||
def set_rate(self, rate_bytes_per_s: float) -> None:
|
||||
"""Retune live (the UI dial). Waiters re-check immediately, so raising
|
||||
the cap (or lifting it with 0) unblocks a mid-download wait at once."""
|
||||
with self._cond:
|
||||
self._refill_locked() # settle elapsed time at the OLD rate
|
||||
self._rate = max(0.0, float(rate_bytes_per_s))
|
||||
self._level = min(self._level, self._rate)
|
||||
self._cond.notify_all()
|
||||
|
||||
def _refill_locked(self) -> None:
|
||||
now = time.monotonic()
|
||||
self._level = min(self._rate, self._level + (now - self._stamp) * self._rate)
|
||||
self._stamp = now
|
||||
|
||||
def take(self, n: int) -> None:
|
||||
"""Charge n bytes and block until the budget recovers (stills path)."""
|
||||
with self._cond:
|
||||
self.consumed += n
|
||||
if self._rate <= 0:
|
||||
return
|
||||
self._refill_locked()
|
||||
self._level -= n
|
||||
while self._level < 0:
|
||||
# Wake early on set_rate; cap the wait so a big debt is paid in
|
||||
# re-checked slices rather than one long uninterruptible sleep.
|
||||
self._cond.wait(min(-self._level / self._rate, 0.5))
|
||||
if self._rate <= 0:
|
||||
return
|
||||
self._refill_locked()
|
||||
|
||||
def charge(self, n: int) -> float:
|
||||
"""Charge n bytes WITHOUT blocking; return seconds of debt (0 = within
|
||||
budget). The ffmpeg governor can't block the subprocess's own reads, so
|
||||
it SIGSTOPs the process for (about) the returned debt instead."""
|
||||
with self._cond:
|
||||
self.consumed += n
|
||||
if self._rate <= 0:
|
||||
return 0.0
|
||||
self._refill_locked()
|
||||
self._level -= n
|
||||
return max(0.0, -self._level / self._rate)
|
||||
|
||||
|
||||
class PidReadMeter:
|
||||
"""Cumulative read-bytes meter for a subprocess, via /proc/<pid>/io.
|
||||
|
||||
`rchar` counts every read() syscall's bytes — for a streaming ffmpeg the
|
||||
network reads dominate, so the delta is a good-enough aggregate-bandwidth
|
||||
signal (it's a governor, not a billing meter). Returns None when /proc is
|
||||
unavailable (process exited, or a non-Linux host): the caller then simply
|
||||
doesn't govern — degrade to unthrottled rather than break video sampling.
|
||||
"""
|
||||
|
||||
def __init__(self, pid: int):
|
||||
self._path = f"/proc/{pid}/io"
|
||||
self._last = 0
|
||||
|
||||
def delta(self) -> int | None:
|
||||
try:
|
||||
with open(self._path, "rb") as f:
|
||||
for line in f:
|
||||
if line.startswith(b"rchar:"):
|
||||
total = int(line.split()[1])
|
||||
d, self._last = total - self._last, total
|
||||
return max(0, d)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
return None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,20 +0,0 @@
|
||||
# CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace).
|
||||
dghs-imgutils>=0.4
|
||||
# GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
|
||||
# server-side fallback run. The extras declare the CUDA/cuDNN pip packages its
|
||||
# CUDA provider loads (fc_agent/accel.py preloads them) rather than relying on
|
||||
# torch happening to install the same ones.
|
||||
onnxruntime-gpu[cuda,cudnn]
|
||||
# The crop EMBEDDER (concept bag). torch + torchvision are installed separately
|
||||
# in the Dockerfile from the cu130 wheel index, so pip never swaps them out;
|
||||
# transformers loads whatever SigLIP-family model the server announces.
|
||||
transformers>=4.45
|
||||
# Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic
|
||||
# panel) that decide where to crop. Uses the torch already installed above.
|
||||
ultralytics>=8.3
|
||||
# Control surface + HTTP.
|
||||
fastapi
|
||||
uvicorn[standard]
|
||||
requests
|
||||
pillow
|
||||
numpy
|
||||
@@ -1,7 +0,0 @@
|
||||
# The agent runs on the CUDA base image's Python 3.12 (Ubuntu 24.04) — NOT the
|
||||
# 3.14 that CI's ci-python image and the repo-root ruff.toml target. Pin the
|
||||
# agent to py312 so ruff enforces 3.12 compatibility and never auto-applies a
|
||||
# 3.14-only fix (e.g. unquoting a self-referential annotation, which PEP 649
|
||||
# makes safe on 3.14 but NameErrors on 3.12). Inherit the root lint rules.
|
||||
extend = "../ruff.toml"
|
||||
target-version = "py312"
|
||||
+1
-25
@@ -1,28 +1,13 @@
|
||||
"""Alembic environment — reads DATABASE_URL from app config."""
|
||||
|
||||
import os
|
||||
import re
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config, pool, text
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from alembic import context
|
||||
from backend.app.config import get_config
|
||||
from backend.app.models import Base
|
||||
|
||||
# Fail a blocked migration FAST instead of hanging forever. Migrations run
|
||||
# against the live DB while workers hold locks; 0040's `ALTER series_page` queued
|
||||
# behind a tag-merge that held a series_page lock for minutes (the merge runs an
|
||||
# unindexed full scan over image_record while repointing series_page) and hung
|
||||
# with no timeout — silent, indefinite (operator-flagged 2026-06-07). With a
|
||||
# lock_timeout a blocked DDL errors ("canceling statement due to lock timeout")
|
||||
# and the entrypoint's `alembic upgrade head` exits non-zero, so the deploy
|
||||
# retries / surfaces loudly rather than wedging. Override via env when a known
|
||||
# slow-lock window is expected.
|
||||
_MIGRATION_LOCK_TIMEOUT = os.environ.get("MIGRATION_LOCK_TIMEOUT", "30s")
|
||||
if not re.fullmatch(r"\d+\s*(ms|s|min)?", _MIGRATION_LOCK_TIMEOUT.strip()):
|
||||
_MIGRATION_LOCK_TIMEOUT = "30s" # ignore a malformed override
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
@@ -53,15 +38,6 @@ def run_migrations_online() -> None:
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
# Session-level lock_timeout for every DDL statement in this run. Set
|
||||
# (and commit) before alembic opens its own transaction so the GUC
|
||||
# persists on this connection regardless of how alembic structures its
|
||||
# transactions. Value is from our own env, so f-string interpolation is
|
||||
# safe (and it's been pattern-validated above); SET takes no bind params.
|
||||
connection.execute(
|
||||
text(f"SET lock_timeout = '{_MIGRATION_LOCK_TIMEOUT}'")
|
||||
)
|
||||
connection.commit()
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
"""initial unified schema
|
||||
|
||||
Revision ID: 0001
|
||||
Revises:
|
||||
Create Date: 2026-05-13
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
revision: str = "0001"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
|
||||
op.create_table(
|
||||
"artist",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("slug", sa.String(length=255), nullable=False),
|
||||
sa.Column("notes", sa.Text(), nullable=True),
|
||||
sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()),
|
||||
sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("check_interval_seconds", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_artist"),
|
||||
sa.UniqueConstraint("name", name="uq_artist_name"),
|
||||
sa.UniqueConstraint("slug", name="uq_artist_slug"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"source",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("artist_id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()),
|
||||
sa.Column("config_overrides", sa.JSON(), nullable=True),
|
||||
sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("check_interval_override", sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_source"),
|
||||
)
|
||||
op.create_index("ix_source_artist_id", "source", ["artist_id"])
|
||||
|
||||
op.create_table(
|
||||
"credential",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False),
|
||||
sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="active"),
|
||||
sa.Column(
|
||||
"captured_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_credential"),
|
||||
sa.UniqueConstraint("platform", name="uq_credential_platform"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"post",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("external_post_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("post_url", sa.Text(), nullable=True),
|
||||
sa.Column("post_title", sa.Text(), nullable=True),
|
||||
sa.Column("post_date", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("raw_metadata", sa.JSON(), nullable=True),
|
||||
sa.Column(
|
||||
"downloaded_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_post"),
|
||||
sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||
)
|
||||
op.create_index("ix_post_source_id", "post", ["source_id"])
|
||||
|
||||
op.create_table(
|
||||
"image_record",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("phash", sa.String(length=32), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column("mime", sa.String(length=64), nullable=False),
|
||||
sa.Column("width", sa.Integer(), nullable=True),
|
||||
sa.Column("height", sa.Integer(), nullable=True),
|
||||
sa.Column("thumbnail_path", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"origin",
|
||||
sa.Enum(
|
||||
"downloaded",
|
||||
"imported_filesystem",
|
||||
"uploaded",
|
||||
name="origin_enum",
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("primary_post_id", sa.Integer(), nullable=True),
|
||||
sa.Column("wd14_predictions", sa.JSON(), nullable=True),
|
||||
sa.Column("wd14_model_version", sa.String(length=128), nullable=True),
|
||||
sa.Column("siglip_embedding", Vector(1152), nullable=True),
|
||||
sa.Column("siglip_model_version", sa.String(length=128), nullable=True),
|
||||
sa.Column("centroid_scores", sa.JSON(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["primary_post_id"],
|
||||
["post.id"],
|
||||
name="fk_image_record_primary_post_id_post",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_image_record"),
|
||||
sa.UniqueConstraint("path", name="uq_image_record_path"),
|
||||
sa.UniqueConstraint("sha256", name="uq_image_record_sha256"),
|
||||
)
|
||||
op.create_index("ix_image_record_sha256", "image_record", ["sha256"])
|
||||
op.create_index("ix_image_record_phash", "image_record", ["phash"])
|
||||
op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"])
|
||||
|
||||
op.create_table(
|
||||
"image_provenance",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||
sa.Column("post_id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("captured_metadata", sa.JSON(), nullable=True),
|
||||
sa.Column(
|
||||
"captured_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_record_id"],
|
||||
["image_record.id"],
|
||||
name="fk_image_provenance_image_record_id_image_record",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["post_id"],
|
||||
["post.id"],
|
||||
name="fk_image_provenance_post_id_post",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
["source.id"],
|
||||
name="fk_image_provenance_source_id_source",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_image_provenance"),
|
||||
)
|
||||
op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"])
|
||||
op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"])
|
||||
op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"])
|
||||
|
||||
op.create_table(
|
||||
"tag",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("namespace", sa.String(length=64), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_tag"),
|
||||
sa.UniqueConstraint("name", name="uq_tag_name"),
|
||||
)
|
||||
op.create_index("ix_tag_name", "tag", ["name"])
|
||||
op.create_index("ix_tag_namespace", "tag", ["namespace"])
|
||||
|
||||
op.create_table(
|
||||
"image_tag",
|
||||
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_record_id"],
|
||||
["image_record.id"],
|
||||
name="fk_image_tag_image_record_id_image_record",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"),
|
||||
)
|
||||
|
||||
op.create_table(
|
||||
"download_event",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("post_id", sa.Integer(), nullable=True),
|
||||
sa.Column("status", sa.String(length=32), nullable=False),
|
||||
sa.Column(
|
||||
"started_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"),
|
||||
sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"],
|
||||
["source.id"],
|
||||
name="fk_download_event_source_id_source",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["post_id"],
|
||||
["post.id"],
|
||||
name="fk_download_event_post_id_post",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_download_event"),
|
||||
)
|
||||
op.create_index("ix_download_event_source_id", "download_event", ["source_id"])
|
||||
op.create_index("ix_download_event_post_id", "download_event", ["post_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("download_event")
|
||||
op.drop_table("image_tag")
|
||||
op.drop_table("tag")
|
||||
op.drop_table("image_provenance")
|
||||
op.drop_table("image_record")
|
||||
op.execute("DROP TYPE IF EXISTS origin_enum")
|
||||
op.drop_table("post")
|
||||
op.drop_table("credential")
|
||||
op.drop_table("source")
|
||||
op.drop_table("artist")
|
||||
op.execute("DROP EXTENSION IF EXISTS vector")
|
||||
@@ -0,0 +1,208 @@
|
||||
"""fc2a: tag kinds, import_task, import_batch, integrity_status
|
||||
|
||||
Revision ID: 0002
|
||||
Revises: 0001
|
||||
Create Date: 2026-05-14
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0002"
|
||||
down_revision: Union[str, None] = "0001"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
TAG_KINDS = (
|
||||
"artist",
|
||||
"character",
|
||||
"fandom",
|
||||
"general",
|
||||
"series",
|
||||
"archive",
|
||||
"post",
|
||||
"meta",
|
||||
"rating",
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# --- Tag kind enum + fandom_id ---
|
||||
tag_kind = sa.Enum(*TAG_KINDS, name="tag_kind")
|
||||
tag_kind.create(op.get_bind(), checkfirst=True)
|
||||
|
||||
op.add_column(
|
||||
"tag",
|
||||
sa.Column("kind", tag_kind, nullable=False, server_default="general"),
|
||||
)
|
||||
op.add_column(
|
||||
"tag",
|
||||
sa.Column("fandom_id", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_tag_fandom_id_tag",
|
||||
"tag",
|
||||
"tag",
|
||||
["fandom_id"],
|
||||
["id"],
|
||||
ondelete="SET NULL",
|
||||
)
|
||||
|
||||
# Drop the old global uniqueness on name; add kind+fandom-aware uniqueness.
|
||||
op.drop_constraint("uq_tag_name", "tag", type_="unique")
|
||||
op.drop_index("ix_tag_name", table_name="tag")
|
||||
op.execute(
|
||||
"""
|
||||
CREATE UNIQUE INDEX uq_tag_name_kind_fandom
|
||||
ON tag (name, kind, COALESCE(fandom_id, 0))
|
||||
"""
|
||||
)
|
||||
|
||||
# CHECK: fandom_id is only allowed for character kind.
|
||||
op.create_check_constraint(
|
||||
"ck_tag_fandom_requires_character",
|
||||
"tag",
|
||||
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||
)
|
||||
|
||||
# Drop the old namespace column — superseded by kind.
|
||||
op.drop_index("ix_tag_namespace", table_name="tag")
|
||||
op.drop_column("tag", "namespace")
|
||||
|
||||
# --- ImportBatch ---
|
||||
op.create_table(
|
||||
"import_batch",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("triggered_by", sa.String(length=32), nullable=False),
|
||||
sa.Column("source_path", sa.Text(), nullable=False),
|
||||
sa.Column("scan_mode", sa.String(length=16), nullable=False),
|
||||
sa.Column(
|
||||
"started_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("total_files", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("imported", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("skipped", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("failed", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("status", sa.String(length=16), nullable=False, server_default="running"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_import_batch"),
|
||||
)
|
||||
op.create_index("ix_import_batch_status", "import_batch", ["status"])
|
||||
|
||||
# --- ImportTask ---
|
||||
op.create_table(
|
||||
"import_task",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("batch_id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_path", sa.Text(), nullable=False),
|
||||
sa.Column("task_type", sa.String(length=16), nullable=False),
|
||||
sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"),
|
||||
sa.Column("result_image_id", sa.Integer(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["batch_id"],
|
||||
["import_batch.id"],
|
||||
name="fk_import_task_batch_id_import_batch",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["result_image_id"],
|
||||
["image_record.id"],
|
||||
name="fk_import_task_result_image_id_image_record",
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_import_task"),
|
||||
)
|
||||
op.create_index("ix_import_task_batch_id", "import_task", ["batch_id"])
|
||||
op.create_index("ix_import_task_status", "import_task", ["status"])
|
||||
op.create_index(
|
||||
"ix_import_task_created_at_desc",
|
||||
"import_task",
|
||||
[sa.text("created_at DESC")],
|
||||
)
|
||||
|
||||
# --- ImportSettings (single-row table) ---
|
||||
op.create_table(
|
||||
"import_settings",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("import_scan_path", sa.Text(), nullable=False, server_default="/import"),
|
||||
sa.Column("min_width", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("min_height", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column(
|
||||
"skip_transparent", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||
),
|
||||
sa.Column(
|
||||
"transparency_threshold",
|
||||
sa.Float(),
|
||||
nullable=False,
|
||||
server_default="0.9",
|
||||
),
|
||||
sa.Column(
|
||||
"skip_single_color", sa.Boolean(), nullable=False, server_default=sa.false()
|
||||
),
|
||||
sa.Column(
|
||||
"single_color_threshold",
|
||||
sa.Float(),
|
||||
nullable=False,
|
||||
server_default="0.95",
|
||||
),
|
||||
sa.Column("single_color_tolerance", sa.Integer(), nullable=False, server_default="30"),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_import_settings"),
|
||||
sa.CheckConstraint("id = 1", name="ck_import_settings_singleton"),
|
||||
)
|
||||
# Seed the single row immediately so callers can always SELECT id=1.
|
||||
op.execute("INSERT INTO import_settings (id) VALUES (1)")
|
||||
|
||||
# --- ImageRecord additions ---
|
||||
op.add_column(
|
||||
"image_record",
|
||||
sa.Column(
|
||||
"integrity_status",
|
||||
sa.String(length=24),
|
||||
nullable=False,
|
||||
server_default="unknown",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_image_record_integrity_status",
|
||||
"image_record",
|
||||
["integrity_status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_image_record_integrity_status", table_name="image_record")
|
||||
op.drop_column("image_record", "integrity_status")
|
||||
|
||||
op.drop_table("import_settings")
|
||||
op.drop_index("ix_import_task_created_at_desc", table_name="import_task")
|
||||
op.drop_index("ix_import_task_status", table_name="import_task")
|
||||
op.drop_index("ix_import_task_batch_id", table_name="import_task")
|
||||
op.drop_table("import_task")
|
||||
op.drop_index("ix_import_batch_status", table_name="import_batch")
|
||||
op.drop_table("import_batch")
|
||||
|
||||
op.drop_constraint("ck_tag_fandom_requires_character", "tag", type_="check")
|
||||
op.execute("DROP INDEX uq_tag_name_kind_fandom")
|
||||
op.add_column("tag", sa.Column("namespace", sa.String(length=64), nullable=True))
|
||||
op.create_index("ix_tag_namespace", "tag", ["namespace"])
|
||||
op.create_index("ix_tag_name", "tag", ["name"], unique=False)
|
||||
op.create_unique_constraint("uq_tag_name", "tag", ["name"])
|
||||
op.drop_constraint("fk_tag_fandom_id_tag", "tag", type_="foreignkey")
|
||||
op.drop_column("tag", "fandom_id")
|
||||
op.drop_column("tag", "kind")
|
||||
sa.Enum(name="tag_kind").drop(op.get_bind(), checkfirst=True)
|
||||
@@ -0,0 +1,172 @@
|
||||
"""fc2b: ML pipeline — allowlist, aliases, centroids, ml_settings
|
||||
|
||||
Revision ID: 0003
|
||||
Revises: 0002
|
||||
Create Date: 2026-05-15
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
revision: str = "0003"
|
||||
down_revision: Union[str, None] = "0002"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# 3.1 rename wd14_* -> tagger_*
|
||||
op.alter_column("image_record", "wd14_predictions", new_column_name="tagger_predictions")
|
||||
op.alter_column(
|
||||
"image_record", "wd14_model_version", new_column_name="tagger_model_version"
|
||||
)
|
||||
|
||||
# 3.2 tag_allowlist
|
||||
op.create_table(
|
||||
"tag_allowlist",
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"min_confidence", sa.Float(), nullable=False, server_default="0.95"
|
||||
),
|
||||
sa.Column(
|
||||
"added_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"], name="fk_tag_allowlist_tag_id_tag",
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("tag_id", name="pk_tag_allowlist"),
|
||||
sa.CheckConstraint(
|
||||
"min_confidence > 0 AND min_confidence <= 1",
|
||||
name="ck_tag_allowlist_confidence_range",
|
||||
),
|
||||
)
|
||||
|
||||
# 3.3 tag_suggestion_rejection
|
||||
op.create_table(
|
||||
"tag_suggestion_rejection",
|
||||
sa.Column("image_record_id", sa.Integer(), nullable=False),
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"rejected_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_record_id"], ["image_record.id"],
|
||||
name="fk_tsr_image_record_id_image_record", ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"], name="fk_tsr_tag_id_tag", ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"image_record_id", "tag_id", name="pk_tag_suggestion_rejection"
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_tag_suggestion_rejection_tag", "tag_suggestion_rejection", ["tag_id"]
|
||||
)
|
||||
|
||||
# 3.4 tag_alias
|
||||
op.create_table(
|
||||
"tag_alias",
|
||||
sa.Column("alias_string", sa.String(length=255), nullable=False),
|
||||
sa.Column("alias_category", sa.String(length=32), nullable=False),
|
||||
sa.Column("canonical_tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["canonical_tag_id"], ["tag.id"],
|
||||
name="fk_tag_alias_canonical_tag_id_tag", ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"alias_string", "alias_category", name="pk_tag_alias"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_tag_alias_canonical", "tag_alias", ["canonical_tag_id"])
|
||||
|
||||
# 3.5 tag_reference_embedding (centroids)
|
||||
op.create_table(
|
||||
"tag_reference_embedding",
|
||||
sa.Column("tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column("embedding", Vector(1152), nullable=False),
|
||||
sa.Column("reference_count", sa.Integer(), nullable=False),
|
||||
sa.Column("model_version", sa.String(length=128), nullable=False),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["tag_id"], ["tag.id"],
|
||||
name="fk_tag_reference_embedding_tag_id_tag", ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("tag_id", name="pk_tag_reference_embedding"),
|
||||
)
|
||||
|
||||
# 3.6 ml_settings singleton
|
||||
op.create_table(
|
||||
"ml_settings",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"suggestion_threshold_artist", sa.Float(), nullable=False,
|
||||
server_default="0.30",
|
||||
),
|
||||
sa.Column(
|
||||
"suggestion_threshold_character", sa.Float(), nullable=False,
|
||||
server_default="0.50",
|
||||
),
|
||||
sa.Column(
|
||||
"suggestion_threshold_copyright", sa.Float(), nullable=False,
|
||||
server_default="0.50",
|
||||
),
|
||||
sa.Column(
|
||||
"suggestion_threshold_general", sa.Float(), nullable=False,
|
||||
server_default="0.95",
|
||||
),
|
||||
sa.Column(
|
||||
"centroid_similarity_threshold", sa.Float(), nullable=False,
|
||||
server_default="0.55",
|
||||
),
|
||||
sa.Column(
|
||||
"min_reference_images", sa.Integer(), nullable=False,
|
||||
server_default="5",
|
||||
),
|
||||
sa.Column(
|
||||
"tagger_model_version", sa.String(length=128), nullable=False,
|
||||
server_default="camie-tagger-v2",
|
||||
),
|
||||
sa.Column(
|
||||
"embedder_model_version", sa.String(length=128), nullable=False,
|
||||
server_default="siglip-so400m-patch14-384",
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True), nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name="pk_ml_settings"),
|
||||
sa.CheckConstraint("id = 1", name="ck_ml_settings_singleton"),
|
||||
)
|
||||
op.execute("INSERT INTO ml_settings (id) VALUES (1)")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("ml_settings")
|
||||
op.drop_table("tag_reference_embedding")
|
||||
op.drop_index("ix_tag_alias_canonical", table_name="tag_alias")
|
||||
op.drop_table("tag_alias")
|
||||
op.drop_index(
|
||||
"ix_tag_suggestion_rejection_tag", table_name="tag_suggestion_rejection"
|
||||
)
|
||||
op.drop_table("tag_suggestion_rejection")
|
||||
op.drop_table("tag_allowlist")
|
||||
op.alter_column(
|
||||
"image_record", "tagger_model_version", new_column_name="wd14_model_version"
|
||||
)
|
||||
op.alter_column(
|
||||
"image_record", "tagger_predictions", new_column_name="wd14_predictions"
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
"""fc2c-i: enable tsm_system_rows for scalable random sampling
|
||||
|
||||
Revision ID: 0004
|
||||
Revises: 0003
|
||||
Create Date: 2026-05-15
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0004"
|
||||
down_revision: Union[str, None] = "0003"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.execute("DROP EXTENSION IF EXISTS tsm_system_rows")
|
||||
@@ -0,0 +1,50 @@
|
||||
"""fc2c-iii-a: series_page ordered membership
|
||||
|
||||
Revision ID: 0005
|
||||
Revises: 0004
|
||||
Create Date: 2026-05-16
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0005"
|
||||
down_revision: Union[str, None] = "0004"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"series_page",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("series_tag_id", sa.Integer(), nullable=False),
|
||||
sa.Column("image_id", sa.Integer(), nullable=False),
|
||||
sa.Column("page_number", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["series_tag_id"], ["tag.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["image_id"], ["image_record.id"], ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("image_id", name="uq_series_page_image"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_series_page_series_tag_id", "series_page", ["series_tag_id"]
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_series_page_series_tag_id", table_name="series_page")
|
||||
op.drop_table("series_page")
|
||||
@@ -0,0 +1,30 @@
|
||||
"""fc2d: import_settings.phash_threshold
|
||||
|
||||
Revision ID: 0006
|
||||
Revises: 0005
|
||||
Create Date: 2026-05-17
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0006"
|
||||
down_revision: Union[str, None] = "0005"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"phash_threshold", sa.Integer(),
|
||||
nullable=False, server_default="10",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "phash_threshold")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""fc2d-iv: post.description + post.attachment_count
|
||||
|
||||
Revision ID: 0007
|
||||
Revises: 0006
|
||||
Create Date: 2026-05-18
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0007"
|
||||
down_revision: Union[str, None] = "0006"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"post", sa.Column("description", sa.Text(), nullable=True)
|
||||
)
|
||||
op.add_column(
|
||||
"post",
|
||||
sa.Column("attachment_count", sa.Integer(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("post", "attachment_count")
|
||||
op.drop_column("post", "description")
|
||||
@@ -0,0 +1,52 @@
|
||||
"""fc2d-vii-c: image_record.artist_id + backfill + drop artist tags
|
||||
|
||||
Revision ID: 0008
|
||||
Revises: 0007
|
||||
Create Date: 2026-05-18
|
||||
|
||||
Internal forward-correctness migration (the big legacy-import migration
|
||||
stays deferred). downgrade() does NOT recreate deleted artist tags;
|
||||
downgrade is dev-only and the data is reconstructable by re-import.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
from backend.app.utils.artist_backfill import (
|
||||
BACKFILL_PRIMARY_SQL,
|
||||
BACKFILL_PROVENANCE_SQL,
|
||||
BACKFILL_TAG_SQL,
|
||||
DELETE_ARTIST_TAGS_SQL,
|
||||
)
|
||||
|
||||
revision: str = "0008"
|
||||
down_revision: Union[str, None] = "0007"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"image_record",
|
||||
sa.Column("artist_id", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_foreign_key(
|
||||
"fk_image_record_artist_id", "image_record", "artist",
|
||||
["artist_id"], ["id"], ondelete="SET NULL",
|
||||
)
|
||||
op.create_index(
|
||||
"ix_image_record_artist_id", "image_record", ["artist_id"],
|
||||
)
|
||||
op.execute(BACKFILL_PRIMARY_SQL)
|
||||
op.execute(BACKFILL_PROVENANCE_SQL)
|
||||
op.execute(BACKFILL_TAG_SQL)
|
||||
op.execute(DELETE_ARTIST_TAGS_SQL)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_image_record_artist_id", table_name="image_record")
|
||||
op.drop_constraint(
|
||||
"fk_image_record_artist_id", "image_record", type_="foreignkey"
|
||||
)
|
||||
op.drop_column("image_record", "artist_id")
|
||||
@@ -0,0 +1,68 @@
|
||||
"""fc2d-iii: post_attachment + import_batch.attachments
|
||||
|
||||
Revision ID: 0009
|
||||
Revises: 0008
|
||||
Create Date: 2026-05-19
|
||||
|
||||
Internal forward-correctness migration (big legacy-import migration
|
||||
stays deferred). No backfill — no attachments exist yet.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0009"
|
||||
down_revision: Union[str, None] = "0008"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"post_attachment",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column(
|
||||
"post_id", sa.Integer(),
|
||||
sa.ForeignKey("post.id", ondelete="SET NULL"), nullable=True,
|
||||
),
|
||||
sa.Column(
|
||||
"artist_id", sa.Integer(),
|
||||
sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True,
|
||||
),
|
||||
sa.Column("sha256", sa.String(64), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("original_filename", sa.Text(), nullable=False),
|
||||
sa.Column("ext", sa.String(32), nullable=False),
|
||||
sa.Column("mime", sa.String(128), nullable=True),
|
||||
sa.Column("size_bytes", sa.BigInteger(), nullable=False),
|
||||
sa.Column(
|
||||
"captured_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.func.now(), nullable=False,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_post_attachment_sha256", "post_attachment", ["sha256"],
|
||||
unique=True,
|
||||
)
|
||||
op.create_index(
|
||||
"ix_post_attachment_post_id", "post_attachment", ["post_id"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_post_attachment_artist_id", "post_attachment", ["artist_id"],
|
||||
)
|
||||
op.add_column(
|
||||
"import_batch",
|
||||
sa.Column(
|
||||
"attachments", sa.Integer(), nullable=False,
|
||||
server_default="0",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_batch", "attachments")
|
||||
op.drop_index("ix_post_attachment_artist_id", table_name="post_attachment")
|
||||
op.drop_index("ix_post_attachment_post_id", table_name="post_attachment")
|
||||
op.drop_index("ix_post_attachment_sha256", table_name="post_attachment")
|
||||
op.drop_table("post_attachment")
|
||||
@@ -0,0 +1,32 @@
|
||||
"""fc3a: unique(source.artist_id, source.platform, source.url)
|
||||
|
||||
Revision ID: 0010
|
||||
Revises: 0009
|
||||
Create Date: 2026-05-20
|
||||
|
||||
Enforces FC-3a's dedup invariant at the DB level. No backfill — no
|
||||
existing rows are expected to collide; if they do the migration will
|
||||
fail loudly (intended).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0010"
|
||||
down_revision: Union[str, None] = "0009"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_unique_constraint(
|
||||
"uq_source_artist_platform_url",
|
||||
"source",
|
||||
["artist_id", "platform", "url"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_constraint(
|
||||
"uq_source_artist_platform_url", "source", type_="unique"
|
||||
)
|
||||
@@ -0,0 +1,41 @@
|
||||
"""fc3b: rename credential.kind -> credential_type, drop status, add last_verified
|
||||
|
||||
Revision ID: 0011
|
||||
Revises: 0010
|
||||
Create Date: 2026-05-20
|
||||
|
||||
Aligns the credential table with the GallerySubscriber wire-field names
|
||||
so the existing browser extension can POST to FC unmodified. Greenfield —
|
||||
no rows exist in production yet, so no data preservation logic is
|
||||
needed; the rename uses ALTER COLUMN rather than copy-then-drop.
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0011"
|
||||
down_revision: Union[str, None] = "0010"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("credential", "kind", new_column_name="credential_type")
|
||||
op.drop_column("credential", "status")
|
||||
op.add_column(
|
||||
"credential",
|
||||
sa.Column("last_verified", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("credential", "last_verified")
|
||||
op.add_column(
|
||||
"credential",
|
||||
sa.Column(
|
||||
"status", sa.String(length=32), nullable=False,
|
||||
server_default="active",
|
||||
),
|
||||
)
|
||||
op.alter_column("credential", "credential_type", new_column_name="kind")
|
||||
@@ -0,0 +1,36 @@
|
||||
"""fc3b: app_setting key/value table
|
||||
|
||||
Revision ID: 0012
|
||||
Revises: 0011
|
||||
Create Date: 2026-05-20
|
||||
|
||||
A simple key/value table for small app settings that don't fit
|
||||
ImportSettings. Initially seeds only `extension_api_key` (done in
|
||||
create_app on first boot — not in the migration, to keep it
|
||||
deterministic and independent of randomness).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0012"
|
||||
down_revision: Union[str, None] = "0011"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"app_setting",
|
||||
sa.Column("key", sa.String(length=64), primary_key=True),
|
||||
sa.Column("value", sa.Text(), nullable=False),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("app_setting")
|
||||
@@ -0,0 +1,52 @@
|
||||
"""fc3c: download_event.metadata + import_settings downloader fields
|
||||
|
||||
Revision ID: 0013
|
||||
Revises: 0012
|
||||
Create Date: 2026-05-20
|
||||
|
||||
Additive only. download_event.metadata is the rich JSONB blob FC-3c
|
||||
populates per run (run_stats, stdout/stderr, quarantined paths, import
|
||||
summary). import_settings gains two operator-tunable downloader knobs:
|
||||
download_rate_limit_seconds (gallery-dl extractor.sleep) and
|
||||
download_validate_files (toggle the magic-byte validator).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0013"
|
||||
down_revision: Union[str, None] = "0012"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"download_event",
|
||||
sa.Column(
|
||||
"metadata", postgresql.JSONB,
|
||||
nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_rate_limit_seconds", sa.Float(),
|
||||
nullable=False, server_default="3.0",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_validate_files", sa.Boolean(),
|
||||
nullable=False, server_default=sa.true(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "download_validate_files")
|
||||
op.drop_column("import_settings", "download_rate_limit_seconds")
|
||||
op.drop_column("download_event", "metadata")
|
||||
@@ -0,0 +1,58 @@
|
||||
"""fc3d: scheduling + source health columns
|
||||
|
||||
Revision ID: 0014
|
||||
Revises: 0013
|
||||
Create Date: 2026-05-21
|
||||
|
||||
Additive only. source.consecutive_failures (default 0, DownloadService
|
||||
finalize hook owns the writes). import_settings gains the three
|
||||
scheduling knobs (global default interval, event retention, failure
|
||||
warning threshold).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0014"
|
||||
down_revision: Union[str, None] = "0013"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"source",
|
||||
sa.Column(
|
||||
"consecutive_failures", sa.Integer(),
|
||||
nullable=False, server_default="0",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_schedule_default_seconds", sa.Integer(),
|
||||
nullable=False, server_default="28800",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_event_retention_days", sa.Integer(),
|
||||
nullable=False, server_default="90",
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_failure_warning_threshold", sa.Integer(),
|
||||
nullable=False, server_default="5",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "download_failure_warning_threshold")
|
||||
op.drop_column("import_settings", "download_event_retention_days")
|
||||
op.drop_column("import_settings", "download_schedule_default_seconds")
|
||||
op.drop_column("source", "consecutive_failures")
|
||||
@@ -0,0 +1,51 @@
|
||||
"""fc5: migration_run table
|
||||
|
||||
Revision ID: 0015
|
||||
Revises: 0014
|
||||
Create Date: 2026-05-22
|
||||
|
||||
Additive only. New table tracks each invocation of the FC-5 migration
|
||||
tooling (backup, gs, ir, ml_queue, verify, rollback). kind/status are
|
||||
plain String(32) — values validated at the API layer per the spec, not
|
||||
a Postgres ENUM (so adding kinds later doesn't need a schema migration).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0015"
|
||||
down_revision: Union[str, None] = "0014"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"migration_run",
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("kind", sa.String(32), nullable=False, index=True),
|
||||
sa.Column("status", sa.String(32), nullable=False, index=True),
|
||||
sa.Column(
|
||||
"dry_run", sa.Boolean(), nullable=False, server_default=sa.false(),
|
||||
),
|
||||
sa.Column(
|
||||
"started_at", sa.DateTime(timezone=True),
|
||||
nullable=False, server_default=sa.func.now(),
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"counts", postgresql.JSONB,
|
||||
nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"metadata", postgresql.JSONB,
|
||||
nullable=False, server_default=sa.text("'{}'::jsonb"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("migration_run")
|
||||
@@ -1,836 +0,0 @@
|
||||
"""The whole schema, in one migration.
|
||||
|
||||
This replaces alembic revisions 0001..0089 — the entire build-out of the
|
||||
project, 89 files and ~6,000 lines that a new installation used to replay in
|
||||
order to arrive at a schema this file creates in one pass. Nothing about the
|
||||
resulting database changes; what goes away is the requirement that a stranger
|
||||
re-run our development history to get it.
|
||||
|
||||
## Why the revision id is 0089
|
||||
|
||||
`revision = "0089"` and `down_revision = None` are both deliberate, and the
|
||||
combination is the entire migration strategy for existing installations.
|
||||
|
||||
An already-deployed database has `alembic_version = '0089'`, because it ran the
|
||||
real 0089. This file claims that same id, so alembic reads the version table,
|
||||
sees head already reached, and does nothing at all. No stamp is needed — which
|
||||
matters because `alembic stamp` writes a version string without validating
|
||||
anything about the schema it is writing it against, and a stamp that is wrong
|
||||
is indistinguishable from one that is right until the next migration fails.
|
||||
|
||||
An empty database has no version row, so alembic runs this file and then
|
||||
records `0089`. Both paths converge on the same schema and the same version,
|
||||
and neither requires anyone to assert anything by hand.
|
||||
|
||||
The next migration written after this one is `0090`, exactly as it would have
|
||||
been. The numbering is continuous across the collapse on purpose.
|
||||
|
||||
## What was added to the generated output, and why
|
||||
|
||||
`alembic revision --autogenerate` produced almost all of this from the models,
|
||||
which is only true because #3275 first made the models actually describe the
|
||||
schema. Before that reconciliation the generator silently omitted eleven
|
||||
indexes and three uniqueness guarantees, and an earlier attempt at this squash
|
||||
had to be reverted for exactly that reason.
|
||||
|
||||
Four things still had to be added by hand, because they are not in the models:
|
||||
|
||||
1. **`CREATE EXTENSION vector`** (from 0001) and **`tsm_system_rows`** (0004).
|
||||
Extensions are database objects, not table metadata, so no model can carry
|
||||
them. `IF NOT EXISTS` because a re-run must not fail.
|
||||
|
||||
2. **Three seed inserts** — the two settings singletons (0002, 0003) and the
|
||||
three hygiene system tags (0075). Some migrations did not only build schema;
|
||||
they inserted rows the product needs in order to function, and nothing in
|
||||
the application ever creates them. Every consumer reads them with
|
||||
`scalar_one()`, which RAISES `NoResultFound` on an empty result rather than
|
||||
returning None, so their absence is a crash and not a degradation.
|
||||
|
||||
Distinguishing these from the other data statements in the chain is the
|
||||
whole trick, and the rule turns out to be mechanical:
|
||||
|
||||
* `INSERT ... VALUES (...)` with literal values is a SEED. It creates
|
||||
something the product ships. It must be carried.
|
||||
* `INSERT ... SELECT ... FROM <table>` is a BACKFILL. It derives rows
|
||||
from rows that already exist, so on an empty database it inserts
|
||||
nothing and carrying it would be pointless. 0034 (artist_visit), 0040
|
||||
and 0047 (series_chapter) are all of this shape and are correctly
|
||||
absent here.
|
||||
|
||||
This category is invisible to every automated check this project has:
|
||||
`baseline.yml` compares SCHEMA, and a baseline missing all three seeds still
|
||||
produces a byte-identical schema and a perfectly green diff. What caught the
|
||||
system tags was the integration suite — 36 tests failing on
|
||||
`NoResultFound` — after a first version of this file shipped with only the
|
||||
two settings rows. A first-run check against the real application is the
|
||||
only thing that finds this class of defect.
|
||||
|
||||
3. **The `pgvector` import.** Autogenerate emits qualified
|
||||
`pgvector.sqlalchemy.vector.VECTOR(...)` references without importing the
|
||||
package, so the file it writes cannot execute — `NameError: name 'pgvector'
|
||||
is not defined`, observed on run 4988.
|
||||
|
||||
The other data statements in the old chain were deliberately NOT carried over.
|
||||
0023's `DELETE FROM tag WHERE kind IN (...)`, and 0047's `series_page` /
|
||||
`series_chapter` deletes, are historical cleanups that operate on rows an empty
|
||||
database does not have.
|
||||
|
||||
## Downgrade
|
||||
|
||||
There is none. A baseline's downgrade would be "drop the entire schema", which
|
||||
is not a migration but a data-loss event wearing one as a disguise. Restore
|
||||
from a backup instead — that is what backup_run exists for.
|
||||
|
||||
Revision ID: 0089
|
||||
Revises:
|
||||
Create Date: 2026-09-01
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
import pgvector.sqlalchemy.vector
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0089"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Extensions first: image_record.siglip_embedding is a vector column and
|
||||
# cannot be created before the type exists. From 0001 and 0004.
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows")
|
||||
|
||||
op.create_table('app_setting',
|
||||
sa.Column('key', sa.String(length=64), nullable=False),
|
||||
sa.Column('value', sa.Text(), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('key', name=op.f('pk_app_setting'))
|
||||
)
|
||||
op.create_table('artist',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('slug', sa.String(length=255), nullable=False),
|
||||
sa.Column('notes', sa.Text(), nullable=True),
|
||||
sa.Column('is_subscription', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('auto_check', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('check_interval_seconds', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_artist')),
|
||||
sa.UniqueConstraint('slug', name=op.f('uq_artist_slug'))
|
||||
)
|
||||
op.create_table('backup_run',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('kind', sa.String(length=16), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||
sa.Column('tag', sa.String(length=64), nullable=True),
|
||||
sa.Column('triggered_by', sa.String(length=32), nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('sql_path', sa.Text(), nullable=True),
|
||||
sa.Column('tar_path', sa.Text(), nullable=True),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=True),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.Column('manifest', sa.JSON(), server_default='{}', nullable=False),
|
||||
sa.Column('restored_from_id', sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['restored_from_id'], ['backup_run.id'], name=op.f('fk_backup_run_restored_from_id_backup_run'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_backup_run'))
|
||||
)
|
||||
op.create_index(op.f('ix_backup_run_finished_at'), 'backup_run', ['finished_at'], unique=False)
|
||||
op.create_index('ix_backup_run_kind_started', 'backup_run', ['kind', sa.literal_column('started_at DESC')], unique=False)
|
||||
op.create_index(op.f('ix_backup_run_restored_from_id'), 'backup_run', ['restored_from_id'], unique=False)
|
||||
op.create_index(op.f('ix_backup_run_started_at'), 'backup_run', ['started_at'], unique=False)
|
||||
op.create_index('ix_backup_run_status_finished', 'backup_run', ['status', sa.literal_column('finished_at DESC')], unique=False)
|
||||
op.create_index(op.f('ix_backup_run_tag'), 'backup_run', ['tag'], unique=False)
|
||||
op.create_index('ix_backup_run_tag_partial', 'backup_run', ['tag'], unique=False, postgresql_where=sa.text('tag IS NOT NULL'))
|
||||
op.create_table('credential',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('platform', sa.String(length=64), nullable=False),
|
||||
sa.Column('credential_type', sa.String(length=32), nullable=False),
|
||||
sa.Column('encrypted_blob', sa.LargeBinary(), nullable=False),
|
||||
sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_verified', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_credential')),
|
||||
sa.UniqueConstraint('platform', name=op.f('uq_credential_platform'))
|
||||
)
|
||||
op.create_table('head_auto_apply_run',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('dry_run', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('n_applied', sa.Integer(), nullable=True),
|
||||
sa.Column('report', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_head_auto_apply_run'))
|
||||
)
|
||||
op.create_index(op.f('ix_head_auto_apply_run_status'), 'head_auto_apply_run', ['status'], unique=False)
|
||||
op.create_table('head_training_run',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('n_trained', sa.Integer(), nullable=True),
|
||||
sa.Column('n_skipped', sa.Integer(), nullable=True),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_head_training_run'))
|
||||
)
|
||||
op.create_index(op.f('ix_head_training_run_status'), 'head_training_run', ['status'], unique=False)
|
||||
op.create_table('import_batch',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('triggered_by', sa.String(length=32), nullable=False),
|
||||
sa.Column('source_path', sa.Text(), nullable=False),
|
||||
sa.Column('scan_mode', sa.String(length=16), nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('total_files', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('imported', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('skipped', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('failed', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('attachments', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('refreshed', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_import_batch'))
|
||||
)
|
||||
op.create_index(op.f('ix_import_batch_status'), 'import_batch', ['status'], unique=False)
|
||||
op.create_table('import_settings',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('import_scan_path', sa.Text(), server_default='/import', nullable=False),
|
||||
sa.Column('min_width', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('min_height', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('skip_transparent', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('transparency_threshold', sa.Float(), server_default='0.9', nullable=False),
|
||||
sa.Column('skip_single_color', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('single_color_threshold', sa.Float(), server_default='0.95', nullable=False),
|
||||
sa.Column('single_color_tolerance', sa.Integer(), server_default='30', nullable=False),
|
||||
sa.Column('phash_threshold', sa.Integer(), server_default='10', nullable=False),
|
||||
sa.Column('download_rate_limit_seconds', sa.Float(), server_default='3', nullable=False),
|
||||
sa.Column('download_validate_files', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('download_schedule_default_seconds', sa.Integer(), server_default='28800', nullable=False),
|
||||
sa.Column('download_event_retention_days', sa.Integer(), server_default='90', nullable=False),
|
||||
sa.Column('download_failure_warning_threshold', sa.Integer(), server_default='5', nullable=False),
|
||||
sa.Column('backup_db_nightly_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('backup_db_nightly_hour_utc', sa.Integer(), server_default='3', nullable=False),
|
||||
sa.Column('backup_db_keep_last_n', sa.Integer(), server_default='14', nullable=False),
|
||||
sa.Column('backup_images_keep_last_n', sa.Integer(), server_default='3', nullable=False),
|
||||
sa.Column('series_suggest_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('series_suggest_threshold', sa.Float(), server_default='0.5', nullable=False),
|
||||
sa.Column('extdl_mega_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('extdl_gdrive_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('extdl_mediafire_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('extdl_dropbox_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('extdl_pixeldrain_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('translation_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('interpreter_base_url', sa.Text(), server_default='', nullable=False),
|
||||
sa.Column('translation_target_lang', sa.Text(), server_default='en', nullable=False),
|
||||
sa.Column('translation_min_confidence', sa.Float(), server_default=sa.text('0.9'), nullable=False),
|
||||
sa.Column('wip_title_tagging_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('wip_soft_title_tagging_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.CheckConstraint('id = 1', name=op.f('ck_import_settings_singleton')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_import_settings'))
|
||||
)
|
||||
op.create_table('library_audit_run',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('rule', sa.String(length=32), nullable=False),
|
||||
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('scanned_count', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('matched_count', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('matched_ids', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'[]'::jsonb"), nullable=False),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.Column('resume_after_id', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_library_audit_run'))
|
||||
)
|
||||
op.create_index(op.f('ix_library_audit_run_rule'), 'library_audit_run', ['rule'], unique=False)
|
||||
op.create_index(op.f('ix_library_audit_run_status'), 'library_audit_run', ['status'], unique=False)
|
||||
op.create_table('ml_settings',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('cpu_embed_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('video_frame_interval_seconds', sa.Float(), server_default='4', nullable=False),
|
||||
sa.Column('video_max_frames', sa.Integer(), server_default='64', nullable=False),
|
||||
sa.Column('head_min_positives', sa.Integer(), server_default='8', nullable=False),
|
||||
sa.Column('head_auto_apply_precision', sa.Float(), server_default='0.97', nullable=False),
|
||||
sa.Column('head_auto_apply_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('head_auto_apply_min_positives', sa.Integer(), server_default='30', nullable=False),
|
||||
sa.Column('ccip_match_threshold', sa.Float(), server_default='0.85', nullable=False),
|
||||
sa.Column('ccip_auto_apply_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('ccip_auto_apply_threshold', sa.Float(), server_default='0.92', nullable=False),
|
||||
sa.Column('presentation_auto_apply_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('presentation_auto_apply_threshold', sa.Float(), server_default=sa.text('0.90'), nullable=False),
|
||||
sa.Column('presentation_conflict_threshold', sa.Float(), server_default=sa.text('0.50'), nullable=False),
|
||||
sa.Column('process_auto_apply_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('process_auto_apply_threshold', sa.Float(), server_default='0.90', nullable=False),
|
||||
sa.Column('process_conflict_threshold', sa.Float(), server_default='0.50', nullable=False),
|
||||
sa.Column('embedder_model_version', sa.String(length=128), server_default='siglip2-so400m-patch16-512', nullable=False),
|
||||
sa.Column('embedder_model_name', sa.String(length=128), server_default='google/siglip2-so400m-patch16-512', nullable=False),
|
||||
sa.Column('detector_person_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('detector_person_weights', sa.String(length=512), server_default='yolo11n.pt', nullable=False),
|
||||
sa.Column('detector_person_conf', sa.Float(), server_default=sa.text('0.35'), nullable=False),
|
||||
sa.Column('detector_anatomy_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('detector_anatomy_weights', sa.String(length=512), server_default='https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt', nullable=False),
|
||||
sa.Column('detector_anatomy_conf', sa.Float(), server_default=sa.text('0.30'), nullable=False),
|
||||
sa.Column('detector_panel_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('detector_panel_weights', sa.String(length=512), server_default='mosesb/best-comic-panel-detection::best.pt', nullable=False),
|
||||
sa.Column('detector_panel_conf', sa.Float(), server_default=sa.text('0.30'), nullable=False),
|
||||
sa.Column('detector_max_figures', sa.Integer(), server_default='8', nullable=False),
|
||||
sa.Column('detector_max_components', sa.Integer(), server_default='8', nullable=False),
|
||||
sa.Column('detector_max_panels', sa.Integer(), server_default='8', nullable=False),
|
||||
sa.Column('detector_max_regions', sa.Integer(), server_default='128', nullable=False),
|
||||
sa.Column('detector_dedupe_iou', sa.Float(), server_default=sa.text('0.85'), nullable=False),
|
||||
sa.Column('ccip_ref_signature', sa.String(length=128), nullable=True),
|
||||
sa.Column('ccip_prototype_cap', sa.Integer(), server_default='64', nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.CheckConstraint('id = 1', name=op.f('ck_ml_settings_singleton')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_ml_settings'))
|
||||
)
|
||||
op.create_table('tag',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('kind', sa.Enum('artist', 'character', 'fandom', 'general', 'series', 'archive', 'post', name='tag_kind'), server_default='general', nullable=False),
|
||||
sa.Column('fandom_id', sa.Integer(), nullable=True),
|
||||
sa.Column('is_system', sa.Boolean(), server_default=sa.text('false'), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.CheckConstraint("(fandom_id IS NULL) OR (kind = 'character')", name=op.f('ck_tag_fandom_requires_character')),
|
||||
sa.ForeignKeyConstraint(['fandom_id'], ['tag.id'], name=op.f('fk_tag_fandom_id_tag'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
|
||||
)
|
||||
op.create_index(op.f('ix_tag_fandom_id'), 'tag', ['fandom_id'], unique=False)
|
||||
op.create_index('uq_tag_name_kind_fandom', 'tag', ['name', 'kind', sa.literal_column('COALESCE(fandom_id, 0)')], unique=True)
|
||||
op.create_table('task_run',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('celery_task_id', sa.String(length=64), nullable=False),
|
||||
sa.Column('queue', sa.String(length=32), nullable=False),
|
||||
sa.Column('task_name', sa.String(length=128), nullable=False),
|
||||
sa.Column('target_id', sa.Integer(), nullable=True),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('duration_ms', sa.Integer(), nullable=True),
|
||||
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||
sa.Column('error_type', sa.String(length=128), nullable=True),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
sa.Column('retry_count', sa.Integer(), nullable=True),
|
||||
sa.Column('worker_hostname', sa.String(length=128), nullable=True),
|
||||
sa.Column('args_summary', sa.String(length=255), nullable=True),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_task_run'))
|
||||
)
|
||||
op.create_index(op.f('ix_task_run_celery_task_id'), 'task_run', ['celery_task_id'], unique=False)
|
||||
op.create_index(op.f('ix_task_run_finished_at'), 'task_run', ['finished_at'], unique=False)
|
||||
op.create_index('ix_task_run_name_started', 'task_run', ['task_name', sa.literal_column('started_at DESC')], unique=False)
|
||||
op.create_index('ix_task_run_queue_started', 'task_run', ['queue', sa.literal_column('started_at DESC')], unique=False)
|
||||
op.create_index(op.f('ix_task_run_started_at'), 'task_run', ['started_at'], unique=False)
|
||||
op.create_index('ix_task_run_status_started', 'task_run', ['status', sa.literal_column('started_at DESC')], unique=False)
|
||||
op.create_table('artist_visit',
|
||||
sa.Column('artist_id', sa.Integer(), nullable=False),
|
||||
sa.Column('last_viewed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_artist_visit_artist_id_artist'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('artist_id', name=op.f('pk_artist_visit'))
|
||||
)
|
||||
op.create_table('ccip_prototype_state',
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('fingerprint', sa.String(length=64), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_ccip_prototype_state_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_ccip_prototype_state'))
|
||||
)
|
||||
op.create_table('head_metric',
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('n_misfires', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('n_underfires', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metric_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_head_metric'))
|
||||
)
|
||||
op.create_table('head_metrics_snapshot',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=True),
|
||||
sa.Column('name', sa.String(length=255), nullable=False),
|
||||
sa.Column('snapshot_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('n_auto_applied', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('n_misfires', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('n_underfires', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('ap', sa.Float(), nullable=True),
|
||||
sa.Column('precision_cv', sa.Float(), nullable=True),
|
||||
sa.Column('recall', sa.Float(), nullable=True),
|
||||
sa.Column('n_pos', sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metrics_snapshot_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_head_metrics_snapshot'))
|
||||
)
|
||||
op.create_index(op.f('ix_head_metrics_snapshot_snapshot_at'), 'head_metrics_snapshot', ['snapshot_at'], unique=False)
|
||||
op.create_index(op.f('ix_head_metrics_snapshot_tag_id'), 'head_metrics_snapshot', ['tag_id'], unique=False)
|
||||
op.create_table('source',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('artist_id', sa.Integer(), nullable=False),
|
||||
sa.Column('platform', sa.String(length=64), nullable=False),
|
||||
sa.Column('url', sa.Text(), nullable=False),
|
||||
sa.Column('enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||
sa.Column('config_overrides', sa.JSON(), nullable=True),
|
||||
sa.Column('last_checked_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('error_type', sa.String(length=32), nullable=True),
|
||||
sa.Column('check_interval_override', sa.Integer(), nullable=True),
|
||||
sa.Column('consecutive_failures', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('backfill_runs_remaining', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_source_artist_id_artist'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_source')),
|
||||
sa.UniqueConstraint('artist_id', 'platform', 'url', name='uq_source_artist_platform_url')
|
||||
)
|
||||
op.create_index(op.f('ix_source_artist_id'), 'source', ['artist_id'], unique=False)
|
||||
op.create_index(op.f('ix_source_error_type'), 'source', ['error_type'], unique=False)
|
||||
op.create_table('tag_alias',
|
||||
sa.Column('alias_string', sa.String(length=255), nullable=False),
|
||||
sa.Column('alias_category', sa.String(length=32), nullable=False),
|
||||
sa.Column('canonical_tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['canonical_tag_id'], ['tag.id'], name=op.f('fk_tag_alias_canonical_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('alias_string', 'alias_category', name=op.f('pk_tag_alias'))
|
||||
)
|
||||
op.create_index('ix_tag_alias_canonical', 'tag_alias', ['canonical_tag_id'], unique=False)
|
||||
op.create_table('tag_head',
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('embedding_version', sa.String(length=128), nullable=False),
|
||||
sa.Column('weights', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=False),
|
||||
sa.Column('bias', sa.Float(), nullable=False),
|
||||
sa.Column('suggest_threshold', sa.Float(), nullable=False),
|
||||
sa.Column('auto_apply_threshold', sa.Float(), nullable=True),
|
||||
sa.Column('n_pos', sa.Integer(), nullable=False),
|
||||
sa.Column('n_neg', sa.Integer(), nullable=False),
|
||||
sa.Column('ap', sa.Float(), nullable=False),
|
||||
sa.Column('precision_cv', sa.Float(), nullable=False),
|
||||
sa.Column('recall', sa.Float(), nullable=False),
|
||||
sa.Column('trained_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('train_fingerprint', sa.String(length=128), nullable=True),
|
||||
sa.Column('metrics', postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_head_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_tag_head'))
|
||||
)
|
||||
op.create_table('patreon_failed_media',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||
sa.Column('attempts', sa.Integer(), server_default='1', nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_patreon_failed_media_source_id_source'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_patreon_failed_media')),
|
||||
sa.UniqueConstraint('source_id', 'filehash', name='uq_patreon_failed_media_source_id')
|
||||
)
|
||||
op.create_index(op.f('ix_patreon_failed_media_source_id'), 'patreon_failed_media', ['source_id'], unique=False)
|
||||
op.create_table('patreon_seen_media',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||
sa.Column('post_id', sa.String(length=64), nullable=True),
|
||||
sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_patreon_seen_media_source_id_source'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_patreon_seen_media')),
|
||||
sa.UniqueConstraint('source_id', 'filehash', name='uq_patreon_seen_media_source_id')
|
||||
)
|
||||
op.create_index(op.f('ix_patreon_seen_media_source_id'), 'patreon_seen_media', ['source_id'], unique=False)
|
||||
op.create_table('pixiv_failed_media',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||
sa.Column('attempts', sa.Integer(), server_default='1', nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_pixiv_failed_media_source_id_source'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_pixiv_failed_media')),
|
||||
sa.UniqueConstraint('source_id', 'filehash', name='uq_pixiv_failed_media_source_id')
|
||||
)
|
||||
op.create_index(op.f('ix_pixiv_failed_media_source_id'), 'pixiv_failed_media', ['source_id'], unique=False)
|
||||
op.create_table('pixiv_seen_media',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||
sa.Column('post_id', sa.String(length=64), nullable=True),
|
||||
sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_pixiv_seen_media_source_id_source'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_pixiv_seen_media')),
|
||||
sa.UniqueConstraint('source_id', 'filehash', name='uq_pixiv_seen_media_source_id')
|
||||
)
|
||||
op.create_index(op.f('ix_pixiv_seen_media_source_id'), 'pixiv_seen_media', ['source_id'], unique=False)
|
||||
op.create_table('post',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=True),
|
||||
sa.Column('artist_id', sa.Integer(), nullable=False),
|
||||
sa.Column('external_post_id', sa.String(length=128), nullable=False),
|
||||
sa.Column('post_url', sa.Text(), nullable=True),
|
||||
sa.Column('post_title', sa.Text(), nullable=True),
|
||||
sa.Column('post_date', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('raw_metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('description', sa.Text(), nullable=True),
|
||||
sa.Column('attachment_count', sa.Integer(), nullable=True),
|
||||
sa.Column('post_title_translated', sa.Text(), nullable=True),
|
||||
sa.Column('description_translated', sa.Text(), nullable=True),
|
||||
sa.Column('translated_source_lang', sa.String(length=8), nullable=True),
|
||||
sa.Column('translation_engine_version', sa.String(length=128), nullable=True),
|
||||
sa.Column('translated_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('translation_override', sa.String(length=16), server_default='auto', nullable=False),
|
||||
sa.Column('downloaded_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.CheckConstraint("translation_override IN ('auto', 'force', 'original')", name=op.f('ck_post_translation_override')),
|
||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_artist_id_artist'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_post_source_id_source'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_post')),
|
||||
sa.UniqueConstraint('source_id', 'external_post_id', name='uq_post_source_external_id')
|
||||
)
|
||||
op.create_index(op.f('ix_post_artist_id'), 'post', ['artist_id'], unique=False)
|
||||
op.create_index(op.f('ix_post_source_id'), 'post', ['source_id'], unique=False)
|
||||
op.create_index('uq_post_artist_external_id_null_source', 'post', ['artist_id', 'external_post_id'], unique=True, postgresql_where=sa.text('source_id IS NULL'))
|
||||
op.create_table('subscribestar_failed_media',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||
sa.Column('attempts', sa.Integer(), server_default='1', nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_subscribestar_failed_media_source_id_source'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_subscribestar_failed_media')),
|
||||
sa.UniqueConstraint('source_id', 'filehash', name='uq_subscribestar_failed_media_source_id')
|
||||
)
|
||||
op.create_index(op.f('ix_subscribestar_failed_media_source_id'), 'subscribestar_failed_media', ['source_id'], unique=False)
|
||||
op.create_table('subscribestar_seen_media',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||
sa.Column('post_id', sa.String(length=64), nullable=True),
|
||||
sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_subscribestar_seen_media_source_id_source'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_subscribestar_seen_media')),
|
||||
sa.UniqueConstraint('source_id', 'filehash', name='uq_subscribestar_seen_media_source_id')
|
||||
)
|
||||
op.create_index(op.f('ix_subscribestar_seen_media_source_id'), 'subscribestar_seen_media', ['source_id'], unique=False)
|
||||
op.create_table('download_event',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||
sa.Column('post_id', sa.Integer(), nullable=True),
|
||||
sa.Column('status', sa.String(length=32), nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('bytes_downloaded', sa.BigInteger(), server_default='0', nullable=False),
|
||||
sa.Column('files_count', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
|
||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_download_event_post_id_post'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_download_event_source_id_source'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_download_event'))
|
||||
)
|
||||
op.create_index(op.f('ix_download_event_post_id'), 'download_event', ['post_id'], unique=False)
|
||||
op.create_index(op.f('ix_download_event_source_id'), 'download_event', ['source_id'], unique=False)
|
||||
op.create_table('image_record',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('path', sa.Text(), nullable=False),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('phash', sa.String(length=32), nullable=True),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||
sa.Column('mime', sa.String(length=64), nullable=False),
|
||||
sa.Column('width', sa.Integer(), nullable=True),
|
||||
sa.Column('height', sa.Integer(), nullable=True),
|
||||
sa.Column('duration_seconds', sa.Float(), nullable=True),
|
||||
sa.Column('integrity_status', sa.String(length=24), server_default='unknown', nullable=False),
|
||||
sa.Column('thumbnail_path', sa.Text(), nullable=True),
|
||||
sa.Column('source_url', sa.Text(), nullable=True),
|
||||
sa.Column('source_filehash', sa.String(length=32), nullable=True),
|
||||
sa.Column('origin', sa.Enum('downloaded', 'imported_filesystem', 'uploaded', name='origin_enum'), nullable=False),
|
||||
sa.Column('primary_post_id', sa.Integer(), nullable=True),
|
||||
sa.Column('artist_id', sa.Integer(), nullable=True),
|
||||
sa.Column('siglip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=True),
|
||||
sa.Column('siglip_model_version', sa.String(length=128), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('effective_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('earliest_post_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name='fk_image_record_artist_id', ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['primary_post_id'], ['post.id'], name=op.f('fk_image_record_primary_post_id_post'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_image_record')),
|
||||
sa.UniqueConstraint('path', name=op.f('uq_image_record_path')),
|
||||
sa.UniqueConstraint('sha256', name='uq_image_record_sha256')
|
||||
)
|
||||
op.create_index(op.f('ix_image_record_artist_id'), 'image_record', ['artist_id'], unique=False)
|
||||
op.create_index('ix_image_record_earliest_post_date', 'image_record', [sa.literal_column('earliest_post_date DESC'), sa.literal_column('id DESC')], unique=False)
|
||||
op.create_index('ix_image_record_effective_date', 'image_record', [sa.literal_column('effective_date DESC'), sa.literal_column('id DESC')], unique=False)
|
||||
op.create_index(op.f('ix_image_record_integrity_status'), 'image_record', ['integrity_status'], unique=False)
|
||||
op.create_index(op.f('ix_image_record_phash'), 'image_record', ['phash'], unique=False)
|
||||
op.create_index(op.f('ix_image_record_primary_post_id'), 'image_record', ['primary_post_id'], unique=False)
|
||||
op.create_index('ix_image_record_siglip_hnsw', 'image_record', ['siglip_embedding'], unique=False, postgresql_using='hnsw', postgresql_ops={'siglip_embedding': 'vector_cosine_ops'})
|
||||
op.create_index(op.f('ix_image_record_source_filehash'), 'image_record', ['source_filehash'], unique=False)
|
||||
op.create_table('post_attachment',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('post_id', sa.Integer(), nullable=True),
|
||||
sa.Column('artist_id', sa.Integer(), nullable=True),
|
||||
sa.Column('sha256', sa.String(length=64), nullable=False),
|
||||
sa.Column('path', sa.Text(), nullable=False),
|
||||
sa.Column('original_filename', sa.Text(), nullable=False),
|
||||
sa.Column('ext', sa.String(length=32), nullable=False),
|
||||
sa.Column('mime', sa.String(length=128), nullable=True),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=False),
|
||||
sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_attachment_artist_id_artist'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_post_attachment_post_id_post'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_post_attachment'))
|
||||
)
|
||||
op.create_index(op.f('ix_post_attachment_artist_id'), 'post_attachment', ['artist_id'], unique=False)
|
||||
op.create_index(op.f('ix_post_attachment_post_id'), 'post_attachment', ['post_id'], unique=False)
|
||||
op.create_index(op.f('ix_post_attachment_sha256'), 'post_attachment', ['sha256'], unique=False)
|
||||
op.create_index('uq_post_attachment_null_post_sha', 'post_attachment', ['sha256'], unique=True, postgresql_where=sa.text('post_id IS NULL'))
|
||||
op.create_index('uq_post_attachment_post_sha', 'post_attachment', ['post_id', 'sha256'], unique=True, postgresql_where=sa.text('post_id IS NOT NULL'))
|
||||
op.create_table('series_suggestion',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('post_id', sa.Integer(), nullable=False),
|
||||
sa.Column('series_tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('score', sa.Float(), nullable=False),
|
||||
sa.Column('signals', sa.JSON(), nullable=True),
|
||||
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_series_suggestion_post_id_post'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_suggestion_series_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_series_suggestion')),
|
||||
sa.UniqueConstraint('post_id', 'series_tag_id', name='uq_series_suggestion_post_series')
|
||||
)
|
||||
op.create_index(op.f('ix_series_suggestion_post_id'), 'series_suggestion', ['post_id'], unique=False)
|
||||
op.create_index(op.f('ix_series_suggestion_series_tag_id'), 'series_suggestion', ['series_tag_id'], unique=False)
|
||||
op.create_index(op.f('ix_series_suggestion_status'), 'series_suggestion', ['status'], unique=False)
|
||||
op.create_table('external_link',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('post_id', sa.Integer(), nullable=False),
|
||||
sa.Column('artist_id', sa.Integer(), nullable=True),
|
||||
sa.Column('host', sa.String(length=16), nullable=False),
|
||||
sa.Column('url', sa.Text(), nullable=False),
|
||||
sa.Column('label', sa.Text(), nullable=True),
|
||||
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||
sa.Column('attempts', sa.Integer(), server_default=sa.text('0'), nullable=False),
|
||||
sa.Column('last_error', sa.Text(), nullable=True),
|
||||
sa.Column('attachment_id', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('duration_seconds', sa.Float(), nullable=True),
|
||||
sa.CheckConstraint("host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')", name=op.f('ck_external_link_host')),
|
||||
sa.CheckConstraint("status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')", name=op.f('ck_external_link_status')),
|
||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_external_link_artist_id_artist'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['attachment_id'], ['post_attachment.id'], name=op.f('fk_external_link_attachment_id_post_attachment'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_external_link_post_id_post'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_external_link'))
|
||||
)
|
||||
op.create_index(op.f('ix_external_link_artist_id'), 'external_link', ['artist_id'], unique=False)
|
||||
op.create_index('ix_external_link_attachment_id', 'external_link', ['attachment_id'], unique=False)
|
||||
op.create_index('ix_external_link_status', 'external_link', ['status'], unique=False)
|
||||
op.create_index('uq_external_link_post_url', 'external_link', ['post_id', 'url'], unique=True)
|
||||
op.create_table('gpu_job',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||
sa.Column('task', sa.String(length=32), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||
sa.Column('lease_token', sa.String(length=64), nullable=True),
|
||||
sa.Column('leased_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('lease_expires_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('attempts', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.Column('triage_status', sa.String(length=16), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_gpu_job_image_record_id_image_record'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_gpu_job'))
|
||||
)
|
||||
op.create_index(op.f('ix_gpu_job_image_record_id'), 'gpu_job', ['image_record_id'], unique=False)
|
||||
op.create_index('ix_gpu_job_leased_expires', 'gpu_job', ['lease_expires_at'], unique=False, postgresql_where=sa.text("status = 'leased'"))
|
||||
op.create_index('ix_gpu_job_pending', 'gpu_job', ['id'], unique=False, postgresql_where=sa.text("status = 'pending'"))
|
||||
op.create_index(op.f('ix_gpu_job_status'), 'gpu_job', ['status'], unique=False)
|
||||
op.create_table('image_provenance',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||
sa.Column('post_id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_id', sa.Integer(), nullable=True),
|
||||
sa.Column('from_attachment_id', sa.Integer(), nullable=True),
|
||||
sa.Column('captured_metadata', sa.JSON(), nullable=True),
|
||||
sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['from_attachment_id'], ['post_attachment.id'], name='fk_image_provenance_from_attachment', ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_provenance_image_record_id_image_record'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_image_provenance_post_id_post'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_image_provenance_source_id_source'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_image_provenance')),
|
||||
sa.UniqueConstraint('image_record_id', 'post_id', name='uq_image_provenance_image_post')
|
||||
)
|
||||
op.create_index(op.f('ix_image_provenance_from_attachment_id'), 'image_provenance', ['from_attachment_id'], unique=False)
|
||||
op.create_index(op.f('ix_image_provenance_image_record_id'), 'image_provenance', ['image_record_id'], unique=False)
|
||||
op.create_index(op.f('ix_image_provenance_post_id'), 'image_provenance', ['post_id'], unique=False)
|
||||
op.create_index(op.f('ix_image_provenance_source_id'), 'image_provenance', ['source_id'], unique=False)
|
||||
op.create_table('image_region',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||
sa.Column('kind', sa.String(length=16), nullable=False),
|
||||
sa.Column('frame_time', sa.Float(), nullable=True),
|
||||
sa.Column('rx', sa.Float(), nullable=False),
|
||||
sa.Column('ry', sa.Float(), nullable=False),
|
||||
sa.Column('rw', sa.Float(), nullable=False),
|
||||
sa.Column('rh', sa.Float(), nullable=False),
|
||||
sa.Column('score', sa.Float(), nullable=True),
|
||||
sa.Column('detector_version', sa.String(length=64), nullable=True),
|
||||
sa.Column('crop_version', sa.String(length=64), nullable=True),
|
||||
sa.Column('embedding_version', sa.String(length=128), nullable=True),
|
||||
sa.Column('ccip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=768), nullable=True),
|
||||
sa.Column('siglip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_region_image_record_id_image_record'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_image_region'))
|
||||
)
|
||||
op.create_index(op.f('ix_image_region_image_record_id'), 'image_region', ['image_record_id'], unique=False)
|
||||
op.create_table('image_tag',
|
||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('source', sa.String(length=32), server_default='manual', nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_tag_image_record_id_image_record'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_image_tag_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_image_tag'))
|
||||
)
|
||||
op.create_index('ix_image_tag_tag_id', 'image_tag', ['tag_id'], unique=False)
|
||||
op.create_table('import_task',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('batch_id', sa.Integer(), nullable=False),
|
||||
sa.Column('source_path', sa.Text(), nullable=False),
|
||||
sa.Column('task_type', sa.String(length=16), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||
sa.Column('recovery_count', sa.Integer(), server_default='0', nullable=False),
|
||||
sa.Column('refetched', sa.Boolean(), server_default='false', nullable=False),
|
||||
sa.Column('result_image_id', sa.Integer(), nullable=True),
|
||||
sa.Column('error', sa.Text(), nullable=True),
|
||||
sa.Column('size_bytes', sa.BigInteger(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.ForeignKeyConstraint(['batch_id'], ['import_batch.id'], name=op.f('fk_import_task_batch_id_import_batch'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['result_image_id'], ['image_record.id'], name=op.f('fk_import_task_result_image_id_image_record'), ondelete='SET NULL'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_import_task'))
|
||||
)
|
||||
op.create_index(op.f('ix_import_task_batch_id'), 'import_task', ['batch_id'], unique=False)
|
||||
op.create_index('ix_import_task_created_at_desc', 'import_task', [sa.literal_column('created_at DESC')], unique=False)
|
||||
op.create_index('ix_import_task_result_image_id', 'import_task', ['result_image_id'], unique=False)
|
||||
op.create_index(op.f('ix_import_task_status'), 'import_task', ['status'], unique=False)
|
||||
op.create_table('presentation_review',
|
||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('conflict_tag_id', sa.Integer(), nullable=True),
|
||||
sa.Column('conflict_score', sa.Float(), nullable=False),
|
||||
sa.Column('mode', sa.String(length=16), server_default='chrome', nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True),
|
||||
sa.ForeignKeyConstraint(['conflict_tag_id'], ['tag.id'], name=op.f('fk_presentation_review_conflict_tag_id_tag'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_presentation_review_image_record_id_image_record'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_presentation_review_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_presentation_review'))
|
||||
)
|
||||
op.create_index('ix_presentation_review_conflict_tag_id', 'presentation_review', ['conflict_tag_id'], unique=False)
|
||||
op.create_index('ix_presentation_review_resolved_at', 'presentation_review', ['resolved_at'], unique=False)
|
||||
op.create_index('ix_presentation_review_tag_id', 'presentation_review', ['tag_id'], unique=False)
|
||||
op.create_table('series_page',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('series_tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('image_id', sa.Integer(), nullable=False),
|
||||
sa.Column('status', sa.String(length=16), server_default='placed', nullable=False),
|
||||
sa.Column('page_number', sa.Integer(), nullable=True),
|
||||
sa.Column('stated_page', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], name=op.f('fk_series_page_image_id_image_record'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_page_series_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_series_page')),
|
||||
sa.UniqueConstraint('image_id', name='uq_series_page_image')
|
||||
)
|
||||
op.create_index(op.f('ix_series_page_series_tag_id'), 'series_page', ['series_tag_id'], unique=False)
|
||||
op.create_table('tag_positive_confirmation',
|
||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('confirmed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_tag_positive_confirmation_image_record_id_image_record'), ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_positive_confirmation_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_positive_confirmation'))
|
||||
)
|
||||
op.create_index(op.f('ix_tag_positive_confirmation_tag_id'), 'tag_positive_confirmation', ['tag_id'], unique=False)
|
||||
op.create_table('tag_suggestion_rejection',
|
||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('rejected_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name='fk_tsr_image_record_id_image_record', ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name='fk_tsr_tag_id_tag', ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_suggestion_rejection'))
|
||||
)
|
||||
op.create_index('ix_tag_suggestion_rejection_tag', 'tag_suggestion_rejection', ['tag_id'], unique=False)
|
||||
op.create_table('character_prototype',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('ccip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=768), nullable=False),
|
||||
sa.Column('region_id', sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(['region_id'], ['image_region.id'], name=op.f('fk_character_prototype_region_id_image_region'), ondelete='SET NULL'),
|
||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_character_prototype_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_character_prototype'))
|
||||
)
|
||||
op.create_index(op.f('ix_character_prototype_region_id'), 'character_prototype', ['region_id'], unique=False)
|
||||
op.create_index(op.f('ix_character_prototype_tag_id'), 'character_prototype', ['tag_id'], unique=False)
|
||||
op.create_table('series_chapter',
|
||||
sa.Column('id', sa.Integer(), nullable=False),
|
||||
sa.Column('series_tag_id', sa.Integer(), nullable=False),
|
||||
sa.Column('anchor_page_id', sa.Integer(), nullable=False),
|
||||
sa.Column('title', sa.Text(), nullable=True),
|
||||
sa.Column('stated_part', sa.Integer(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||
sa.ForeignKeyConstraint(['anchor_page_id'], ['series_page.id'], name='fk_series_chapter_anchor_page', ondelete='CASCADE'),
|
||||
sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_chapter_series_tag_id_tag'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_series_chapter')),
|
||||
sa.UniqueConstraint('anchor_page_id', name='uq_series_chapter_anchor_page')
|
||||
)
|
||||
op.create_index(op.f('ix_series_chapter_series_tag_id'), 'series_chapter', ['series_tag_id'], unique=False)
|
||||
|
||||
# The singleton settings rows. NOT schema — see the note above; the app
|
||||
# reads these with scalar_one() and never creates them, so a fresh
|
||||
# install without these two rows raises NoResultFound on first use.
|
||||
# From 0002 and 0003.
|
||||
op.execute("INSERT INTO import_settings (id) VALUES (1)")
|
||||
op.execute("INSERT INTO ml_settings (id) VALUES (1)")
|
||||
|
||||
# The three hygiene system tags, from 0075. These are PRODUCT data, not
|
||||
# operator configuration — 0075's own docstring says so: "the fix keys on
|
||||
# SYSTEM tags the product ships". The presentation and process auto-apply
|
||||
# sweeps look them up with scalar_one(), so without these rows those
|
||||
# features raise NoResultFound rather than degrading.
|
||||
#
|
||||
# 0075 adopted an existing same-name general tag before inserting, because
|
||||
# an operator might already have tagged `wip` by hand. That cannot happen
|
||||
# on the empty database this file runs against, but the guard is kept: it
|
||||
# costs nothing and makes the statement safe to re-run.
|
||||
for _name in ("wip", "banner", "editor screenshot"):
|
||||
op.execute(
|
||||
sa.text(
|
||||
"INSERT INTO tag (name, kind, is_system) "
|
||||
"SELECT :name, 'general', true WHERE NOT EXISTS ("
|
||||
" SELECT 1 FROM tag WHERE lower(name) = lower(:name)"
|
||||
")"
|
||||
).bindparams(name=_name)
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Deliberately not implemented.
|
||||
|
||||
Downgrading a baseline means dropping every table in the database. That is
|
||||
not a migration, and offering it as one invites someone to run it. Restore
|
||||
from a backup instead.
|
||||
"""
|
||||
raise NotImplementedError(
|
||||
"0089 is the baseline; there is nothing below it. Restore from a backup."
|
||||
)
|
||||
@@ -1,64 +0,0 @@
|
||||
"""service_seen — the learned roster that makes a stopped part observable.
|
||||
|
||||
Milestone 365. Nothing in FabledCurator knew what was SUPPOSED to be running:
|
||||
`celery inspect` reports the workers that answer, so a dead worker was a
|
||||
shorter list rather than a red light, and the only surface that could tell an
|
||||
operator otherwise was Portainer. This table is the memory that turns an
|
||||
absence into something the app can see.
|
||||
|
||||
Keyed on the queue set for a celery role and on agent_id for the GPU agent —
|
||||
NOT on the celery worker name, which here is `celery@<container id>` and is
|
||||
minted fresh on every deploy. See the model docstring for why that choice is
|
||||
the whole design.
|
||||
|
||||
## First migration on the collapsed baseline
|
||||
|
||||
0089 is the single generated baseline that replaced revisions 0001..0089
|
||||
(milestone 328). This is the first revision written on top of it, so it is
|
||||
also the first evidence that the chain steps forward from the collapse rather
|
||||
than merely reproducing the schema — which nothing had demonstrated yet.
|
||||
|
||||
An existing install is at 0089 because it ran the real 0089; a fresh one is at
|
||||
0089 because it ran the baseline. Both arrive here identically, which was the
|
||||
property the collapse was designed around.
|
||||
|
||||
Revision ID: 0090
|
||||
Revises: 0089
|
||||
Create Date: 2026-09-02
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0090"
|
||||
down_revision: Union[str, None] = "0089"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"service_seen",
|
||||
sa.Column("key", sa.String(length=128), nullable=False),
|
||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||
sa.Column("display_name", sa.String(length=64), nullable=False),
|
||||
sa.Column(
|
||||
"first_seen_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"last_seen_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("key", name=op.f("pk_service_seen")),
|
||||
)
|
||||
# No secondary indexes, deliberately: one row per moving part means every
|
||||
# read is a handful of rows and an index would be write cost buying
|
||||
# nothing (#3301 removed seven of exactly that shape).
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("service_seen")
|
||||
@@ -1,81 +0,0 @@
|
||||
"""platform_membership — the learned roster of what the account actually pays for.
|
||||
|
||||
Milestone 387, phase C. FC knows which creators it was told to follow and
|
||||
nothing about which ones the operator is subscribed to; this table is the
|
||||
memory that makes the drift in both directions observable. See the model
|
||||
docstring for why the roster is learned rather than looked up live, and why
|
||||
`status` holds the platform's own word rather than a normalised FC value.
|
||||
|
||||
## Nothing populates this yet, on purpose
|
||||
|
||||
The sweep that fills it (C3) depends on a client seam (C2) that depends on
|
||||
characterising Patreon's real membership response from a captured sample (C0),
|
||||
which needs the operator's authenticated browser session. The table's SHAPE
|
||||
does not wait on that: it is deliberately free-form where C0's findings would
|
||||
otherwise dictate a column — `status` is an unconstrained String and `details`
|
||||
keeps the raw payload — so no capture can invalidate what is created here.
|
||||
|
||||
An empty table is the correct intermediate state. It is not dead code: C5 reads
|
||||
it to explain a tier-limited source, and C4 reads it to reconcile.
|
||||
|
||||
Revision ID: 0091
|
||||
Revises: 0090
|
||||
Create Date: 2026-09-10
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0091"
|
||||
down_revision: Union[str, None] = "0090"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"platform_membership",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||
# Text, not a bounded String: an opaque upstream identifier we do not
|
||||
# mint, and guessing a ceiling for one is how a walk dies on a silent
|
||||
# truncation.
|
||||
sa.Column("external_campaign_id", sa.Text(), nullable=False),
|
||||
sa.Column("display_name", sa.Text(), nullable=True),
|
||||
sa.Column("url", sa.Text(), nullable=True),
|
||||
# No CHECK, deliberately (rule 36 considered and declined): the
|
||||
# vocabulary is each platform's own and is not ours to fix before C0
|
||||
# has characterised even one of them. The service owns the whitelist.
|
||||
sa.Column("status", sa.String(length=32), nullable=True),
|
||||
sa.Column("tier_names", sa.JSON(), nullable=True),
|
||||
sa.Column("amount_cents", sa.Integer(), nullable=True),
|
||||
sa.Column("currency", sa.String(length=8), nullable=True),
|
||||
sa.Column(
|
||||
"first_seen_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"last_seen_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column("details", sa.JSON(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_platform_membership")),
|
||||
# The upsert's conflict target. Named explicitly because
|
||||
# touch_membership references it by name in ON CONFLICT — an
|
||||
# autogenerated name would make that call break on a rename nobody
|
||||
# connected to it.
|
||||
sa.UniqueConstraint(
|
||||
"platform", "external_campaign_id",
|
||||
name="uq_platform_membership_platform_campaign",
|
||||
),
|
||||
)
|
||||
# No secondary indexes. This table holds one row per subscription — tens,
|
||||
# not millions — so every query against it is a short scan and an index
|
||||
# would be write cost buying nothing (#3301 removed seven of that shape).
|
||||
# The unique constraint above already backs the only lookup that matters.
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("platform_membership")
|
||||
@@ -1,92 +0,0 @@
|
||||
"""Synthetic posts — FC authors a post for content that arrived as chat.
|
||||
|
||||
Milestone 388, step E2. Discord is a delivery channel, not a publisher: one
|
||||
message is not one post, and today every message becomes its own `post` row
|
||||
competing with authored work for the same surface. This adds the three columns
|
||||
that let FC group a creator's variant drop into a post it wrote itself, while
|
||||
keeping that fact visible and the grouping reversible.
|
||||
|
||||
## Why a flag and a back-pointer rather than a separate table
|
||||
|
||||
A synthetic post has to BE a post — same row, same columns — or every existing
|
||||
surface (feed, provenance, translation, attachments, series) would need a
|
||||
second code path for it. `synthesized_by` marks the ones FC authored;
|
||||
`absorbed_by_post_id` points a member message-post at the post that replaced
|
||||
it in the feed. The members are not deleted: they remain the images' true
|
||||
origin, and destroying them would make the grouping un-auditable at exactly
|
||||
the moment somebody wants to check it.
|
||||
|
||||
Reversal is one DELETE. `absorbed_by_post_id` is ON DELETE SET NULL, so
|
||||
removing a synthetic post releases its members and they return to the feed
|
||||
unaided.
|
||||
|
||||
Revision ID: 0092
|
||||
Revises: 0091
|
||||
Create Date: 2026-09-10
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0092"
|
||||
down_revision: Union[str, None] = "0091"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# No CHECK on synthesized_by (rule 36 considered and declined): there is one
|
||||
# grouper today and a second would be a new VALUE, not a new invariant —
|
||||
# matching source.error_type and service_seen.kind.
|
||||
op.add_column("post", sa.Column("synthesized_by", sa.String(length=32), nullable=True))
|
||||
op.add_column("post", sa.Column("synthesis_details", sa.JSON(), nullable=True))
|
||||
op.add_column(
|
||||
"post", sa.Column("absorbed_by_post_id", sa.Integer(), nullable=True),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_post_absorbed_by_post_id"), "post", ["absorbed_by_post_id"],
|
||||
)
|
||||
# SET NULL, not CASCADE: deleting the synthetic post must RELEASE its
|
||||
# members, never take them with it. The members are the real capture.
|
||||
op.create_foreign_key(
|
||||
"fk_post_absorbed_by_post_id_post", "post", "post",
|
||||
["absorbed_by_post_id"], ["id"], ondelete="SET NULL",
|
||||
)
|
||||
|
||||
# Grouping tunables. Every one of these is operator-facing (project rule
|
||||
# 25) because the quality bar here is a judgement call no test can settle:
|
||||
# too greedy merges distinct pieces, too shy leaves a drop scattered.
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_grouping_enabled", sa.Boolean(),
|
||||
server_default="true", nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_max_distance", sa.Float(),
|
||||
server_default=sa.text("0.10"), nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_window_minutes", sa.Float(),
|
||||
server_default=sa.text("60"), nullable=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("ml_settings", "discord_group_window_minutes")
|
||||
op.drop_column("ml_settings", "discord_group_max_distance")
|
||||
op.drop_column("ml_settings", "discord_grouping_enabled")
|
||||
op.drop_constraint("fk_post_absorbed_by_post_id_post", "post", type_="foreignkey")
|
||||
op.drop_index(op.f("ix_post_absorbed_by_post_id"), table_name="post")
|
||||
op.drop_column("post", "absorbed_by_post_id")
|
||||
op.drop_column("post", "synthesis_details")
|
||||
op.drop_column("post", "synthesized_by")
|
||||
@@ -1,86 +0,0 @@
|
||||
"""An open grouping — a synthetic post that a later drop can still join.
|
||||
|
||||
Milestone 388, step E3. E2's synthetic post was sealed at creation: a creator
|
||||
who added two more variants the next day started a second post. These two
|
||||
columns let the group stay open and absorb the follow-up, without the post
|
||||
either freezing or thrashing the feed.
|
||||
|
||||
## Why openness is derived rather than stored
|
||||
|
||||
There is no `closed_at` here on purpose. A group is open if it grew (or
|
||||
started) within `ml_settings.discord_group_close_after_hours`, so openness is a
|
||||
comparison rather than a state — which means lowering the setting closes old
|
||||
groups and raising it reopens them, with nothing to repair either way. A stored
|
||||
flag would need its own sweep to set it and its own repair path to ever change
|
||||
the policy, for no gain.
|
||||
|
||||
## Why `resurfaced_at` is separate from `last_grew_at`
|
||||
|
||||
They answer different questions. `last_grew_at` is when the group last
|
||||
absorbed something — it decides how long the group stays joinable and is what
|
||||
the card shows. `resurfaced_at` is the FEED POSITION, advanced only when the
|
||||
anti-thrash rule fires, so a group that gains one image a day updates in place
|
||||
while a genuine second wave moves once. Folding them together would make every
|
||||
addition a bump, which is the annoyance this step exists to avoid.
|
||||
|
||||
Both are NULL on every ordinary post, so the feed's sort key can COALESCE
|
||||
through `resurfaced_at` without moving anything that is not a grouping.
|
||||
|
||||
Revision ID: 0093
|
||||
Revises: 0092
|
||||
Create Date: 2026-09-10
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0093"
|
||||
down_revision: Union[str, None] = "0092"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"post", sa.Column("last_grew_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
op.add_column(
|
||||
"post", sa.Column("resurfaced_at", sa.DateTime(timezone=True), nullable=True),
|
||||
)
|
||||
# No index on either. The feed already sorts on an unindexed
|
||||
# COALESCE(post_date, downloaded_at) expression, so adding resurfaced_at to
|
||||
# that COALESCE changes nothing about how the query plans — and inventing a
|
||||
# functional index here would be guessing at the fix for a cost nobody has
|
||||
# measured. Measuring it is step B2's job.
|
||||
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_close_after_hours", sa.Float(),
|
||||
server_default=sa.text("168"), nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_resurface_min_images", sa.Integer(),
|
||||
server_default="2", nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"ml_settings",
|
||||
sa.Column(
|
||||
"discord_group_resurface_cooldown_hours", sa.Float(),
|
||||
server_default=sa.text("24"), nullable=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("ml_settings", "discord_group_resurface_cooldown_hours")
|
||||
op.drop_column("ml_settings", "discord_group_resurface_min_images")
|
||||
op.drop_column("ml_settings", "discord_group_close_after_hours")
|
||||
op.drop_column("post", "resurfaced_at")
|
||||
op.drop_column("post", "last_grew_at")
|
||||
@@ -1,124 +0,0 @@
|
||||
"""post_association — "this Patreon post announced that Discord drop".
|
||||
|
||||
Milestone 388, step E5.
|
||||
|
||||
Two of the operator's artists post a deliberately cropped fragment on Patreon
|
||||
to signal that the real thing has landed in their Discord. This table holds the
|
||||
proposed and accepted links between the announcement and the drop.
|
||||
|
||||
Directional and confirm-only. The pair is asymmetric (the teaser announces the
|
||||
drop, not the reverse), the two posts are never merged (the creator published
|
||||
twice, deliberately — flattening that hides the behaviour being modelled), and
|
||||
nothing is linked until the operator accepts, following the FC-6.3 series
|
||||
matcher. A wrongly-asserted association tells them two different pieces are
|
||||
one, which is worse than no link at all.
|
||||
|
||||
Dismissed rows are KEPT. The row is what remembers the rejection, and
|
||||
re-proposing a rejected pair on every scan is what makes a review queue get
|
||||
ignored.
|
||||
|
||||
Revision ID: 0094
|
||||
Revises: 0093
|
||||
Create Date: 2026-09-10
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0094"
|
||||
down_revision: Union[str, None] = "0093"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"post_association",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("announcement_post_id", sa.Integer(), nullable=False),
|
||||
sa.Column("payload_post_id", sa.Integer(), nullable=False),
|
||||
sa.Column("score", sa.Float(), nullable=False),
|
||||
sa.Column("signals", sa.JSON(), nullable=True),
|
||||
# No CHECK on status (rule 36 considered and declined), matching
|
||||
# series_suggestion.status — the same review-queue vocabulary, and the
|
||||
# same check-existing-enums lesson.
|
||||
sa.Column(
|
||||
"status", sa.String(length=16), server_default="pending", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_post_association")),
|
||||
# CASCADE on both sides: an association to a post that no longer exists
|
||||
# is not a fact worth keeping, and E3's reversal path (delete the
|
||||
# grouping) must not leave a dangling proposal behind.
|
||||
sa.ForeignKeyConstraint(
|
||||
["announcement_post_id"], ["post.id"], ondelete="CASCADE",
|
||||
name=op.f("fk_post_association_announcement_post_id_post"),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["payload_post_id"], ["post.id"], ondelete="CASCADE",
|
||||
name=op.f("fk_post_association_payload_post_id_post"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"announcement_post_id", "payload_post_id",
|
||||
name="uq_post_association_pair",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_post_association_announcement_post_id"),
|
||||
"post_association", ["announcement_post_id"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_post_association_payload_post_id"),
|
||||
"post_association", ["payload_post_id"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_post_association_status"), "post_association", ["status"],
|
||||
)
|
||||
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"discord_link_enabled", sa.Boolean(),
|
||||
server_default="true", nullable=False,
|
||||
),
|
||||
)
|
||||
# 0.60 sits ABOVE the largest single signal weight on purpose — see
|
||||
# post_association_service.WEIGHTS. That is what makes "time proximity
|
||||
# alone is never sufficient" arithmetic rather than aspirational.
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"discord_link_threshold", sa.Float(),
|
||||
server_default="0.60", nullable=False,
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"discord_link_window_hours", sa.Float(),
|
||||
server_default="24", nullable=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "discord_link_window_hours")
|
||||
op.drop_column("import_settings", "discord_link_threshold")
|
||||
op.drop_column("import_settings", "discord_link_enabled")
|
||||
op.drop_index(op.f("ix_post_association_status"), table_name="post_association")
|
||||
op.drop_index(
|
||||
op.f("ix_post_association_payload_post_id"), table_name="post_association",
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_post_association_announcement_post_id"), table_name="post_association",
|
||||
)
|
||||
op.drop_table("post_association")
|
||||
@@ -1,63 +0,0 @@
|
||||
"""membership_sync — whether the roster actually synced, and when.
|
||||
|
||||
Milestone 387, step C3.
|
||||
|
||||
`platform_membership` (0091) records what was SEEN. This records whether
|
||||
looking happened at all — a different fact, and the one that makes an empty
|
||||
roster readable.
|
||||
|
||||
Without it, three situations collapse into one: the account subscribes to
|
||||
nothing, the sweep never ran, or the sweep failed. All three leave zero rows
|
||||
in `platform_membership`. "You are tracking 12 sources you no longer subscribe
|
||||
to" is correct in the first case and an invitation to cancel things the
|
||||
operator is actively paying for in the other two, which is why C4 gates its
|
||||
CONCLUSIONS on `last_success_at` rather than merely displaying it.
|
||||
|
||||
Two timestamps rather than one, deliberately: `last_attempt_at` moves every
|
||||
run, `last_success_at` only on a clean walk, and the gap between them is what
|
||||
lets the UI say "last synced 3 days ago, tried 20 minutes ago, failing".
|
||||
|
||||
Revision ID: 0095
|
||||
Revises: 0094
|
||||
Create Date: 2026-09-11
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0095"
|
||||
down_revision: Union[str, None] = "0094"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"membership_sync",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||
sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_success_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("last_count", sa.Integer(), nullable=True),
|
||||
# No CHECK: this carries an exception class name, and the vocabulary is
|
||||
# whatever the client raises — same call as source.error_type.
|
||||
sa.Column("last_error_type", sa.String(length=64), nullable=True),
|
||||
sa.Column("last_error_message", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_membership_sync")),
|
||||
# The upsert's conflict target, named explicitly because the service
|
||||
# references it by name in ON CONFLICT.
|
||||
sa.UniqueConstraint("platform", name="uq_membership_sync_platform"),
|
||||
)
|
||||
# No secondary indexes: one row per platform, so every read is a short scan
|
||||
# and an index would be write cost buying nothing (#3301 removed seven of
|
||||
# that shape). Same reasoning as platform_membership in 0091.
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("membership_sync")
|
||||
@@ -1,106 +0,0 @@
|
||||
"""artist_membership_suggestion — proposing that a creator and a membership match.
|
||||
|
||||
Milestone 388, step E4.
|
||||
|
||||
## What this migration deliberately does NOT add
|
||||
|
||||
No association table between Artist and Source, and no schema change to either.
|
||||
E4's first job was to verify what was actually missing, and the answer was
|
||||
neither the model nor the flows: `Source.artist_id` is a plain FK so many
|
||||
sources per artist already works, `POST /api/sources` already takes an
|
||||
`artist_id`, the add-source dialog already attaches to an EXISTING artist, and
|
||||
`SourceService.reassign` already moves a source between artists with post and
|
||||
image re-attribution. Building a parallel association table for a relationship
|
||||
the schema already expresses would have been the mistake rule 28 names.
|
||||
|
||||
What was missing is the SUGGESTION, and that is all this table holds.
|
||||
|
||||
Accepting a suggestion adds a SOURCE under the existing artist — it never
|
||||
merges two artists. Adding a source is trivially undone; a wrong merge silently
|
||||
mixes two creators' work and corrupts tagging, series and provenance with
|
||||
nothing left to separate them by.
|
||||
|
||||
Revision ID: 0096
|
||||
Revises: 0095
|
||||
Create Date: 2026-09-11
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0096"
|
||||
down_revision: Union[str, None] = "0095"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"artist_membership_suggestion",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("platform_membership_id", sa.Integer(), nullable=False),
|
||||
sa.Column("artist_id", sa.Integer(), nullable=False),
|
||||
sa.Column("score", sa.Float(), nullable=False),
|
||||
sa.Column("signals", sa.JSON(), nullable=True),
|
||||
# No CHECK on status (rule 36 considered and declined), matching
|
||||
# series_suggestion and post_association — the same review-queue
|
||||
# vocabulary and the same check-existing-enums lesson.
|
||||
sa.Column(
|
||||
"status", sa.String(length=16), server_default="pending", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_artist_membership_suggestion")),
|
||||
# CASCADE both ways: a suggestion about a membership or an artist that
|
||||
# no longer exists is not a fact worth keeping, and a dangling proposal
|
||||
# would render as a broken row in the review queue.
|
||||
sa.ForeignKeyConstraint(
|
||||
["platform_membership_id"], ["platform_membership.id"],
|
||||
ondelete="CASCADE",
|
||||
name=op.f("fk_artist_membership_suggestion_membership"),
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["artist_id"], ["artist.id"], ondelete="CASCADE",
|
||||
name=op.f("fk_artist_membership_suggestion_artist_id_artist"),
|
||||
),
|
||||
sa.UniqueConstraint(
|
||||
"platform_membership_id", "artist_id",
|
||||
name="uq_artist_membership_suggestion_pair",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_artist_membership_suggestion_platform_membership_id"),
|
||||
"artist_membership_suggestion", ["platform_membership_id"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_artist_membership_suggestion_artist_id"),
|
||||
"artist_membership_suggestion", ["artist_id"],
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_artist_membership_suggestion_status"),
|
||||
"artist_membership_suggestion", ["status"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index(
|
||||
op.f("ix_artist_membership_suggestion_status"),
|
||||
table_name="artist_membership_suggestion",
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_artist_membership_suggestion_artist_id"),
|
||||
table_name="artist_membership_suggestion",
|
||||
)
|
||||
op.drop_index(
|
||||
op.f("ix_artist_membership_suggestion_platform_membership_id"),
|
||||
table_name="artist_membership_suggestion",
|
||||
)
|
||||
op.drop_table("artist_membership_suggestion")
|
||||
@@ -1,68 +0,0 @@
|
||||
"""Disable sources on retired platforms, so the scheduler stops selecting them.
|
||||
|
||||
Milestone #406, phase 1 (switch pixiv off). Rule #171 records the scope decision.
|
||||
|
||||
## Why this is a migration and not a button
|
||||
|
||||
The live instance had one pixiv source still ENABLED when pixiv was retired
|
||||
(read 2026-09-13, step 1) even though the operator believed it gone. Unregistering
|
||||
a platform removes it from code; it does not touch the `source` rows that name it.
|
||||
Left enabled, that row keeps being picked by the scheduler every interval, and
|
||||
`download_backends` now refuses it with `unsupported_url` — forever, as a
|
||||
climbing failure count on a source the operator has already given up.
|
||||
|
||||
A migration reaches the live instance on deploy without depending on anyone
|
||||
finding the row and clicking it. The `run_download` guard is what makes a stale
|
||||
enabled row SAFE; this is what makes it QUIET.
|
||||
|
||||
## Deliberately NOT done here
|
||||
|
||||
- **No rows are deleted.** Deleting a source sets its posts' `source_id` to NULL
|
||||
(FK `ON DELETE SET NULL`), and `uq_post_artist_external_id_null_source` can
|
||||
reject that if a source-less copy of one of those posts already exists. That
|
||||
needs checking against real data first, which is phase 2's job (step 6). A
|
||||
disable cannot collide with anything.
|
||||
- **No posts or images are touched.** The art stays.
|
||||
- **deviantart is included** because #3069 retired it and nothing disabled its
|
||||
rows either. The read found none, so for it this is a no-op — written anyway,
|
||||
so the statement names every retired platform rather than just the latest one.
|
||||
|
||||
## Hardcoded platform names
|
||||
|
||||
A migration is a record of one event, frozen in time, so it names the platforms
|
||||
it acted on rather than importing today's registry — the registry will keep
|
||||
changing and this revision must not.
|
||||
|
||||
Revision ID: 0097
|
||||
Revises: 0096
|
||||
Create Date: 2026-09-13
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0097"
|
||||
down_revision: Union[str, None] = "0096"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Clears the failure state the same way `SourceService.update` does when a
|
||||
# source is disabled through the app (issue #1285), so a retired source
|
||||
# does not keep showing as failing after it stops being polled. A disable
|
||||
# done here and one done by clicking must leave identical rows.
|
||||
op.execute(
|
||||
"UPDATE source SET enabled = false, last_error = NULL, "
|
||||
"error_type = NULL, consecutive_failures = 0 "
|
||||
"WHERE enabled AND platform IN ('pixiv', 'deviantart')"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Irreversible by design: which of these rows were enabled before is not
|
||||
# recorded, and re-enabling every retired-platform source would resume
|
||||
# polling services the product no longer supports. Rule #22 owes no
|
||||
# migration story back to a dropped platform.
|
||||
pass
|
||||
@@ -1,90 +0,0 @@
|
||||
"""Widen image_record.phash to 256-bit and re-hash the library (issue #4223).
|
||||
|
||||
The operator reported a 15-image variant pack landing as 3 records, and then
|
||||
that variants were STILL being dropped with `phash_threshold` at 0. Zero was
|
||||
already the floor of the dial, so no setting could have fixed it: at
|
||||
`hash_size=8` a pHash is 64 bits of coarse light/dark layout, and variant
|
||||
artwork sharing a composition produces the SAME 64 bits. Distance 0 meant
|
||||
"identical hash", never "identical image".
|
||||
|
||||
`utils/phash.py` moves to `hash_size=16` (256 bits, what ImageRepo always
|
||||
used) and adds an aspect-ratio gate plus a pixel-level confirm, so a merge is
|
||||
accepted on the files rather than on the hash.
|
||||
|
||||
## Why this NULLs every phash
|
||||
|
||||
Widening the column does not correct the values already in it. Every stored
|
||||
hash is a 64-bit hash of an image the app will now hash at 256 bits, and the
|
||||
two cannot be compared — `find_similar` skips a mismatched-length candidate
|
||||
rather than guessing, so leaving them would silently mean "no dedup, forever,
|
||||
for everything imported before today". NULL is the state `backfill_phash`
|
||||
already knows how to repair: it is NULL-only, keyset-paginated and
|
||||
restart-safe, and the beat schedule runs it daily.
|
||||
|
||||
Until that backfill finishes, image dedup degrades to sha256 only —
|
||||
duplicates may be kept. That is the safe direction, and the only one
|
||||
available: the alternative is comparing hashes of different widths, which
|
||||
would drop artwork. NOTHING here deletes or supersedes a file.
|
||||
|
||||
## Why the threshold is reset rather than carried over
|
||||
|
||||
`phash_threshold` counts bits, and the denominator went from 64 to 256. The
|
||||
stored number would keep its value while meaning something four times
|
||||
tighter. There is no honest carry-over, so every row goes to the new default
|
||||
of 24 — including the operator's 0, which was a workaround for the bug this
|
||||
revision fixes.
|
||||
|
||||
Revision ID: 0098
|
||||
Revises: 0097
|
||||
Create Date: 2026-09-21
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0098"
|
||||
down_revision: Union[str, None] = "0097"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# varchar(32) -> varchar(64): widening a length limit is a catalog-only
|
||||
# change in Postgres, so this does not rewrite the table or its index.
|
||||
op.alter_column(
|
||||
"image_record", "phash",
|
||||
existing_type=sa.String(32),
|
||||
type_=sa.String(64),
|
||||
existing_nullable=True,
|
||||
)
|
||||
op.execute("UPDATE image_record SET phash = NULL WHERE phash IS NOT NULL")
|
||||
op.alter_column(
|
||||
"import_settings", "phash_threshold",
|
||||
existing_type=sa.Integer(),
|
||||
server_default="24",
|
||||
existing_nullable=False,
|
||||
)
|
||||
op.execute("UPDATE import_settings SET phash_threshold = 24")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The 64-bit hashes this replaced are gone, and a 64-char value does not
|
||||
# fit back into varchar(32) — so the column is cleared again on the way
|
||||
# down and left for backfill_phash to refill at whatever HASH_SIZE the
|
||||
# code is running. Rule #22: no legacy to preserve.
|
||||
op.execute("UPDATE image_record SET phash = NULL WHERE phash IS NOT NULL")
|
||||
op.alter_column(
|
||||
"image_record", "phash",
|
||||
existing_type=sa.String(64),
|
||||
type_=sa.String(32),
|
||||
existing_nullable=True,
|
||||
)
|
||||
op.alter_column(
|
||||
"import_settings", "phash_threshold",
|
||||
existing_type=sa.Integer(),
|
||||
server_default="10",
|
||||
existing_nullable=False,
|
||||
)
|
||||
op.execute("UPDATE import_settings SET phash_threshold = 10")
|
||||
@@ -1,99 +0,0 @@
|
||||
"""library_placement_run — the placement reconciler's plan/apply/undo ledger.
|
||||
|
||||
Milestone #421 step 3. The survey (#4245) measured 33,789 ImageRecord rows
|
||||
sitting outside their artist's canonical directory, across 56 artists. This
|
||||
table holds one run of the sweep that trues them up: the plan, what it did,
|
||||
and where every file came from.
|
||||
|
||||
## Why the moves live in a table rather than a log line
|
||||
|
||||
`ImageRecord.path` is the only pointer at the bytes, so a move rewrites the
|
||||
row. Once that write lands, the previous location exists nowhere — unless it
|
||||
was recorded first. `moves` is that record, which is what makes a 33,789-file
|
||||
operation something the operator can undo per artist after looking at the
|
||||
result, rather than a one-way door.
|
||||
|
||||
An `applied` row is therefore HISTORY, not state (lesson #4226). Any future
|
||||
retention on this table may prune `ready`, `cancelled` and `error` runs; an
|
||||
`applied` one is only disposable once someone decides undo is no longer
|
||||
wanted. That is deliberately not a timer's decision, and no pruning is added
|
||||
here.
|
||||
|
||||
Revision ID: 0099
|
||||
Revises: 0098
|
||||
Create Date: 2026-09-21
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0099"
|
||||
down_revision: Union[str, None] = "0098"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"library_placement_run",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status", sa.String(length=16), server_default="running",
|
||||
nullable=False,
|
||||
),
|
||||
# SET NULL, not CASCADE: deleting an artist must not destroy the
|
||||
# record of where their files were moved.
|
||||
sa.Column("artist_id", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"started_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"planned_count", sa.Integer(), server_default="0", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"moved_count", sa.Integer(), server_default="0", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"refused_count", sa.Integer(), server_default="0", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"moves", postgresql.JSONB(astext_type=sa.Text()),
|
||||
server_default=sa.text("'[]'::jsonb"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"refusals", postgresql.JSONB(astext_type=sa.Text()),
|
||||
server_default=sa.text("'[]'::jsonb"), nullable=False,
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["artist_id"], ["artist.id"],
|
||||
name="fk_library_placement_run_artist_id", ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_placement_run_status", "library_placement_run", ["status"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_placement_run_artist_id", "library_placement_run",
|
||||
["artist_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Dropping this table destroys the only record of where moved files came
|
||||
# from. That is correct for a downgrade — the code that reads it is going
|
||||
# away too — but it is worth saying out loud rather than discovering.
|
||||
op.drop_index(
|
||||
"ix_library_placement_run_artist_id",
|
||||
table_name="library_placement_run",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_library_placement_run_status", table_name="library_placement_run",
|
||||
)
|
||||
op.drop_table("library_placement_run")
|
||||
@@ -1,109 +0,0 @@
|
||||
"""Drop library_placement_run — the placement reconciler is removed.
|
||||
|
||||
Milestone #421 built a sweep that compared each image's `artist_id` to the
|
||||
name of the directory its file sat in, and called every mismatch a misplaced
|
||||
file. On the operator's library that reported 33,789 of 63,605 images as
|
||||
wrongly filed.
|
||||
|
||||
That number was an artefact of the comparison, not a fact about the library:
|
||||
|
||||
- **97.1%** of it was one artist's own folder spelled differently —
|
||||
`Telepurte/` versus `telepurte/`. Same artist, same art, nothing wrong.
|
||||
- Of the 1% that sat in a differently-named folder, querying `ImageProvenance`
|
||||
— which records the post and source each file was actually downloaded from —
|
||||
showed 87 where provenance agreed with the FOLDER and not the record, and 40
|
||||
genuinely posted by several creators. The sweep would have misfiled or
|
||||
arbitrarily picked for roughly 41% of that set.
|
||||
|
||||
The system already knows where every file came from. The reconciler inferred
|
||||
it from a column and a directory name instead, and manufactured work out of a
|
||||
naming convention. Operator's call, 2026-09-21: *"the current system
|
||||
consistently records where items are and where they came from this is just
|
||||
complicating something works and doesn't need fixing."*
|
||||
|
||||
Rule #22 — no legacy to preserve. The table goes with the code.
|
||||
|
||||
## What is deliberately kept
|
||||
|
||||
`utils.paths.canonical_subdir` stays: new filesystem imports derive their
|
||||
directory from the artist's slug, matching what the downloader has always
|
||||
done. It is not part of this tool and removing it would be churn for no fix.
|
||||
Run 1's 327 moved files (`InsoUwu/` -> `insouwu/`) also stay where they are —
|
||||
same artist either way, and the gallery renders them correctly.
|
||||
|
||||
Revision ID: 0100
|
||||
Revises: 0099
|
||||
Create Date: 2026-09-21
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "0100"
|
||||
down_revision: Union[str, None] = "0099"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.drop_index(
|
||||
"ix_library_placement_run_artist_id",
|
||||
table_name="library_placement_run",
|
||||
)
|
||||
op.drop_index(
|
||||
"ix_library_placement_run_status", table_name="library_placement_run",
|
||||
)
|
||||
op.drop_table("library_placement_run")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Recreates the table only. The three runs it held (one applied, two
|
||||
# planned-and-never-run) are not restored and are not worth restoring —
|
||||
# the code that reads them is gone.
|
||||
op.create_table(
|
||||
"library_placement_run",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"status", sa.String(length=16), server_default="running",
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("artist_id", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"started_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"planned_count", sa.Integer(), server_default="0", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"moved_count", sa.Integer(), server_default="0", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"refused_count", sa.Integer(), server_default="0", nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"moves", postgresql.JSONB(astext_type=sa.Text()),
|
||||
server_default=sa.text("'[]'::jsonb"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"refusals", postgresql.JSONB(astext_type=sa.Text()),
|
||||
server_default=sa.text("'[]'::jsonb"), nullable=False,
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["artist_id"], ["artist.id"],
|
||||
name="fk_library_placement_run_artist_id", ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_placement_run_status", "library_placement_run", ["status"],
|
||||
)
|
||||
op.create_index(
|
||||
"ix_library_placement_run_artist_id", "library_placement_run",
|
||||
["artist_id"],
|
||||
)
|
||||
@@ -1,66 +0,0 @@
|
||||
"""Clear failure state on sources that are disabled (#4279).
|
||||
|
||||
`failing_sources_clause()` now means "enabled AND erroring", so a disabled
|
||||
source no longer counts as failing. That fixes what the surfaces REPORT; it
|
||||
does not touch what the rows already CARRY, and the rows are the reason the
|
||||
operator saw a banner for six days with no way to act on it (lesson #4202 —
|
||||
a guard does not undo the value already stored).
|
||||
|
||||
## The row this exists for
|
||||
|
||||
Ebi77 (source 19): the membership sweep stopped it as `former_patron` at
|
||||
02:50 on 2026-09-15 and correctly cleared its failure state. A deep scan was
|
||||
armed twenty minutes later — `/backfill` had no `enabled` guard, which this
|
||||
release also fixes — and could not complete without access, so the recovery
|
||||
sweep stranded it:
|
||||
|
||||
consecutive_failures = 1
|
||||
last_error = "stranded by recovery sweep (no terminal status after time_limit)"
|
||||
|
||||
Nothing could clear that. A disabled source is never scheduled, so no
|
||||
successful run resets the counter; `SourceService.update` clears failure
|
||||
state only on an explicit disable, and the source was already disabled; and
|
||||
the card's Retry routes to `/check`, which refuses a disabled source.
|
||||
|
||||
## Why every disabled source, not just that one
|
||||
|
||||
The clear matches what `SourceService.update` already does when a source is
|
||||
disabled through the app — "disable the subs you're not paying for without
|
||||
them lingering as failing" — so this brings rows disabled by any OTHER path
|
||||
(the membership sweep, a retired platform in 0097) into line with the rows
|
||||
disabled by hand. Same shape as 0097: a repair migration reaches the live
|
||||
instance on deploy rather than waiting for someone to find the row.
|
||||
|
||||
Enabled sources are untouched — a real failure on a live source must keep
|
||||
showing.
|
||||
|
||||
Revision ID: 0101
|
||||
Revises: 0100
|
||||
Create Date: 2026-09-21
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0101"
|
||||
down_revision: Union[str, None] = "0100"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
"UPDATE source SET last_error = NULL, error_type = NULL, "
|
||||
"consecutive_failures = 0 "
|
||||
"WHERE NOT enabled "
|
||||
"AND (last_error IS NOT NULL OR error_type IS NOT NULL "
|
||||
" OR consecutive_failures <> 0)"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Irreversible by design: the cleared strings and counts are not recorded
|
||||
# anywhere, and restoring a failure state nobody can act on would only
|
||||
# re-create the banner this removes. Rule #22 owes no story backwards.
|
||||
pass
|
||||
@@ -1,138 +0,0 @@
|
||||
"""Drop pixiv's ledgers, and delete credentials for platforms FC no longer has.
|
||||
|
||||
Milestone #406 step 6 (with issue #3980 folded in). Phase 1 unregistered pixiv
|
||||
and the commit alongside this one deleted its client, downloader, ingester and
|
||||
models. This removes the data those models described, and the stored secrets of
|
||||
every platform that has been retired.
|
||||
|
||||
## The two ledger tables
|
||||
|
||||
`pixiv_seen_media` and `pixiv_failed_media` are the per-source seen / dead-letter
|
||||
ledgers for a downloader that no longer exists. They were created in
|
||||
`0089_baseline.py`, so dropping them needs a new revision rather than an edit
|
||||
there.
|
||||
|
||||
## The credentials
|
||||
|
||||
Written as *delete every credential whose platform is not registered* rather
|
||||
than as `platform = 'pixiv'`, at the explicit ask in this step's plan. That is
|
||||
what makes one migration cover two retirements:
|
||||
|
||||
- **pixiv** — a live OAuth refresh token for a service FC no longer talks to.
|
||||
- **deviantart** — issue #3980. #3069 retired DeviantArt in code on 2026-08-27
|
||||
and left its stored session behind; seven weeks later it was still there.
|
||||
|
||||
And it is the only way either row can go. The credentials UI
|
||||
(`subscriptions/SettingsTab.vue`) renders one card per platform returned by
|
||||
`/api/platforms`, then looks the credential up by key — so a row whose platform
|
||||
is unregistered has no card, no Remove button, and no way for the operator to
|
||||
reach it. `CredentialService.list()` would return it; nothing asks.
|
||||
|
||||
The registered set is written out literally instead of importing
|
||||
`known_platform_keys()`. A migration is a statement about one moment in the
|
||||
schema's history: if it imported the live registry, retiring a fifth platform
|
||||
in 2027 would silently change what this 2026 revision did on a fresh database.
|
||||
The list below is the registry as of 2026-09-21.
|
||||
|
||||
## What is deliberately kept
|
||||
|
||||
**Every pixiv `Source` row.** The original plan deleted them; the operator's
|
||||
call on 2026-09-21 was to keep them, and the reason is that `platform` is
|
||||
stored ONLY on `Source` — neither `Post` nor `ImageRecord` carries it. Both
|
||||
FKs are `ON DELETE SET NULL`, so a delete would not lose the art, but it would
|
||||
drop every pixiv image into the gallery's `__unsourced__` bucket and strip the
|
||||
platform chip off every pixiv post. The rows stay disabled (0097) and their
|
||||
platform is unregistered, so nothing schedules them, nothing downloads through
|
||||
them, and `POST /api/sources` will not make another. Keeping them costs
|
||||
nothing and keeps the attribution the milestone's goal — *"the art already
|
||||
downloaded from pixiv stays"* — is actually about.
|
||||
|
||||
Every pixiv `Post` and `ImageRecord` is likewise untouched.
|
||||
|
||||
Revision ID: 0102
|
||||
Revises: 0101
|
||||
Create Date: 2026-09-21
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0102"
|
||||
down_revision: Union[str, None] = "0101"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# services/platforms/__init__.py's PLATFORMS as of this revision. See the
|
||||
# docstring for why this is a literal and not an import.
|
||||
_REGISTERED_PLATFORMS = ("patreon", "subscribestar", "hentaifoundry", "discord")
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.execute(
|
||||
sa.text(
|
||||
"DELETE FROM credential WHERE platform NOT IN :registered"
|
||||
).bindparams(
|
||||
sa.bindparam("registered", value=_REGISTERED_PLATFORMS, expanding=True)
|
||||
)
|
||||
)
|
||||
op.drop_index("ix_pixiv_failed_media_source_id", table_name="pixiv_failed_media")
|
||||
op.drop_table("pixiv_failed_media")
|
||||
op.drop_index("ix_pixiv_seen_media_source_id", table_name="pixiv_seen_media")
|
||||
op.drop_table("pixiv_seen_media")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The tables come back empty, and the credentials do not come back at all:
|
||||
# they were encrypted blobs, this migration does not copy them anywhere,
|
||||
# and restoring a live token for a platform FC cannot talk to would only
|
||||
# re-create the liability. Rule #22 owes no story backwards.
|
||||
op.create_table(
|
||||
"pixiv_seen_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_pixiv_seen_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_pixiv_seen_media")),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_seen_media_source_id",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_pixiv_seen_media_source_id"), "pixiv_seen_media", ["source_id"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"pixiv_failed_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("url", sa.Text(), nullable=True),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
|
||||
sa.Column(
|
||||
"created_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_pixiv_failed_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_pixiv_failed_media")),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_pixiv_failed_media_source_id",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_pixiv_failed_media_source_id"), "pixiv_failed_media", ["source_id"],
|
||||
unique=False,
|
||||
)
|
||||
@@ -1,121 +0,0 @@
|
||||
"""worker_lane — settings-backed slots for each celery lane.
|
||||
|
||||
Milestone 422 step 1. One row per lane, holding only what an operator can
|
||||
change: how many slots it runs, the ceiling they have set for themselves, and
|
||||
whether it consumes its queues at all.
|
||||
|
||||
## What is deliberately not a column
|
||||
|
||||
**The queues.** They are decided by `celery_app.py`'s `task_routes`, not by
|
||||
preference, so a stored copy could contradict the routing table with nothing
|
||||
to notice until a queue had no consumer. They live in
|
||||
`services/worker_lanes.py`.
|
||||
|
||||
**The derived ceiling.** Computed from the container's cgroup limits on every
|
||||
read. A row written on a 32GB host and later run in a 4GB container must be
|
||||
bounded by the 4GB; a stored ceiling would quietly authorise what the box can
|
||||
no longer hold.
|
||||
|
||||
## The seeded values
|
||||
|
||||
Written out literally rather than imported from `worker_lanes.LANES`. A
|
||||
migration is a statement about one moment in the schema's history — if it
|
||||
imported the live defaults, changing them in 2027 would silently change what
|
||||
this 2026 revision does on a fresh database. The two are allowed to diverge
|
||||
afterwards, and that is correct: `LANES` supplies defaults for a lane added
|
||||
later, this file records what was seeded today.
|
||||
|
||||
lane slots cap enabled
|
||||
worker 1 4 yes
|
||||
scheduler 1 2 yes
|
||||
maintenance_long 1 2 yes
|
||||
ml 0 1 NO
|
||||
|
||||
One of each, per the operator (2026-09-22: *"that starting value should be one
|
||||
of each"*), and far below their own production numbers — worker 8 and ml 2 are
|
||||
tuned for their hardware and are not a sane first boot for a stranger.
|
||||
|
||||
**ml ships at zero and disabled**, which is milestone 422 step 6's requirement
|
||||
arriving early: enabling the lane is what triggers the SigLIP download, and
|
||||
rule 164 permits a runtime fetch only for a feature that is "optional and
|
||||
clearly off". Seeding it on would make every fresh install reach HuggingFace.
|
||||
|
||||
The caps start low on purpose. A cap that begins at the ceiling is a rubber
|
||||
stamp; starting at 4/2/2/1 means raising slots within the cap is ordinary and
|
||||
raising the cap is a deliberate act.
|
||||
|
||||
## Existing installs
|
||||
|
||||
Nothing is migrated FROM. The `CELERY_QUEUES` / `CELERY_CONCURRENCY` env vars
|
||||
stay exactly as they are and remain the baseline each lane boots at; these
|
||||
rows are the adjustment applied on top (step 3). So this migration changes no
|
||||
behaviour on a running stack — it only makes the numbers storable.
|
||||
|
||||
Revision ID: 0103
|
||||
Revises: 0102
|
||||
Create Date: 2026-09-22
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0103"
|
||||
down_revision: Union[str, None] = "0102"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
# (name, slots, slots_cap, enabled) — see the docstring for why these are
|
||||
# literals and not an import.
|
||||
_SEED = (
|
||||
("worker", 1, 4, True),
|
||||
("scheduler", 1, 2, True),
|
||||
("maintenance_long", 1, 2, True),
|
||||
("ml", 0, 1, False),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
worker_lane = op.create_table(
|
||||
"worker_lane",
|
||||
sa.Column("name", sa.String(length=32), nullable=False),
|
||||
sa.Column("slots", sa.Integer(), nullable=False),
|
||||
sa.Column("slots_cap", sa.Integer(), nullable=False),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False),
|
||||
sa.Column(
|
||||
"updated_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("name", name=op.f("pk_worker_lane")),
|
||||
# Bare constraint names: Base.metadata's naming convention prepends
|
||||
# ck_worker_lane_, and pre-prefixing doubles it — the defect alembic
|
||||
# 0088 had to rename four constraints for (#3275). op.f() marks these
|
||||
# as already-final so autogenerate does not propose renaming them.
|
||||
sa.CheckConstraint("slots >= 0", name=op.f("ck_worker_lane_slots_non_negative")),
|
||||
sa.CheckConstraint("slots_cap >= 0", name=op.f("ck_worker_lane_cap_non_negative")),
|
||||
# The invariant that makes the cap mean anything, in the database
|
||||
# rather than only in the service: a row violating it is not a
|
||||
# rejected request, it is a lane that step 3's reconcile will drive UP
|
||||
# to a number the operator capped.
|
||||
sa.CheckConstraint("slots <= slots_cap", name=op.f("ck_worker_lane_slots_within_cap")),
|
||||
)
|
||||
# No index beyond the primary key, deliberately — four rows, forever. Same
|
||||
# reasoning as service_seen, and the lesson of #3301, which removed seven
|
||||
# indexes that were write cost buying nothing.
|
||||
op.bulk_insert(
|
||||
worker_lane,
|
||||
[
|
||||
{"name": name, "slots": slots, "slots_cap": cap, "enabled": enabled}
|
||||
for name, slots, cap, enabled in _SEED
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# The rows go with the table. They are settings with shipped defaults, not
|
||||
# operator data that predates this revision — a downgrade returns the stack
|
||||
# to reading its concurrency from env, which is where it reads it from
|
||||
# today anyway.
|
||||
op.drop_table("worker_lane")
|
||||
@@ -1,51 +0,0 @@
|
||||
"""worker_lane.autoscale — may this lane grow itself?
|
||||
|
||||
Milestone 422 step 7. One boolean, defaulting FALSE on every existing row and
|
||||
on every new one.
|
||||
|
||||
## Why the default is false and not "sensible"
|
||||
|
||||
This is the only part of the milestone that acts without anyone watching. The
|
||||
manual dial (step 4) and the reconcile (step 3) both do exactly what someone
|
||||
asked for; this one decides. Shipping it on would mean every install starts
|
||||
with a process that changes its own resource usage based on a heuristic tuned
|
||||
against nobody's workload.
|
||||
|
||||
Off also makes the failure mode benign: if the signal is wrong, nothing
|
||||
happens until an operator opts a lane in, and they opted in while watching.
|
||||
|
||||
## Why per lane and not one global switch
|
||||
|
||||
The lanes are not alike in what a slot costs. A `worker` slot is a process;
|
||||
an `ml` slot is another copy of a ~3.5GB model. A global switch would enable
|
||||
growth on a lane whose behaviour under load nobody has observed, and the one
|
||||
it would hurt most is the one whose cost is least visible.
|
||||
|
||||
Revision ID: 0104
|
||||
Revises: 0103
|
||||
Create Date: 2026-09-22
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0104"
|
||||
down_revision: Union[str, None] = "0103"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column(
|
||||
"worker_lane",
|
||||
sa.Column(
|
||||
"autoscale", sa.Boolean(),
|
||||
server_default=sa.text("false"), nullable=False,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("worker_lane", "autoscale")
|
||||
@@ -1,133 +0,0 @@
|
||||
"""worker_lane — one number: the cap. `slots`, `enabled` and `autoscale` go.
|
||||
|
||||
Milestone 422, reshaped by the operator 2026-09-23:
|
||||
|
||||
"auto should be always on, not a setting, so that idle instances quiet
|
||||
down when not running. the number that is visible and something the user
|
||||
can tweak and manage should be the cap itself the number of running
|
||||
workers is handled by the autoscaling function which is always on."
|
||||
|
||||
## What each dropped column was, and why it is not needed
|
||||
|
||||
**`slots`** — how many workers the lane should run. That is a MEASUREMENT,
|
||||
not a preference: the autoscaler moves the live pool between one and the cap
|
||||
according to the backlog, and reads it back from the worker every minute.
|
||||
Storing it made it look like something to keep in agreement with the cap,
|
||||
which is exactly what the operator had to do.
|
||||
|
||||
**`autoscale`** — whether the lane was allowed to size itself. It gated the
|
||||
mechanism behind a per-lane opt-in, so a lane nobody enabled simply never
|
||||
gave its slots back. Always on now, which is the only way "idle instances
|
||||
quiet down" can be true of an install nobody has configured.
|
||||
|
||||
**`enabled`** — whether the lane consumes its queues. Derived from `cap > 0`.
|
||||
It and `slots = 0` were two spellings of one fact and were free to disagree;
|
||||
this migration picks the one an operator can see.
|
||||
|
||||
## Why the caps are rewritten rather than preserved
|
||||
|
||||
The old defaults were 4 / 2 / 2 / 1, chosen when the number meant "the most
|
||||
you may raise SLOTS to" — a bound on a manual control, deliberately loose
|
||||
because moving within it was the ordinary act. The number now means "the most
|
||||
workers this lane may actually use", which is a different promise, and
|
||||
carrying the old figure over would silently quadruple the worker lane on
|
||||
every existing install at the moment this deploys.
|
||||
|
||||
So every row is reset to the new defaults: **one for each required lane, zero
|
||||
for ML.** That loses whatever an operator had set — which is the honest
|
||||
trade, because what they set was an answer to a different question. The UI
|
||||
now tells a busy lane's operator to raise its cap, which is how the number
|
||||
gets back up on an install that needs it.
|
||||
|
||||
ML at zero also keeps rule 164's carve-out intact: no consumers, so no model
|
||||
download until someone raises the cap.
|
||||
|
||||
Revision ID: 0105
|
||||
Revises: 0104
|
||||
Create Date: 2026-09-23
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0105"
|
||||
down_revision: Union[str, None] = "0104"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# (name, cap) — the same values `services/worker_lanes.LANES` declares. Seeded
|
||||
# here as literals rather than imported: a migration must describe the schema
|
||||
# at ITS point in history, and importing the live table would make this file
|
||||
# change meaning every time that table does.
|
||||
_CAPS = (
|
||||
("worker", 1),
|
||||
("scheduler", 1),
|
||||
("maintenance_long", 1),
|
||||
("ml", 0),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# The constraints go first: they name `slots`, so dropping the column out
|
||||
# from under them fails on Postgres.
|
||||
#
|
||||
# `op.f()` around each name, and it is load-bearing. Without it alembic
|
||||
# runs the name through Base.metadata's naming convention, which prepends
|
||||
# `ck_worker_lane_` to a string that already carries it — and the DROP
|
||||
# goes looking for `ck_worker_lane_ck_worker_lane_slots_within_cap`, which
|
||||
# no database has. That is #3275 exactly, from the other direction:
|
||||
# alembic 0088 had to RENAME four constraints created with the same
|
||||
# doubling. Caught here by the integration lane, run 7365.
|
||||
op.drop_constraint(
|
||||
op.f("ck_worker_lane_slots_within_cap"), "worker_lane", type_="check",
|
||||
)
|
||||
op.drop_constraint(
|
||||
op.f("ck_worker_lane_slots_non_negative"), "worker_lane", type_="check",
|
||||
)
|
||||
op.drop_column("worker_lane", "slots")
|
||||
op.drop_column("worker_lane", "enabled")
|
||||
op.drop_column("worker_lane", "autoscale")
|
||||
|
||||
# Reset to the new meaning. See the docstring: the old value answered a
|
||||
# different question, and carrying it over would raise every lane.
|
||||
for name, cap in _CAPS:
|
||||
op.execute(
|
||||
sa.text("UPDATE worker_lane SET slots_cap = :cap WHERE name = :name")
|
||||
.bindparams(cap=cap, name=name)
|
||||
)
|
||||
|
||||
# A lane the old seed never wrote — or one an operator added by hand — is
|
||||
# left alone rather than guessed at. `_rows_by_name` creates any missing
|
||||
# row at the lane's default on first read.
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.add_column(
|
||||
"worker_lane",
|
||||
sa.Column("slots", sa.Integer(), nullable=False, server_default="1"),
|
||||
)
|
||||
op.add_column(
|
||||
"worker_lane",
|
||||
sa.Column(
|
||||
"enabled", sa.Boolean(), nullable=False, server_default=sa.text("true"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"worker_lane",
|
||||
sa.Column(
|
||||
"autoscale", sa.Boolean(), nullable=False, server_default=sa.text("false"),
|
||||
),
|
||||
)
|
||||
# Restore the pre-0105 invariants. `slots` comes back as 1 everywhere and
|
||||
# the caps are 1/1/1/0, so a lane at cap 0 would violate `slots <= cap` —
|
||||
# hence the clamp before the constraint is added.
|
||||
op.execute(sa.text("UPDATE worker_lane SET slots = 0 WHERE slots_cap = 0"))
|
||||
op.execute(sa.text("UPDATE worker_lane SET enabled = (slots_cap > 0)"))
|
||||
op.create_check_constraint(
|
||||
op.f("ck_worker_lane_slots_non_negative"), "worker_lane", "slots >= 0",
|
||||
)
|
||||
op.create_check_constraint(
|
||||
op.f("ck_worker_lane_slots_within_cap"), "worker_lane", "slots <= slots_cap",
|
||||
)
|
||||
@@ -1,70 +0,0 @@
|
||||
"""service_seen — delete the roster rows the fixed code can no longer write.
|
||||
|
||||
Operator, 2026-09-23: *"clean up the stale service_seen rows"*. They were not
|
||||
stale. They were PHANTOMS, written on purpose by code that identified a celery
|
||||
worker from the queues it was consuming.
|
||||
|
||||
A lane at cap 0 has its consumers cancelled, so it answers `active_queues()`
|
||||
with an empty list. The roster grouped on that empty set, wrote it under the
|
||||
key `celery:` and rendered `role_display_name(())` as the display name — a row
|
||||
called **`Worker ()`**, reported as running, beside the real lane's row going
|
||||
stale because nothing updated it any more.
|
||||
|
||||
`worker_lanes.lane_for_node` fixes the cause: a worker is attributed by its
|
||||
NODE NAME, which survives having no consumers. Nothing will write `celery:`
|
||||
again.
|
||||
|
||||
## Why a migration and not a retention sweep
|
||||
|
||||
Lesson #4202: a guard that refuses to produce a bad value does not undo the
|
||||
bad value already stored. The row is the thing that has to change.
|
||||
|
||||
And it must be deleted rather than aged out, because the roster deliberately
|
||||
NEVER forgets — *"anything that has run at least once stays listed, that is
|
||||
what lets a stopped one be noticed rather than simply vanishing"*. A row that
|
||||
merely goes quiet is exactly what the roster is for. Only a row that cannot
|
||||
correspond to anything real is safe to remove, and `celery:` is precisely
|
||||
that: the empty queue set, which no correctly-attributed worker can produce.
|
||||
|
||||
## What is deliberately NOT deleted
|
||||
|
||||
**Celery rows with a real but unmatched queue set.** A deployment slicing
|
||||
`CELERY_QUEUES` differently is supported and its rows are true. It is not this
|
||||
migration's business to decide that somebody else's worker is obsolete.
|
||||
|
||||
**Agent rows, including a possible `agent:agent` from a build that omitted
|
||||
`agent_id`.** Nothing here can tell an abandoned agent id from a second agent
|
||||
that is currently down, and deleting a real one would hide a genuinely dead
|
||||
GPU agent — the one thing the roster exists to show. If such a row is present
|
||||
it needs a person to look at it, not a migration guessing.
|
||||
|
||||
Revision ID: 0106
|
||||
Revises: 0105
|
||||
Create Date: 2026-09-23
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0106"
|
||||
down_revision: Union[str, None] = "0105"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Exactly the one key the empty queue set produced. Matched literally
|
||||
# rather than by a LIKE or a prefix: `celery:` with nothing after it is
|
||||
# the phantom, and `celery:ml` is a real lane.
|
||||
op.execute(
|
||||
sa.text("DELETE FROM service_seen WHERE key = :key").bindparams(key="celery:")
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# Nothing. The row carried no information — an empty queue set and a
|
||||
# timestamp — and the roster re-learns anything real on its next refresh.
|
||||
# Re-creating it would put a phantom back.
|
||||
pass
|
||||
@@ -1,63 +0,0 @@
|
||||
"""worker_lane_sample — where the sizing sweep leaves what it measured.
|
||||
|
||||
Operator, 2026-09-23, on the System tab: *"there is a repull every time this
|
||||
page loads — is there a reason this info isn't being tracked in the
|
||||
background and stored in some way?"*
|
||||
|
||||
`/api/system/workers` ran a full celery inspect on every call — four
|
||||
broadcasts on an eleven-second budget — and the page polls it every fifteen
|
||||
seconds. `size_worker_lanes` was already inspecting on a timer to decide pool
|
||||
sizes, computing exactly these numbers and discarding them. This table is
|
||||
where they land instead, and the endpoint becomes a plain read.
|
||||
|
||||
## Why a new table rather than columns on `worker_lane`
|
||||
|
||||
`worker_lane` holds the one number an operator sets. Putting a measurement
|
||||
beside it is the mistake alembic 0105 undid: `slots` sat next to `slots_cap`,
|
||||
and a measurement next to a preference reads as a second preference.
|
||||
|
||||
No backfill. A row appears when the sweep first runs (within its period), and
|
||||
until then the lane reads as not-yet-measured, which is true.
|
||||
|
||||
Revision ID: 0107
|
||||
Revises: 0106
|
||||
Create Date: 2026-09-23
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0107"
|
||||
down_revision: Union[str, None] = "0106"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"worker_lane_sample",
|
||||
sa.Column("lane", sa.String(length=32), primary_key=True),
|
||||
# Nullable=False with no server_default: the sweep writes every column
|
||||
# on every upsert, so a row only ever exists complete.
|
||||
sa.Column("present", sa.Boolean(), nullable=False),
|
||||
sa.Column("replicas", sa.Integer(), nullable=False),
|
||||
# Nullable on purpose — unknown, never zero. A worker that answered
|
||||
# without reporting its pool, and a queue the broker did not answer
|
||||
# for, must not be summed as empty.
|
||||
sa.Column("pool", sa.Integer(), nullable=True),
|
||||
sa.Column("active", sa.Integer(), nullable=False),
|
||||
sa.Column("reserved", sa.Integer(), nullable=False),
|
||||
sa.Column("queue_depth", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"measured_at",
|
||||
sa.DateTime(timezone=True),
|
||||
nullable=False,
|
||||
server_default=sa.func.now(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table("worker_lane_sample")
|
||||
@@ -1,54 +0,0 @@
|
||||
"""download_revisit_days — how far back a tick keeps looking for EDITED posts.
|
||||
|
||||
Operator, 2026-09-23, pointing at a Floppystack post: *"this post has been
|
||||
updated as he implements hot fixes — any chance we have a way to scan for or
|
||||
see updated posts so we can update ours to match and pull the new attachments
|
||||
and pictures etc."*
|
||||
|
||||
A tick stopped after 20 contiguous already-have-it items. That is the right
|
||||
instinct and the wrong unit: a post edited three days after publication sits
|
||||
well below twenty seen items, so the walk turned around before reaching it. The
|
||||
walk now needs BOTH a run of seen items and a post older than this many days
|
||||
before it stops.
|
||||
|
||||
A settings row rather than a constant (rule 25) because the right window is a
|
||||
property of the CREATOR, not of FabledCurator — one artist appends hotfix
|
||||
builds for a fortnight, another never touches a post again. 0 turns the revisit
|
||||
off entirely and restores the pure count early-out.
|
||||
|
||||
30 days is the operator's own number, 2026-09-23.
|
||||
|
||||
Revision ID: 0108
|
||||
Revises: 0107
|
||||
Create Date: 2026-09-23
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0108"
|
||||
down_revision: Union[str, None] = "0107"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# server_default so the existing single settings row gets the window without
|
||||
# a data migration — and so an install that predates this column reads 30
|
||||
# rather than 0. 0 is a real, meaningful value here (revisit off), so the
|
||||
# column must never be allowed to arrive at it by omission.
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"download_revisit_days",
|
||||
sa.Integer(),
|
||||
nullable=False,
|
||||
server_default="30",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("import_settings", "download_revisit_days")
|
||||
@@ -1,44 +0,0 @@
|
||||
"""discord_link_auto — whether FC links a conclusive pair without asking.
|
||||
|
||||
Operator, 2026-09-24: *"I don't want this to be manual that defeats the
|
||||
convenience that I'm going for."*
|
||||
|
||||
Confirm-only was right while every signal was circumstantial. Time proximity
|
||||
and a body that mentions Discord can never be more than suggestive, so asking
|
||||
was the honest response. A shared working name is different in kind: when the
|
||||
creator's own name for a piece appears in exactly these two posts and nowhere
|
||||
else in their library, there is nothing left for the operator to adjudicate,
|
||||
and asking is just a chore FC invented for them.
|
||||
|
||||
Defaults ON, which is a real change of posture and deliberate. It only governs
|
||||
the conclusive band — weaker evidence still queues — and a link is a row the
|
||||
operator can dismiss, so the reversal is a click rather than a migration.
|
||||
|
||||
Revision ID: 0109
|
||||
Revises: 0108
|
||||
Create Date: 2026-09-24
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0109"
|
||||
down_revision = "0108"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"discord_link_auto",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.text("true"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("import_settings", "discord_link_auto")
|
||||
@@ -1,59 +0,0 @@
|
||||
"""The unified post card — fold window, family window, and who linked a pair.
|
||||
|
||||
Milestone 388, #4402 and #4401. A Patreon teaser's card shows the Discord drop
|
||||
it announced, and the rest of that piece's variants, by REFERENCE: nothing is
|
||||
absorbed, nothing changes owner, and every Discord post keeps its own place.
|
||||
|
||||
Three columns:
|
||||
|
||||
* `import_settings.discord_link_fold_hours` — a linked drop leaves the feed
|
||||
only when it is this close to its teaser (the same release, shown twice).
|
||||
* `import_settings.discord_family_window_days` — how far from the teaser the
|
||||
card reaches for variants. 60 is measured: named families spread up to 44
|
||||
days on artist 8, every collision found over 500.
|
||||
* `post_association.linked_by` — "fc" or "operator", so a link FC made by
|
||||
itself can say so on the card and offer the undo the operator asked for.
|
||||
|
||||
Revision ID: 0110
|
||||
Revises: 0109
|
||||
Create Date: 2026-09-24
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0110"
|
||||
down_revision = "0109"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"discord_link_fold_hours",
|
||||
sa.Float(),
|
||||
nullable=False,
|
||||
server_default=sa.text("24"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"discord_family_window_days",
|
||||
sa.Float(),
|
||||
nullable=False,
|
||||
server_default=sa.text("60"),
|
||||
),
|
||||
)
|
||||
op.add_column(
|
||||
"post_association",
|
||||
sa.Column("linked_by", sa.String(length=16), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_column("post_association", "linked_by")
|
||||
op.drop_column("import_settings", "discord_family_window_days")
|
||||
op.drop_column("import_settings", "discord_link_fold_hours")
|
||||
@@ -1,75 +0,0 @@
|
||||
"""Discord native ingester ledgers — seen and dead-letter, per source.
|
||||
|
||||
Milestone 428, #4415. Discord moves off gallery-dl onto the native core, which
|
||||
keeps its memory of what a source has already fetched in these two tables
|
||||
instead of gallery-dl's archive. Same shape as the SubscribeStar pair.
|
||||
|
||||
Revision ID: 0111
|
||||
Revises: 0110
|
||||
Create Date: 2026-09-24
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0111"
|
||||
down_revision = "0110"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"discord_seen_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("post_id", sa.String(length=64), nullable=True),
|
||||
sa.Column(
|
||||
"seen_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_discord_seen_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_seen_media")),
|
||||
sa.UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_discord_seen_media_source_id"), "discord_seen_media", ["source_id"],
|
||||
)
|
||||
op.create_table(
|
||||
"discord_failed_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"first_failed_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"last_failed_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_discord_failed_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_failed_media")),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_discord_failed_media_source_id",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_discord_failed_media_source_id"), "discord_failed_media", ["source_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index(op.f("ix_discord_failed_media_source_id"), table_name="discord_failed_media")
|
||||
op.drop_table("discord_failed_media")
|
||||
op.drop_index(op.f("ix_discord_seen_media_source_id"), table_name="discord_seen_media")
|
||||
op.drop_table("discord_seen_media")
|
||||
+7
-55
@@ -3,23 +3,13 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Quart, request
|
||||
from quart import Quart
|
||||
|
||||
from .api import all_blueprints
|
||||
from .config import get_config
|
||||
from .frontend import frontend_bp
|
||||
from .services.credential_crypto import CredentialCrypto
|
||||
|
||||
# Browser-extension origins. The FabledCurator extension fetches from
|
||||
# moz-extension://<uuid>/ on Firefox and chrome-extension://<uuid>/ on
|
||||
# Chromium-based browsers. Operator-flagged 2026-05-26: extension's
|
||||
# 'Test connection' returned `NetworkError` because the X-Extension-Key
|
||||
# header on /api/credentials triggers a CORS preflight that our routes
|
||||
# don't handle. Whitelisting only these two schemes (not opening CORS
|
||||
# up generally) lets the extension talk to a plain-HTTP self-hosted FC
|
||||
# without weakening the no-CORS posture for normal browser usage.
|
||||
_EXTENSION_ORIGIN_SCHEMES = ("moz-extension://", "chrome-extension://")
|
||||
|
||||
_CREDENTIAL_KEY_PATH = Path("/images/secrets/credential_key.b64")
|
||||
|
||||
|
||||
@@ -33,56 +23,18 @@ def create_app() -> Quart:
|
||||
|
||||
app = Quart(__name__)
|
||||
app.secret_key = cfg.secret_key
|
||||
|
||||
# Stream files in 4 MiB chunks instead of Quart's 8 KiB default. The image
|
||||
# library lives on a CIFS/SMB share (mounted rsize=4 MiB), so 8 KiB reads
|
||||
# meant ~19k network round-trips for one large original — 30–58s downloads
|
||||
# that starved both the GPU agent and the browser (operator-flagged
|
||||
# 2026-07-01). 4 MiB matches the mount's read size → one round-trip per read,
|
||||
# ~500× fewer. buffer_size is the MAX read, so small thumbnails still read in
|
||||
# a single gulp, and Range/mime/ETag/conditional handling lives on Response,
|
||||
# so this keeps all of it. Guarded so a future Quart-internal change can't
|
||||
# break boot — worst case we fall back to the slow default.
|
||||
try:
|
||||
from quart.wrappers.response import FileBody
|
||||
FileBody.buffer_size = 4 * 1024 * 1024
|
||||
except Exception:
|
||||
logging.getLogger(__name__).warning(
|
||||
"could not raise FileBody.buffer_size — file serving stays on 8 KiB chunks"
|
||||
)
|
||||
# FC-5: legacy IR ingest JSON can run to tens of MB (hundreds of
|
||||
# thousands of image_tag_associations). Werkzeug's default form
|
||||
# memory cap is 500KB; raise both ceilings so the multipart upload
|
||||
# for /api/migrate/ir_ingest doesn't 413.
|
||||
app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024 # 1 GB
|
||||
app.config["MAX_FORM_MEMORY_SIZE"] = 1024 * 1024 * 1024 # 1 GB
|
||||
|
||||
for bp in all_blueprints():
|
||||
app.register_blueprint(bp)
|
||||
# Registered last so /api/* routes win over the SPA catch-all.
|
||||
app.register_blueprint(frontend_bp)
|
||||
|
||||
@app.before_request
|
||||
async def _extension_cors_preflight():
|
||||
# Short-circuit OPTIONS preflight from the browser extension with a
|
||||
# 204 + CORS headers (the after_request hook below adds them).
|
||||
# Without this, OPTIONS lands on routes that only declared POST/GET
|
||||
# methods and 405s before the after_request gets a chance.
|
||||
if request.method != "OPTIONS":
|
||||
return None
|
||||
origin = request.headers.get("Origin", "")
|
||||
if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
|
||||
return "", 204
|
||||
return None
|
||||
|
||||
@app.after_request
|
||||
async def _extension_cors_headers(response):
|
||||
origin = request.headers.get("Origin", "")
|
||||
if any(origin.startswith(s) for s in _EXTENSION_ORIGIN_SCHEMES):
|
||||
response.headers["Access-Control-Allow-Origin"] = origin
|
||||
response.headers["Access-Control-Allow-Methods"] = (
|
||||
"GET, POST, PATCH, DELETE, OPTIONS"
|
||||
)
|
||||
response.headers["Access-Control-Allow-Headers"] = (
|
||||
"Content-Type, X-Extension-Key"
|
||||
)
|
||||
response.headers["Access-Control-Max-Age"] = "86400"
|
||||
return response
|
||||
|
||||
@app.after_serving
|
||||
async def _dispose_db_engine() -> None:
|
||||
from .extensions import dispose_engine
|
||||
|
||||
@@ -14,20 +14,16 @@ api_bp.add_url_rule("/health", view_func=health.get_health, methods=["GET"])
|
||||
|
||||
|
||||
def all_blueprints() -> list[Blueprint]:
|
||||
from .admin import admin_bp
|
||||
from .aliases import aliases_bp
|
||||
from .allowlist import allowlist_bp
|
||||
from .artist import artist_bp
|
||||
from .artists import artists_bp
|
||||
from .attachments import attachments_bp
|
||||
from .ccip import ccip_bp
|
||||
from .cleanup import cleanup_bp
|
||||
from .credentials import credentials_bp
|
||||
from .downloads import downloads_bp
|
||||
from .extension import extension_bp
|
||||
from .gallery import gallery_bp
|
||||
from .gpu import gpu_bp
|
||||
from .heads import heads_bp
|
||||
from .import_admin import import_admin_bp
|
||||
from .migrate import migrate_bp
|
||||
from .ml_admin import ml_admin_bp
|
||||
from .platforms import platforms_bp
|
||||
from .posts import posts_bp
|
||||
@@ -36,12 +32,7 @@ def all_blueprints() -> list[Blueprint]:
|
||||
from .showcase import showcase_bp
|
||||
from .sources import sources_bp
|
||||
from .suggestions import suggestions_bp
|
||||
from .system_activity import system_activity_bp
|
||||
from .system_backup import system_backup_bp
|
||||
from .system_health import system_health_bp
|
||||
from .tags import tags_bp
|
||||
from .thumbnails import thumbnails_bp
|
||||
from .workers import workers_bp
|
||||
return [
|
||||
api_bp,
|
||||
attachments_bp,
|
||||
@@ -52,24 +43,15 @@ def all_blueprints() -> list[Blueprint]:
|
||||
artists_bp,
|
||||
showcase_bp,
|
||||
settings_bp,
|
||||
system_activity_bp,
|
||||
workers_bp,
|
||||
system_health_bp,
|
||||
system_backup_bp,
|
||||
admin_bp,
|
||||
cleanup_bp,
|
||||
import_admin_bp,
|
||||
migrate_bp,
|
||||
suggestions_bp,
|
||||
allowlist_bp,
|
||||
aliases_bp,
|
||||
heads_bp,
|
||||
gpu_bp,
|
||||
ccip_bp,
|
||||
ml_admin_bp,
|
||||
thumbnails_bp,
|
||||
sources_bp,
|
||||
platforms_bp,
|
||||
posts_bp,
|
||||
credentials_bp,
|
||||
extension_bp,
|
||||
downloads_bp,
|
||||
]
|
||||
|
||||
@@ -1,16 +0,0 @@
|
||||
"""Shared API response helpers."""
|
||||
|
||||
from quart import jsonify
|
||||
|
||||
|
||||
def error_response(
|
||||
error: str, *, status: int = 400, detail: str | None = None, **extra,
|
||||
):
|
||||
"""JSON error body + HTTP status. `detail` is included only when given;
|
||||
`extra` keys are merged into the body. Returns the (response, status)
|
||||
tuple Quart expects. Imported as `_bad` by the blueprints."""
|
||||
body = {"error": error}
|
||||
if detail is not None:
|
||||
body["detail"] = detail
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
@@ -1,536 +0,0 @@
|
||||
"""FC-3k: /api/admin — destructive admin actions.
|
||||
|
||||
Action surfaces:
|
||||
POST /api/admin/artists/<slug>/cascade-delete (Tier C)
|
||||
POST /api/admin/images/bulk-delete (Tier C)
|
||||
DELETE /api/admin/tags/<int:tag_id> (Tier B)
|
||||
POST /api/admin/tags/<int:dest_id>/merge (Tier B)
|
||||
POST /api/admin/tags/prune-unused (Tier A)
|
||||
POST /api/admin/posts/prune-bare (Tier A)
|
||||
POST /api/admin/posts/refetch-external (Tier A)
|
||||
GET /api/admin/tags/<int:tag_id>/usage-count (helper)
|
||||
|
||||
Tier-C ops take a dry_run body flag (returns projection inline,
|
||||
no dispatch) and a confirm body field (server-recomputed token).
|
||||
Long-running ops dispatch a maintenance-queue Celery task; the UI
|
||||
tails FC-3i's /api/system/activity/runs to surface progress.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Artist, Post
|
||||
from ..services.cleanup_service import project_artist_cascade, project_bulk_image_delete
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
admin_bp = Blueprint("admin", __name__, url_prefix="/api/admin")
|
||||
|
||||
|
||||
def _bulk_image_confirm_token(image_ids: list[int]) -> str:
|
||||
"""Stable 8-hex token derived from the sorted id list. Mutates
|
||||
when the selection changes; stays the same across modal opens of
|
||||
the same selection so the operator can paste without confusion."""
|
||||
canon = ",".join(str(i) for i in sorted(image_ids))
|
||||
digest = hashlib.sha256(canon.encode("utf-8")).hexdigest()
|
||||
return digest[:8]
|
||||
|
||||
|
||||
async def _run_dry_run_op(service_fn, **service_kwargs):
|
||||
"""Shared body for the Tier-A dry-run/apply endpoints: read the `dry_run`
|
||||
flag, run the cleanup_service predicate under `run_sync`, and return its
|
||||
result dict. The SAME `service_fn` drives both preview and apply (the flag
|
||||
just toggles), so a handler physically can't let its preview diverge from
|
||||
its delete (rule 93). Default False preserves the existing contract — the UI
|
||||
always passes `dry_run` explicitly (true to preview, false to apply). Extra
|
||||
service kwargs (e.g. `source_id`) pass straight through."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda sync_sess: service_fn(sync_sess, dry_run=dry_run, **service_kwargs)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _queued(async_result):
|
||||
"""Standard 202 for an operator-triggered maintenance task: hand the UI the
|
||||
Celery task id so it can tail /maintenance/task-result (or the activity
|
||||
dashboard) for the summary. (trigger_vacuum stays bespoke — the UI doesn't
|
||||
poll it, so it returns no task id.)"""
|
||||
return jsonify({"task_id": async_result.id, "status": "queued"}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/artists/<slug>/cascade-delete", methods=["POST"])
|
||||
async def artist_cascade_delete(slug: str):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
supplied_confirm = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
artist = (await session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return _bad("not_found", status=404)
|
||||
artist_id = artist.id
|
||||
|
||||
projected = await session.run_sync(
|
||||
lambda sync_sess: project_artist_cascade(sync_sess, slug=slug)
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
return jsonify(projected)
|
||||
|
||||
expected = f"delete-artist-{artist_id}"
|
||||
if supplied_confirm != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
|
||||
from ..tasks.admin import delete_artist_cascade_task
|
||||
async_result = delete_artist_cascade_task.delay(artist_id=artist_id)
|
||||
return jsonify({"task_id": async_result.id}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/images/bulk-delete", methods=["POST"])
|
||||
async def images_bulk_delete():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
image_ids = body.get("image_ids")
|
||||
if not isinstance(image_ids, list) or not image_ids:
|
||||
return _bad("invalid_image_ids", detail="image_ids must be non-empty list of int")
|
||||
try:
|
||||
image_ids = [int(i) for i in image_ids]
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_image_ids", detail="image_ids must contain only ints")
|
||||
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
supplied_confirm = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
projected = await session.run_sync(
|
||||
lambda sync_sess: project_bulk_image_delete(
|
||||
sync_sess, image_ids=image_ids,
|
||||
)
|
||||
)
|
||||
|
||||
sha8 = _bulk_image_confirm_token(image_ids)
|
||||
expected = f"delete-images-{sha8}"
|
||||
|
||||
if dry_run:
|
||||
# Hand the canonical Tier-C confirm token back with the
|
||||
# projection so the frontend doesn't have to recompute SHA-256
|
||||
# client-side via crypto.subtle (Secure-Context-gated,
|
||||
# undefined on plain-HTTP origins per the homelab posture).
|
||||
# Operator-flagged 2026-05-27.
|
||||
projected["confirm_token"] = expected
|
||||
return jsonify(projected)
|
||||
|
||||
|
||||
if supplied_confirm != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
|
||||
from ..tasks.admin import bulk_delete_images_task
|
||||
async_result = bulk_delete_images_task.delay(image_ids=image_ids)
|
||||
return jsonify({"task_id": async_result.id}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/tags/<int:tag_id>", methods=["DELETE"])
|
||||
async def tag_delete(tag_id: int):
|
||||
"""Tier-B sync delete. UI yes/no modal is the only confirmation."""
|
||||
from ..services.cleanup_service import delete_tag
|
||||
|
||||
async with get_session() as session:
|
||||
try:
|
||||
result = await session.run_sync(
|
||||
lambda sync_sess: delete_tag(sync_sess, tag_id=tag_id)
|
||||
)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except ValueError as exc:
|
||||
# System tags (#128) — the training-hygiene machinery keys on
|
||||
# these rows.
|
||||
return _bad("system_tag", detail=str(exc))
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_bp.route("/tags/<int:dest_id>/merge", methods=["POST"])
|
||||
async def tag_merge(dest_id: int):
|
||||
"""Wraps TagService.merge. Source repoints to dest, dest survives,
|
||||
source row deleted, protective alias auto-created if source was
|
||||
ML-applied or allowlisted."""
|
||||
from ..services.tag_service import TagMergeConflict, TagService, TagValidationError
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
source_id = body.get("source_id")
|
||||
if not isinstance(source_id, int) or source_id == dest_id:
|
||||
return _bad("invalid_source_id", detail="source_id must be int and differ from dest")
|
||||
|
||||
# dry_run: non-mutating preview (counts + sample) so the operator can
|
||||
# confirm the target before the irreversible merge (#8, rule 93 parity).
|
||||
if body.get("dry_run"):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
p = await TagService(session).merge_preview(
|
||||
source_id=source_id, target_id=dest_id,
|
||||
)
|
||||
except TagValidationError as exc:
|
||||
return _bad("tag_not_found", status=404, detail=str(exc))
|
||||
return jsonify({
|
||||
"preview": {
|
||||
"source_id": p.source_id, "source_name": p.source_name,
|
||||
"target_id": p.target_id, "target_name": p.target_name,
|
||||
"compatible": p.compatible,
|
||||
"images_moving": p.images_moving,
|
||||
"images_already_on_target": p.images_already_on_target,
|
||||
"source_total": p.source_total,
|
||||
"series_pages": p.series_pages,
|
||||
"will_alias": p.will_alias,
|
||||
"sample_thumbnails": p.sample_thumbnails,
|
||||
},
|
||||
})
|
||||
|
||||
async with get_session() as session:
|
||||
try:
|
||||
result = await TagService(session).merge(
|
||||
source_id=source_id, target_id=dest_id,
|
||||
)
|
||||
except TagMergeConflict as exc:
|
||||
return _bad("merge_conflict", status=409, detail=str(exc))
|
||||
except TagValidationError as exc:
|
||||
return _bad("tag_kind_mismatch", detail=str(exc))
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
|
||||
# MergeResult is a frozen dataclass — flatten to dict.
|
||||
return jsonify({
|
||||
"result": {
|
||||
"target_id": result.target_id,
|
||||
"target_name": result.target_name,
|
||||
"target_kind": result.target_kind,
|
||||
"merged_count": result.merged_count,
|
||||
"alias_created": result.alias_created,
|
||||
"source_deleted": result.source_deleted,
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@admin_bp.route("/tags/<int:tag_id>/usage-count", methods=["GET"])
|
||||
async def tag_usage_count(tag_id: int):
|
||||
"""Helper for the Tier-B yes/no prompt; surfaces "N associations"
|
||||
in the dialog so the operator knows what they're nuking."""
|
||||
from ..services.cleanup_service import count_tag_associations
|
||||
|
||||
async with get_session() as session:
|
||||
count = await session.run_sync(
|
||||
lambda sync_sess: count_tag_associations(
|
||||
sync_sess, tag_id=tag_id,
|
||||
)
|
||||
)
|
||||
return jsonify({"count": count})
|
||||
|
||||
|
||||
@admin_bp.route("/tags/prune-unused", methods=["POST"])
|
||||
async def tags_prune_unused():
|
||||
"""Tier-A: dry-run preview list IS the prompt. UI calls with
|
||||
dry_run=true first, shows the list, operator clicks button to
|
||||
re-call with dry_run=false."""
|
||||
from ..services.cleanup_service import prune_unused_tags
|
||||
|
||||
return await _run_dry_run_op(prune_unused_tags)
|
||||
|
||||
|
||||
@admin_bp.route("/posts/prune-bare", methods=["POST"])
|
||||
async def posts_prune_bare():
|
||||
"""Tier-A: delete bare posts — Post rows with no linked images (primary OR
|
||||
provenance) and no attachments. Dry-run preview list IS the prompt: UI calls
|
||||
with dry_run=true first, shows the count + sample, operator confirms by
|
||||
re-calling with dry_run=false. Same preview/apply-parity predicate as the
|
||||
prune itself, so the preview can't diverge from the delete."""
|
||||
from ..services.cleanup_service import prune_bare_posts
|
||||
|
||||
return await _run_dry_run_op(prune_bare_posts)
|
||||
|
||||
|
||||
@admin_bp.route("/posts/reconcile-duplicates", methods=["POST"])
|
||||
async def posts_reconcile_duplicates():
|
||||
"""Tier-A: unify duplicate post rows for the same real post — the gallery-dl
|
||||
(attachment-id) + native (post-id) duplicates — onto ONE post-id-keyed keeper,
|
||||
moving image/provenance/attachment/link rows over. Images are untouched.
|
||||
dry_run=true returns {groups, posts_to_merge, sample}; dry_run=false applies
|
||||
and returns {groups, merged, sample}. Optional source_id scopes to one source.
|
||||
Same find_duplicate_post_groups predicate drives preview + apply (rule 93)."""
|
||||
from ..services.cleanup_service import reconcile_duplicate_posts
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
raw_source = body.get("source_id")
|
||||
try:
|
||||
source_id = int(raw_source) if raw_source is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_source_id", detail="source_id must be an integer")
|
||||
|
||||
return await _run_dry_run_op(reconcile_duplicate_posts, source_id=source_id)
|
||||
|
||||
|
||||
@admin_bp.route("/posts/refetch-external", methods=["POST"])
|
||||
async def posts_refetch_external():
|
||||
"""Surgical re-fetch of a post's external file-host links (operator
|
||||
2026-07-03): the normal cadence never re-walks deep back-catalogue posts,
|
||||
so a deleted external file only comes back by resetting its ExternalLink
|
||||
row(s) — this endpoint does that per post and dispatches the fetches.
|
||||
Sha-dedupe discards payload files that still exist, so only what's
|
||||
missing lands. Body: {external_post_id: str, source_id?: int (to
|
||||
disambiguate the same external id across sources)}."""
|
||||
from ..services.external_links import refetch_links_for_post
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
ext_id = str(body.get("external_post_id") or "").strip()
|
||||
if not ext_id:
|
||||
return _bad("missing_external_post_id",
|
||||
detail="external_post_id is required")
|
||||
raw_source = body.get("source_id")
|
||||
try:
|
||||
source_id = int(raw_source) if raw_source is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_source_id", detail="source_id must be an integer")
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = select(Post.id).where(Post.external_post_id == ext_id)
|
||||
if source_id is not None:
|
||||
stmt = stmt.where(Post.source_id == source_id)
|
||||
post_ids = (await session.execute(stmt)).scalars().all()
|
||||
if not post_ids:
|
||||
return _bad("post_not_found", status=404,
|
||||
detail=f"no post with external_post_id {ext_id!r}")
|
||||
results = {}
|
||||
for pid in post_ids:
|
||||
results[str(pid)] = await session.run_sync(
|
||||
lambda s, p=pid: refetch_links_for_post(s, p)
|
||||
)
|
||||
return jsonify({"posts": results})
|
||||
|
||||
|
||||
def _reset_content_confirm_token(projection: dict) -> str:
|
||||
"""Stable 8-hex token derived from the live counts (mirrors the Tier-C
|
||||
bulk-delete token): it changes whenever the data changes, so the apply can
|
||||
only ever run against numbers the operator just previewed."""
|
||||
canon = f"reset-content:{projection.get('count')}:{projection.get('applications')}"
|
||||
return hashlib.sha256(canon.encode("utf-8")).hexdigest()[:8]
|
||||
|
||||
|
||||
@admin_bp.route("/tags/reset-content", methods=["POST"])
|
||||
async def tags_reset_content():
|
||||
"""Full-instance reset of the CONTENT vocabulary: deletes ALL general +
|
||||
character tags and their image applications — INCLUDING the examples the
|
||||
tagging heads learned from. Suggestions do NOT repopulate on their own
|
||||
(the Camie predictions that once did are long retired): the operator
|
||||
re-tags from scratch and the heads retrain from the new signal. fandom +
|
||||
series tags + series_page ordering are preserved.
|
||||
|
||||
Deliberately Tier-C-gated despite the Tier-A shape (operator 2026-07-02:
|
||||
the full reset stays, but behind extra steps): dry_run returns the
|
||||
projection + a `confirm` token derived from the live counts; the apply
|
||||
must echo that token back or it is rejected."""
|
||||
from ..services.cleanup_service import reset_content_tagging
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
async with get_session() as session:
|
||||
projection = await session.run_sync(
|
||||
lambda s: reset_content_tagging(s, dry_run=True)
|
||||
)
|
||||
token = _reset_content_confirm_token(projection)
|
||||
if dry_run:
|
||||
projection["confirm"] = token
|
||||
return jsonify(projection)
|
||||
if str(body.get("confirm", "")) != token:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail="run a fresh preview and echo its confirm token",
|
||||
)
|
||||
result = await session.run_sync(
|
||||
lambda s: reset_content_tagging(s, dry_run=False)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@admin_bp.route("/tags/normalize", methods=["POST"])
|
||||
async def tags_normalize():
|
||||
"""#714: retro-normalize existing tags to the #701 canonical form (Title
|
||||
Case + collapsed whitespace) and merge case/whitespace-variant duplicates.
|
||||
|
||||
dry_run=true (default) returns a projection inline — group/collision/rename
|
||||
counts + a sample of the changes — so the UI shows exactly what'll happen.
|
||||
dry_run=false dispatches the long-running maintenance task (the merge FK
|
||||
repoints can touch many tags); the UI tails the activity dashboard for the
|
||||
summary. Idempotent; back up first (the merges are irreversible)."""
|
||||
from ..services.tag_service import normalize_existing_tags
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True))
|
||||
|
||||
if dry_run:
|
||||
async with get_session() as session:
|
||||
result = await normalize_existing_tags(session, dry_run=True)
|
||||
return jsonify(result)
|
||||
|
||||
from ..tasks.admin import normalize_tags_task
|
||||
|
||||
async_result = normalize_tags_task.delay()
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/db-stats", methods=["GET"])
|
||||
async def db_stats():
|
||||
"""Per-table bloat readout (pg_stat_user_tables) for the high-churn tables
|
||||
so the operator can see when a VACUUM is worth running."""
|
||||
from ..tasks.maintenance import VACUUM_TABLES
|
||||
|
||||
wanted = set(VACUUM_TABLES)
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(text(
|
||||
"SELECT relname, n_live_tup, n_dead_tup, last_vacuum, "
|
||||
"last_autovacuum, last_analyze FROM pg_stat_user_tables"
|
||||
))).all()
|
||||
|
||||
def _iso(v):
|
||||
return v.isoformat() if v is not None else None
|
||||
|
||||
out = []
|
||||
for r in rows:
|
||||
if r.relname not in wanted:
|
||||
continue
|
||||
live = r.n_live_tup or 0
|
||||
dead = r.n_dead_tup or 0
|
||||
total = live + dead
|
||||
out.append({
|
||||
"table": r.relname,
|
||||
"live": live,
|
||||
"dead": dead,
|
||||
"dead_pct": round(100 * dead / total, 1) if total else 0.0,
|
||||
"last_vacuum": _iso(r.last_vacuum),
|
||||
"last_autovacuum": _iso(r.last_autovacuum),
|
||||
"last_analyze": _iso(r.last_analyze),
|
||||
})
|
||||
out.sort(key=lambda t: t["dead"], reverse=True)
|
||||
return jsonify({"tables": out})
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/vacuum", methods=["POST"])
|
||||
async def trigger_vacuum():
|
||||
"""Operator-triggered VACUUM (ANALYZE) over the high-churn tables — the
|
||||
same maintenance-queue task the weekly Beat schedule runs."""
|
||||
from ..tasks.maintenance import vacuum_analyze
|
||||
|
||||
vacuum_analyze.delay()
|
||||
return jsonify({"status": "queued"}), 202
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/reextract-archives", methods=["POST"])
|
||||
async def trigger_reextract_archives():
|
||||
"""Operator-triggered re-extract (#713): PostAttachments that are actually
|
||||
archives but were filed opaquely (pre magic-byte gate) get extracted and
|
||||
their members linked to the post. Idempotent; runs on the maintenance queue."""
|
||||
from ..tasks.admin import reextract_archive_attachments_task
|
||||
|
||||
async_result = reextract_archive_attachments_task.delay()
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/prune-missing-files", methods=["POST"])
|
||||
async def trigger_prune_missing_files():
|
||||
"""Operator-triggered orphan repair (#859): delete ImageRecords whose backing
|
||||
file is gone from disk (e.g. left by the external-attach unlink bug), so they
|
||||
stop 404-ing on playback. The task aborts WITHOUT deleting if a large fraction
|
||||
of files look missing (a filesystem/NFS stall). Maintenance queue;
|
||||
operator-triggered only — never an unattended sweep."""
|
||||
from ..tasks.admin import prune_missing_file_records_task
|
||||
|
||||
async_result = prune_missing_file_records_task.delay()
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/reclaim-attachments", methods=["POST"])
|
||||
async def trigger_reclaim_attachments():
|
||||
"""Reclaim orphaned attachments (#3068). Body {"dry_run": bool}: dry_run
|
||||
(the DEFAULT here) projects the orphan rows and unreferenced store blobs
|
||||
without touching either; dry_run=false deletes the rows then unlinks every
|
||||
blob no surviving row references. Maintenance queue; operator-triggered
|
||||
only — never an unattended sweep, since the apply unlinks files. Returns the
|
||||
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import reclaim_orphaned_attachments_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview
|
||||
async_result = reclaim_orphaned_attachments_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/repair-discord-downloads", methods=["POST"])
|
||||
async def trigger_repair_discord_downloads():
|
||||
"""Clean re-download of the Discord files broken by the `None` naming
|
||||
(#3999). Body {"dry_run": bool}; dry_run is the DEFAULT, because the apply
|
||||
deletes files and makes gallery-dl forget every Discord download. Returns the
|
||||
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import repair_discord_downloads_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True))
|
||||
async_result = repair_discord_downloads_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/dedup-videos", methods=["POST"])
|
||||
async def trigger_dedup_videos():
|
||||
"""Tier-1 video dedup (#871). Body {"dry_run": bool}: dry_run=true previews
|
||||
what would be removed (groups / redundant count / reclaimable bytes) WITHOUT
|
||||
deleting; dry_run=false applies it (re-link posts to the keeper, then delete
|
||||
the redundant copies). Either way it first re-probes NULL-duration videos so
|
||||
the existing library participates. Returns the Celery task id — poll
|
||||
/maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import dedup_videos_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview
|
||||
async_result = dedup_videos_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/purge-gated-previews", methods=["POST"])
|
||||
async def trigger_purge_gated_previews():
|
||||
"""Cleanup (#874 follow-up). Body {"dry_run": bool}: dry_run=true previews how
|
||||
many blurred locked-preview images (grabbed from tier-gated Patreon posts
|
||||
before the fix) would be removed WITHOUT deleting; dry_run=false applies it.
|
||||
Re-walks every enabled Patreon source read-only and matches by content hash, so
|
||||
real content downloaded when access existed is provably spared. Returns the
|
||||
Celery task id — poll /maintenance/task-result/<id> for the summary."""
|
||||
from ..tasks.admin import purge_gated_previews_task
|
||||
|
||||
body = await request.get_json(silent=True) or {}
|
||||
dry_run = bool(body.get("dry_run", True)) # default to the SAFE preview
|
||||
async_result = purge_gated_previews_task.delay(dry_run=dry_run)
|
||||
return _queued(async_result)
|
||||
|
||||
|
||||
@admin_bp.route("/maintenance/task-result/<task_id>", methods=["GET"])
|
||||
async def maintenance_task_result(task_id: str):
|
||||
"""Poll a maintenance Celery task's result (the summary dict it returns).
|
||||
Used by the video-dedup card to show the dry-run projection before apply."""
|
||||
from ..celery_app import celery
|
||||
|
||||
res = celery.AsyncResult(task_id)
|
||||
ready = res.ready()
|
||||
return jsonify({
|
||||
"ready": ready,
|
||||
"successful": res.successful() if ready else None,
|
||||
"result": res.result if (ready and res.successful()) else None,
|
||||
})
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Allowlist API: list, adjust threshold, remove."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import TagAllowlist
|
||||
from ..services.ml.allowlist import AllowlistService
|
||||
|
||||
allowlist_bp = Blueprint("allowlist", __name__, url_prefix="/api")
|
||||
|
||||
|
||||
@allowlist_bp.route("/allowlist", methods=["GET"])
|
||||
async def list_allowlist():
|
||||
async with get_session() as session:
|
||||
rows = await AllowlistService(session).list_all()
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"tag_id": r.tag_id,
|
||||
"tag_name": r.tag_name,
|
||||
"tag_kind": r.tag_kind,
|
||||
"min_confidence": r.min_confidence,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["GET"])
|
||||
async def get_one(tag_id: int):
|
||||
async with get_session() as session:
|
||||
row = await session.get(TagAllowlist, tag_id)
|
||||
if row is None:
|
||||
return jsonify({"error": "not on allowlist"}), 404
|
||||
return jsonify(
|
||||
{"min_confidence": row.min_confidence, "added_at": row.added_at.isoformat()}
|
||||
)
|
||||
|
||||
|
||||
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["PATCH"])
|
||||
async def patch_threshold(tag_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "min_confidence" not in body:
|
||||
return jsonify({"error": "min_confidence required"}), 400
|
||||
mc = float(body["min_confidence"])
|
||||
if not (0 < mc <= 1):
|
||||
return jsonify({"error": "min_confidence must be in (0, 1]"}), 400
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).update_threshold(tag_id, mc)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@allowlist_bp.route("/tags/<int:tag_id>/allowlist", methods=["DELETE"])
|
||||
async def remove(tag_id: int):
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).remove(tag_id)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
@@ -31,24 +31,6 @@ async def create_or_get():
|
||||
}), 201
|
||||
|
||||
|
||||
@artists_bp.route("/<int:artist_id>", methods=["PATCH"])
|
||||
async def rename(artist_id: int):
|
||||
"""Rename an artist's DISPLAY NAME (#130). Name only — the slug and every
|
||||
on-disk path stay put, so this is instant and safe. Name is non-unique."""
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict) or not isinstance(body.get("name"), str):
|
||||
return jsonify({"error": "invalid_body"}), 400
|
||||
async with get_session() as session:
|
||||
svc = ArtistService(session)
|
||||
try:
|
||||
artist = await svc.rename(artist_id, body["name"])
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": "empty_name", "detail": str(exc)}), 400
|
||||
if artist is None:
|
||||
return jsonify({"error": "not_found"}), 404
|
||||
return jsonify({"id": artist.id, "name": artist.name, "slug": artist.slug})
|
||||
|
||||
|
||||
@artists_bp.route("/autocomplete", methods=["GET"])
|
||||
async def autocomplete():
|
||||
q = request.args.get("q") or ""
|
||||
@@ -65,16 +47,6 @@ async def autocomplete():
|
||||
])
|
||||
|
||||
|
||||
@artists_bp.route("/names", methods=["GET"])
|
||||
async def names():
|
||||
"""Every artist, id + name + slug, alphabetical. For filter pickers that
|
||||
list artists before anything is typed; `autocomplete` deliberately returns
|
||||
nothing for an empty query."""
|
||||
async with get_session() as session:
|
||||
rows = await ArtistService(session).all_names()
|
||||
return jsonify([{"id": i, "name": n, "slug": s} for i, n, s in rows])
|
||||
|
||||
|
||||
@artists_bp.route("/directory", methods=["GET"])
|
||||
async def directory():
|
||||
"""FC-3f: cursor-paginated artists directory.
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
"""CCIP / region observability API (#114) — read-only, analysis-shaped.
|
||||
|
||||
So the work can be checked through an API as the agent fills in vectors: overall
|
||||
coverage (regions by kind, how many images have figure CCIP vectors, which
|
||||
characters have enough reference examples to match on) + a per-image drill-down
|
||||
(its regions + the CCIP character matches it would get). Mirrors the heads
|
||||
metrics endpoint; no GPU, just reads what's stored.
|
||||
"""
|
||||
|
||||
from quart import Blueprint, jsonify
|
||||
from sqlalchemy import distinct, func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import ImageRegion, Tag, TagKind
|
||||
from ..models.tag import image_tag
|
||||
from ..services.ml.ccip import match_image
|
||||
|
||||
ccip_bp = Blueprint("ccip", __name__, url_prefix="/api/ccip")
|
||||
|
||||
_FIGURE_KINDS = ("face", "figure")
|
||||
|
||||
|
||||
@ccip_bp.route("/overview", methods=["GET"])
|
||||
async def overview():
|
||||
async with get_session() as session:
|
||||
by_kind = dict(
|
||||
(
|
||||
await session.execute(
|
||||
select(ImageRegion.kind, func.count()).group_by(ImageRegion.kind)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
images_with_figure_ccip = (
|
||||
await session.execute(
|
||||
select(func.count(distinct(ImageRegion.image_record_id)))
|
||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||
)
|
||||
).scalar_one()
|
||||
# Concept-crop (SigLIP bag) coverage — how far the back-catalogue embed
|
||||
# has progressed, so the max-over-bag scorer's reach is checkable.
|
||||
images_with_concept_siglip = (
|
||||
await session.execute(
|
||||
select(func.count(distinct(ImageRegion.image_record_id)))
|
||||
.where(ImageRegion.kind == "concept")
|
||||
.where(ImageRegion.siglip_embedding.is_not(None))
|
||||
)
|
||||
).scalar_one()
|
||||
# Per-character reference counts (no vectors loaded) — which characters
|
||||
# have enough examples to match on.
|
||||
ref_rows = (
|
||||
await session.execute(
|
||||
select(image_tag.c.tag_id, Tag.name, func.count())
|
||||
.select_from(ImageRegion)
|
||||
.join(
|
||||
image_tag,
|
||||
image_tag.c.image_record_id == ImageRegion.image_record_id,
|
||||
)
|
||||
.join(Tag, Tag.id == image_tag.c.tag_id)
|
||||
.where(Tag.kind == TagKind.character)
|
||||
.where(ImageRegion.kind.in_(_FIGURE_KINDS))
|
||||
.where(ImageRegion.ccip_embedding.is_not(None))
|
||||
.group_by(image_tag.c.tag_id, Tag.name)
|
||||
.order_by(func.count().desc())
|
||||
)
|
||||
).all()
|
||||
versions = [
|
||||
v for (v,) in (
|
||||
await session.execute(
|
||||
select(distinct(ImageRegion.embedding_version))
|
||||
)
|
||||
).all() if v
|
||||
]
|
||||
auto_applied = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(image_tag).where(
|
||||
image_tag.c.source == "ccip_auto"
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
return jsonify({
|
||||
"regions_by_kind": by_kind,
|
||||
"images_with_figure_ccip": images_with_figure_ccip,
|
||||
"images_with_concept_siglip": images_with_concept_siglip,
|
||||
"characters_with_references": len(ref_rows),
|
||||
"character_references": [
|
||||
{"tag_id": t, "name": n, "n_refs": c} for (t, n, c) in ref_rows
|
||||
],
|
||||
"embedding_versions": versions,
|
||||
"auto_applied": auto_applied,
|
||||
})
|
||||
|
||||
|
||||
@ccip_bp.route("/images/<int:image_id>", methods=["GET"])
|
||||
async def image_detail(image_id: int):
|
||||
"""An image's stored regions + the CCIP character matches it would get —
|
||||
for spot-checking the agent's output + the matcher."""
|
||||
async with get_session() as session:
|
||||
regions = (
|
||||
await session.execute(
|
||||
select(ImageRegion)
|
||||
.where(ImageRegion.image_record_id == image_id)
|
||||
.order_by(ImageRegion.id)
|
||||
)
|
||||
).scalars().all()
|
||||
matches = await match_image(session, image_id)
|
||||
return jsonify({
|
||||
"image_id": image_id,
|
||||
"regions": [
|
||||
{
|
||||
"id": r.id,
|
||||
"kind": r.kind,
|
||||
"bbox": [r.rx, r.ry, r.rw, r.rh],
|
||||
"frame_time": r.frame_time,
|
||||
"score": r.score,
|
||||
"detector_version": r.detector_version,
|
||||
"embedding_version": r.embedding_version,
|
||||
"has_ccip": r.ccip_embedding is not None,
|
||||
"has_siglip": r.siglip_embedding is not None,
|
||||
}
|
||||
for r in regions
|
||||
],
|
||||
"ccip_matches": matches,
|
||||
})
|
||||
@@ -1,198 +0,0 @@
|
||||
"""FC-Cleanup: /api/cleanup/* — retroactive enforcement of import filters.
|
||||
|
||||
Endpoints:
|
||||
POST /min-dimension/preview synchronous SQL audit
|
||||
POST /min-dimension/delete synchronous SQL delete (Tier-C token)
|
||||
POST /audit async transparency / single_color start
|
||||
GET /audit list recent audit_run rows
|
||||
GET /audit/<id> single audit_run row
|
||||
POST /audit/<id>/apply apply matched_ids deletes (Tier-C token)
|
||||
POST /audit/<id>/cancel flip running audit to cancelled
|
||||
|
||||
Unused-tags retroactive prune intentionally NOT in this namespace —
|
||||
TagMaintenanceCard (Maintenance tab → moved to Cleanup tab in v26.05.25.7)
|
||||
uses the existing /api/admin/tags/prune-unused endpoint via the admin
|
||||
store. No duplicate route here.
|
||||
|
||||
Confirm-token format matches modal/DestructiveConfirmModal.vue convention:
|
||||
`delete-min-dim-<sha8(w,h)>` for min-dim delete
|
||||
`delete-audit-<id>` for audit apply
|
||||
(Modal hardcodes action ∈ {'restore', 'delete'}; "apply audit" is semantically a delete of the matched images, so we use `delete-audit-<id>`.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import LibraryAuditRun
|
||||
from ..services import cleanup_service
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
cleanup_bp = Blueprint("cleanup", __name__, url_prefix="/api/cleanup")
|
||||
|
||||
IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
def _min_dim_token(min_w: int, min_h: int) -> str:
|
||||
# SHA-256 (not MD5) — Web Crypto's subtle.digest rejects MD5; both
|
||||
# sides use SHA-256 truncated to 8 hex chars.
|
||||
canon = f"{min_w}x{min_h}"
|
||||
return f"delete-min-dim-{hashlib.sha256(canon.encode()).hexdigest()[:8]}"
|
||||
|
||||
|
||||
def _serialize_audit_run(audit: LibraryAuditRun) -> dict:
|
||||
return {
|
||||
"id": audit.id,
|
||||
"rule": audit.rule,
|
||||
"params": audit.params,
|
||||
"status": audit.status,
|
||||
"started_at": audit.started_at.isoformat() if audit.started_at else None,
|
||||
"finished_at": audit.finished_at.isoformat() if audit.finished_at else None,
|
||||
"scanned_count": audit.scanned_count,
|
||||
"matched_count": audit.matched_count,
|
||||
"matched_ids": audit.matched_ids,
|
||||
"error": audit.error,
|
||||
}
|
||||
|
||||
|
||||
@cleanup_bp.route("/min-dimension/preview", methods=["POST"])
|
||||
async def min_dim_preview():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
min_w = int(body.get("min_width", 0))
|
||||
min_h = int(body.get("min_height", 0))
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_dimensions")
|
||||
if min_w < 0 or min_h < 0:
|
||||
return _bad("invalid_dimensions")
|
||||
async with get_session() as session:
|
||||
projection = await session.run_sync(
|
||||
lambda s: cleanup_service.project_min_dimension_violations(
|
||||
s, min_width=min_w, min_height=min_h,
|
||||
)
|
||||
)
|
||||
# Hand the canonical Tier-C delete token back with the preview so
|
||||
# the frontend doesn't have to recompute SHA-256 client-side.
|
||||
# window.crypto.subtle is Secure-Context-gated and undefined on
|
||||
# plain-HTTP origins (homelab posture); without this the Delete
|
||||
# button silently swallowed the TypeError and never opened the
|
||||
# confirm modal. Operator-flagged 2026-05-27.
|
||||
projection["confirm_token"] = _min_dim_token(min_w, min_h)
|
||||
return jsonify(projection)
|
||||
|
||||
|
||||
@cleanup_bp.route("/min-dimension/delete", methods=["POST"])
|
||||
async def min_dim_delete():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
min_w = int(body.get("min_width", 0))
|
||||
min_h = int(body.get("min_height", 0))
|
||||
except (TypeError, ValueError):
|
||||
return _bad("invalid_dimensions")
|
||||
if min_w < 0 or min_h < 0:
|
||||
return _bad("invalid_dimensions")
|
||||
supplied = body.get("confirm", "")
|
||||
expected = _min_dim_token(min_w, min_h)
|
||||
if supplied != expected:
|
||||
return _bad("confirm_mismatch", expected=expected)
|
||||
async with get_session() as session:
|
||||
deleted = await session.run_sync(
|
||||
lambda s: cleanup_service.delete_min_dimension_violations(
|
||||
s, min_width=min_w, min_height=min_h, images_root=IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"deleted": deleted})
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit", methods=["POST"])
|
||||
async def audit_create():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
rule = body.get("rule")
|
||||
params = body.get("params") or {}
|
||||
if rule not in ("transparency", "single_color"):
|
||||
return _bad("invalid_rule")
|
||||
if not isinstance(params, dict):
|
||||
return _bad("invalid_params")
|
||||
async with get_session() as session:
|
||||
try:
|
||||
audit_id = await session.run_sync(
|
||||
lambda s: cleanup_service.start_audit_run(
|
||||
s, rule=rule, params=params,
|
||||
)
|
||||
)
|
||||
except cleanup_service.AuditAlreadyRunning as running_id:
|
||||
return _bad(
|
||||
"audit_already_running", status=409,
|
||||
running_id=int(str(running_id)),
|
||||
)
|
||||
except ValueError as exc:
|
||||
return _bad(str(exc))
|
||||
await session.commit()
|
||||
return jsonify({"audit_id": audit_id, "status": "running"}), 202
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit/<int:audit_id>", methods=["GET"])
|
||||
async def audit_get(audit_id: int):
|
||||
async with get_session() as session:
|
||||
audit = (await session.execute(
|
||||
select(LibraryAuditRun).where(LibraryAuditRun.id == audit_id)
|
||||
)).scalar_one_or_none()
|
||||
if audit is None:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(_serialize_audit_run(audit))
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit", methods=["GET"])
|
||||
async def audit_history():
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "20")), 100)
|
||||
except ValueError:
|
||||
return _bad("invalid_limit")
|
||||
# Optional rule filter so a card can reconnect to ITS latest run on mount
|
||||
# (?rule=transparency&limit=1) — the audit survives navigation; the UI
|
||||
# rehydrates from this rather than losing the in-flight scan.
|
||||
rule = request.args.get("rule") or None
|
||||
async with get_session() as session:
|
||||
stmt = select(LibraryAuditRun).order_by(LibraryAuditRun.id.desc())
|
||||
if rule is not None:
|
||||
stmt = stmt.where(LibraryAuditRun.rule == rule)
|
||||
rows = (await session.execute(stmt.limit(limit))).scalars().all()
|
||||
return jsonify({"runs": [_serialize_audit_run(r) for r in rows]})
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit/<int:audit_id>/apply", methods=["POST"])
|
||||
async def audit_apply(audit_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
confirm = body.get("confirm", "")
|
||||
async with get_session() as session:
|
||||
try:
|
||||
deleted = await session.run_sync(
|
||||
lambda s: cleanup_service.apply_audit_run(
|
||||
s, audit_id=audit_id, confirm_token=confirm,
|
||||
images_root=IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
except cleanup_service.AuditNotReady as exc:
|
||||
return _bad("audit_not_ready", current_status=str(exc))
|
||||
except cleanup_service.ConfirmTokenMismatch as exc:
|
||||
return _bad("confirm_mismatch", expected=str(exc))
|
||||
except ValueError as exc:
|
||||
return _bad("not_found", status=404, detail=str(exc))
|
||||
await session.commit()
|
||||
return jsonify({"deleted": deleted})
|
||||
|
||||
|
||||
@cleanup_bp.route("/audit/<int:audit_id>/cancel", methods=["POST"])
|
||||
async def audit_cancel(audit_id: int):
|
||||
async with get_session() as session:
|
||||
await session.run_sync(
|
||||
lambda s: cleanup_service.cancel_audit_run(s, audit_id=audit_id)
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"cancelled": True})
|
||||
@@ -20,7 +20,6 @@ from ..services.credential_service import (
|
||||
UnknownPlatformError,
|
||||
WrongAuthTypeError,
|
||||
)
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
credentials_bp = Blueprint("credentials", __name__, url_prefix="/api/credentials")
|
||||
|
||||
@@ -39,6 +38,14 @@ def _get_crypto() -> CredentialCrypto:
|
||||
return _crypto
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
||||
body = {"error": error}
|
||||
if detail is not None:
|
||||
body["detail"] = detail
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
async def _ext_key_ok(session) -> bool:
|
||||
"""If X-Extension-Key is supplied, it must match the stored value.
|
||||
Missing header → True (browser path; accepted per homelab posture).
|
||||
@@ -117,58 +124,3 @@ async def delete_credential(platform: str):
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
return "", 204
|
||||
|
||||
|
||||
@credentials_bp.route("/<platform>/verify", methods=["POST"])
|
||||
async def verify_credential(platform: str):
|
||||
"""Test the stored credential against one of the platform's enabled sources,
|
||||
WITHOUT downloading. Routes through the platform's backend
|
||||
(download_backends.verify_credential) — native ingester for Patreon, an
|
||||
authenticated API page; gallery-dl --simulate for the rest. On success
|
||||
stamps last_verified. Returns {valid: bool|null, reason, last_verified?};
|
||||
valid=null means "couldn't test" (no credential, no enabled source, or an
|
||||
inconclusive network/drift result)."""
|
||||
from ..models import Artist, Source
|
||||
from ..services.download_backends import verify_source_credential
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_ok(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
svc = CredentialService(session, _get_crypto())
|
||||
record = await svc.get(platform)
|
||||
if record is None:
|
||||
return jsonify({"valid": None, "reason": "No credential stored for this platform."})
|
||||
|
||||
# Pick an enabled source for this platform to point the probe at.
|
||||
row = (await session.execute(
|
||||
select(Source, Artist)
|
||||
.join(Artist, Artist.id == Source.artist_id)
|
||||
.where(Source.platform == platform, Source.enabled.is_(True))
|
||||
.order_by(Source.id.asc())
|
||||
)).first()
|
||||
if row is None:
|
||||
return jsonify({
|
||||
"valid": None,
|
||||
"reason": "No enabled source for this platform to verify against — add a subscription first.",
|
||||
})
|
||||
source, artist = row
|
||||
|
||||
cookies_path = await svc.get_cookies_path(platform)
|
||||
auth_token = await svc.get_token(platform)
|
||||
|
||||
ok, message = await verify_source_credential(
|
||||
platform=platform,
|
||||
url=source.url,
|
||||
artist_slug=artist.slug,
|
||||
config_overrides=source.config_overrides or {},
|
||||
cookies_path=str(cookies_path) if cookies_path else None,
|
||||
auth_token=auth_token,
|
||||
images_root=Path("/images"),
|
||||
)
|
||||
|
||||
last_verified = None
|
||||
if ok:
|
||||
async with get_session() as session:
|
||||
ts = await CredentialService(session, _get_crypto()).mark_verified(platform)
|
||||
last_verified = ts.isoformat() if ts else None
|
||||
return jsonify({"valid": ok, "reason": message, "last_verified": last_verified})
|
||||
|
||||
@@ -5,10 +5,8 @@ status/source/artist. Returns slim records.
|
||||
Detail view: full DownloadEvent including the metadata JSONB.
|
||||
"""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Artist, DownloadEvent, Source
|
||||
@@ -44,9 +42,6 @@ def _list_record(event: DownloadEvent, source: Source | None, artist: Artist | N
|
||||
"bytes_downloaded": event.bytes_downloaded,
|
||||
"error": event.error,
|
||||
"summary": _summary_from_metadata(event.metadata_),
|
||||
# plan #709: mid-walk live counts for a RUNNING native-ingester event
|
||||
# (None otherwise; phase 3 overwrites metadata with run_stats on finish).
|
||||
"live": (event.metadata_ or {}).get("live"),
|
||||
}
|
||||
|
||||
|
||||
@@ -100,83 +95,6 @@ async def list_downloads():
|
||||
return jsonify([_list_record(e, s, a) for e, s, a in rows])
|
||||
|
||||
|
||||
@downloads_bp.route("/stats", methods=["GET"])
|
||||
async def downloads_stats():
|
||||
"""Status-grouped count over download_event for the dashboard stat chips.
|
||||
|
||||
`?window_hours=` (default 24) bounds by `started_at`. The full set of
|
||||
statuses is always present in the response (zero for missing) so the
|
||||
UI doesn't have to fill in defaults.
|
||||
"""
|
||||
try:
|
||||
window_hours = int(request.args.get("window_hours", "24"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_window_hours"}), 400
|
||||
if window_hours < 1 or window_hours > 24 * 365:
|
||||
return jsonify({"error": "invalid_window_hours"}), 400
|
||||
|
||||
since = datetime.now(UTC) - timedelta(hours=window_hours)
|
||||
out = {"pending": 0, "running": 0, "ok": 0, "error": 0, "skipped": 0}
|
||||
async with get_session() as session:
|
||||
stmt = (
|
||||
select(DownloadEvent.status, func.count())
|
||||
.where(DownloadEvent.started_at >= since)
|
||||
.group_by(DownloadEvent.status)
|
||||
)
|
||||
for status, n in (await session.execute(stmt)).all():
|
||||
if status in out:
|
||||
out[status] = int(n)
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
@downloads_bp.route("/activity", methods=["GET"])
|
||||
async def downloads_activity():
|
||||
"""Hourly download-event counts over the last `?hours=` (default 24).
|
||||
|
||||
Returns a fixed-length, oldest-first bucket array so the UI can render
|
||||
a sparkline directly. Bucketing is done in Python against UTC to dodge
|
||||
session-timezone ambiguity in SQL date_trunc.
|
||||
"""
|
||||
try:
|
||||
hours = int(request.args.get("hours", "24"))
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_hours"}), 400
|
||||
hours = max(1, min(168, hours))
|
||||
|
||||
now = datetime.now(UTC)
|
||||
end = now.replace(minute=0, second=0, microsecond=0)
|
||||
start = end - timedelta(hours=hours - 1)
|
||||
buckets = [
|
||||
{"hour": (start + timedelta(hours=i)).isoformat(),
|
||||
"ok": 0, "error": 0, "other": 0, "total": 0}
|
||||
for i in range(hours)
|
||||
]
|
||||
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(DownloadEvent.started_at, DownloadEvent.status)
|
||||
.where(DownloadEvent.started_at >= start)
|
||||
)).all()
|
||||
|
||||
for started_at, status in rows:
|
||||
if started_at is None:
|
||||
continue
|
||||
sa = started_at if started_at.tzinfo else started_at.replace(tzinfo=UTC)
|
||||
idx = int((sa - start).total_seconds() // 3600)
|
||||
if not (0 <= idx < hours):
|
||||
continue
|
||||
b = buckets[idx]
|
||||
if status == "ok":
|
||||
b["ok"] += 1
|
||||
elif status == "error":
|
||||
b["error"] += 1
|
||||
else:
|
||||
b["other"] += 1
|
||||
b["total"] += 1
|
||||
|
||||
return jsonify({"hours": hours, "buckets": buckets})
|
||||
|
||||
|
||||
@downloads_bp.route("/<int:event_id>", methods=["GET"])
|
||||
async def get_download(event_id: int):
|
||||
async with get_session() as session:
|
||||
@@ -190,20 +108,3 @@ async def get_download(event_id: int):
|
||||
return jsonify({"error": "not_found"}), 404
|
||||
event, source, artist = row
|
||||
return jsonify(_detail_record(event, source, artist))
|
||||
|
||||
|
||||
@downloads_bp.route("/recover-stalled", methods=["POST"])
|
||||
async def recover_stalled():
|
||||
"""Trigger the recover_stalled_download_events sweep on demand.
|
||||
|
||||
The same sweep runs every 5 min via Beat (see celery_app.beat_schedule);
|
||||
this endpoint exists so the operator can force-clear stuck pending/
|
||||
running download_events from the Subscriptions → Downloads maintenance
|
||||
menu without waiting for the next scheduled tick.
|
||||
"""
|
||||
# Local import: avoids registering maintenance tasks during blueprint
|
||||
# import (Celery task discovery races with the API import otherwise).
|
||||
from ..tasks.maintenance import recover_stalled_download_events
|
||||
|
||||
recover_stalled_download_events.delay()
|
||||
return jsonify({"queued": True}), 202
|
||||
|
||||
@@ -1,195 +0,0 @@
|
||||
"""FC-3g: /api/extension — quick-add-source for the Firefox extension
|
||||
+ install-time manifest for the Settings card.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import hmac
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..build_info import FC_CHANNEL as _FC_CHANNEL
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting
|
||||
from ..services.extension_service import (
|
||||
ExtensionService,
|
||||
InvalidUrlError,
|
||||
UnknownArtistError,
|
||||
UnknownPlatformError,
|
||||
)
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
extension_bp = Blueprint("extension", __name__, url_prefix="/api/extension")
|
||||
|
||||
# Default XPI directory; tests override via monkeypatching this module-
|
||||
# level constant.
|
||||
XPI_DIR = Path("/app/frontend/dist/extension")
|
||||
|
||||
_XPI_VERSION_RE = re.compile(r"fabledcurator-(?P<version>[\w.-]+)\.xpi$")
|
||||
|
||||
# Which channel this image belongs to — "dev" or "main" — baked in at build
|
||||
# time (milestone 271 step 7). Read from build_info rather than the environment
|
||||
# a second time: /api/health reports the same value, and two independent
|
||||
# `os.environ.get` calls are two things that can drift.
|
||||
#
|
||||
# Still bound as a module-level name here, so tests monkeypatch
|
||||
# `extension.FC_CHANNEL` exactly as they did before, same as XPI_DIR above.
|
||||
FC_CHANNEL = _FC_CHANNEL
|
||||
|
||||
|
||||
async def _ext_key_required(session) -> bool:
|
||||
"""Unlike /api/credentials (which accepts the browser path with no
|
||||
header), quick-add-source writes server state and must be explicitly
|
||||
authenticated."""
|
||||
supplied = request.headers.get("X-Extension-Key")
|
||||
if supplied is None:
|
||||
return False
|
||||
stored = (await session.execute(
|
||||
select(AppSetting.value).where(AppSetting.key == "extension_api_key")
|
||||
)).scalar_one_or_none()
|
||||
if stored is None:
|
||||
return False
|
||||
# compare_digest, not `==`: the stored key is a shared secret, and a
|
||||
# short-circuiting compare leaks its prefix through timing. Costs nothing
|
||||
# here — it is not that this route is exposed (#3072). Compared as BYTES:
|
||||
# compare_digest's str form rejects non-ASCII with TypeError, and this
|
||||
# header is attacker-supplied, so a str compare would turn a junk key into
|
||||
# a 500 instead of a 403.
|
||||
return hmac.compare_digest(supplied.encode("utf-8"), stored.encode("utf-8"))
|
||||
|
||||
|
||||
def _extract_version(xpi_name: str) -> str:
|
||||
m = _XPI_VERSION_RE.search(xpi_name)
|
||||
return m.group("version") if m else "unknown"
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fp:
|
||||
for chunk in iter(lambda: fp.read(65536), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
@extension_bp.route("/probe", methods=["GET"])
|
||||
async def probe_source():
|
||||
"""Read-only resolution of a creator-page URL: tells the extension
|
||||
whether this URL is already a Source, is for an Artist that exists
|
||||
but with a different URL, is brand new, or doesn't match any known
|
||||
platform pattern. Drives the content-script chip's color/copy
|
||||
BEFORE the operator clicks, so the button can show 'already added'
|
||||
without requiring an add-attempt."""
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return _bad("invalid_body", detail="url query parameter is required")
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
# crypto lets a Discord probe name the server and channel with the
|
||||
# stored token; every other platform ignores it.
|
||||
result = await ExtensionService(session, _get_crypto()).probe(url)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@extension_bp.route("/quick-add-source", methods=["POST"])
|
||||
async def quick_add_source():
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
url = body.get("url")
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
# Optional: connect the new source to an existing artist (artist_id) or to
|
||||
# the artist of that name (artist_name). A Discord channel names no
|
||||
# creator, so the extension's Add panel always sends one of them.
|
||||
artist_id = body.get("artist_id")
|
||||
if artist_id is not None and (isinstance(artist_id, bool) or not isinstance(artist_id, int)):
|
||||
return _bad("invalid_body", detail="artist_id must be an integer")
|
||||
artist_name = body.get("artist_name")
|
||||
if artist_name is not None and not isinstance(artist_name, str):
|
||||
return _bad("invalid_body", detail="artist_name must be a string")
|
||||
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
# crypto lets an add resolve the artist's display name via the
|
||||
# stored credential (else it falls back to the URL handle). #130.
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(
|
||||
url, artist_id=artist_id, artist_name=artist_name,
|
||||
)
|
||||
except UnknownArtistError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad(
|
||||
"unknown_platform",
|
||||
detail=str(exc),
|
||||
known=sorted(KNOWN_PLATFORMS),
|
||||
)
|
||||
except InvalidUrlError as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
return jsonify(result), (201 if result["created_source"] else 200)
|
||||
|
||||
|
||||
def _read_manifest_sync() -> dict | None:
|
||||
"""All the filesystem-touching work for /api/extension/manifest,
|
||||
in a sync helper so the async route can dispatch it via
|
||||
asyncio.to_thread (ASYNC240: no pathlib I/O in async functions)."""
|
||||
if not XPI_DIR.is_dir():
|
||||
return None
|
||||
# Exclude the `fabledcurator-latest.xpi` alias when picking the file to
|
||||
# extract a version from — it's a copy of the latest versioned XPI,
|
||||
# written at the same mtime by build.yml, and would otherwise tie or
|
||||
# win the sort (operator-flagged 2026-05-26: UI displayed "v latest"
|
||||
# because `_extract_version("fabledcurator-latest.xpi")` returns
|
||||
# the literal "latest"). The alias still serves as `latest_url`.
|
||||
versioned = [
|
||||
p for p in XPI_DIR.glob("fabledcurator-*.xpi")
|
||||
if p.name != "fabledcurator-latest.xpi"
|
||||
]
|
||||
if not versioned:
|
||||
return None
|
||||
versioned.sort(key=lambda p: p.stat().st_mtime)
|
||||
latest = versioned[-1]
|
||||
info = {
|
||||
"installed": True,
|
||||
"version": _extract_version(latest.name),
|
||||
"xpi_url": f"/extension/{latest.name}",
|
||||
"latest_url": "/extension/fabledcurator-latest.xpi",
|
||||
"sha256": _sha256(latest),
|
||||
}
|
||||
# The channel goes BESIDE the version, never inside it. A `-dev` suffix is
|
||||
# what silently disabled the dev channel in the sibling project this design
|
||||
# comes from: the comparator returned nothing for a non-integer segment, so
|
||||
# every dev version compared equal and "no update available" became
|
||||
# indistinguishable from "I cannot read this version".
|
||||
#
|
||||
# Omitted rather than defaulted when unset. Absence already has a meaning
|
||||
# every reader must handle — an image built before this field existed says
|
||||
# exactly the same thing by not having the key — so a blank channel reuses
|
||||
# that path instead of inventing a second "unknown" spelling.
|
||||
#
|
||||
# Reported verbatim, not validated against {"dev", "main"}: if an image
|
||||
# declares something else, showing what it actually claims is more useful
|
||||
# to whoever is debugging it than dropping the value on the floor.
|
||||
if FC_CHANNEL:
|
||||
info["channel"] = FC_CHANNEL
|
||||
return info
|
||||
|
||||
|
||||
@extension_bp.route("/manifest", methods=["GET"])
|
||||
async def extension_manifest():
|
||||
info = await asyncio.to_thread(_read_manifest_sync)
|
||||
if info is None:
|
||||
return jsonify({"installed": False}), 404
|
||||
return jsonify(info)
|
||||
+41
-279
@@ -1,131 +1,52 @@
|
||||
"""Gallery API: cursor scroll, timeline, jump, image detail, facets."""
|
||||
|
||||
from datetime import UTC, datetime, timedelta
|
||||
"""Gallery API: cursor scroll, timeline, jump, image detail."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import delete, select, update
|
||||
from sqlalchemy.orm import aliased
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import (
|
||||
ImageRecord,
|
||||
PresentationReview,
|
||||
Tag,
|
||||
TagSuggestionRejection,
|
||||
)
|
||||
from ..models.tag import image_tag
|
||||
from ..services.gallery_service import GalleryService, image_url, thumbnail_url
|
||||
from ..services.gallery_service import GalleryService
|
||||
|
||||
gallery_bp = Blueprint("gallery", __name__, url_prefix="/api/gallery")
|
||||
|
||||
|
||||
def _image_json(i):
|
||||
"""Serialize a GalleryImage for the scroll/similar list responses."""
|
||||
return {
|
||||
"id": i.id,
|
||||
"sha256": i.sha256,
|
||||
"mime": i.mime,
|
||||
"width": i.width,
|
||||
"height": i.height,
|
||||
"created_at": i.created_at.isoformat(),
|
||||
"posted_at": i.posted_at.isoformat() if i.posted_at else None,
|
||||
"thumbnail_url": i.thumbnail_url,
|
||||
"artist": i.artist,
|
||||
}
|
||||
|
||||
|
||||
def _parse_date(raw):
|
||||
"""Parse a YYYY-MM-DD query value to a UTC midnight datetime, or None.
|
||||
Raises ValueError (→ 400) on a malformed value."""
|
||||
if not raw:
|
||||
return None
|
||||
return datetime.strptime(raw, "%Y-%m-%d").replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def _parse_filters():
|
||||
"""Parse the composable gallery filters from query args, returning
|
||||
``(filters_dict, sort)``. Raises ValueError (→ 400) on malformed ids/dates.
|
||||
|
||||
The structured tag filter (#6) is AND-of-OR plus exclusions:
|
||||
- `tag_id` accepts a single id or a comma-separated list — all ANDed
|
||||
(the include common case; back-compat).
|
||||
- `tag_or` is REPEATABLE; each instance is a comma-separated OR-group, and
|
||||
the image must match at least one tag from EACH group (groups ANDed).
|
||||
- `tag_not` is a comma-separated exclude list (image must carry none).
|
||||
|
||||
`media` is image|video; `sort` is newest|oldest|posted_new|posted_old
|
||||
(default posted_new); `platform` selects one
|
||||
platform (or the UNSOURCED_PLATFORM sentinel); `untagged`/`no_artist` are
|
||||
boolean flags; `date_from`/`date_to` are inclusive calendar-day bounds
|
||||
(date_to is widened by a day so the whole day is covered by the service's
|
||||
half-open `< date_to`)."""
|
||||
tag_raw = request.args.get("tag_id")
|
||||
tag_ids = (
|
||||
[int(x) for x in tag_raw.split(",") if x.strip()] if tag_raw else None
|
||||
) or None
|
||||
tag_or_groups = [
|
||||
grp for raw in request.args.getlist("tag_or")
|
||||
if (grp := [int(x) for x in raw.split(",") if x.strip()])
|
||||
] or None
|
||||
not_raw = request.args.get("tag_not")
|
||||
tag_exclude = (
|
||||
[int(x) for x in not_raw.split(",") if x.strip()] if not_raw else None
|
||||
) or None
|
||||
post_id_raw = request.args.get("post_id")
|
||||
post_id = int(post_id_raw) if post_id_raw else None
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = int(artist_id_raw) if artist_id_raw else None
|
||||
media = request.args.get("media")
|
||||
media_type = media if media in ("image", "video") else None
|
||||
# newest/oldest key off effective_date (primary post / download); posted_new/
|
||||
# posted_old off earliest_post_date (original publish across all posts). The
|
||||
# default is posted_new so the grid leads with original publish date, not the
|
||||
# download/repost the primary post points at (operator-flagged 2026-07-01).
|
||||
sort = request.args.get("sort")
|
||||
sort = sort if sort in ("newest", "oldest", "posted_new", "posted_old") else "posted_new"
|
||||
platform = request.args.get("platform") or None
|
||||
untagged = request.args.get("untagged") in ("1", "true", "yes")
|
||||
no_artist = request.args.get("no_artist") in ("1", "true", "yes")
|
||||
# Show the presentation chrome (banner / editor screenshot) that the default
|
||||
# gallery hides — the Hidden view sets this (milestone 141).
|
||||
include_hidden = request.args.get("include_hidden") in ("1", "true", "yes")
|
||||
date_from = _parse_date(request.args.get("date_from"))
|
||||
date_to = _parse_date(request.args.get("date_to"))
|
||||
if date_to is not None:
|
||||
date_to += timedelta(days=1) # inclusive of the date_to calendar day
|
||||
filters = {
|
||||
"tag_ids": tag_ids, "post_id": post_id, "artist_id": artist_id,
|
||||
"media_type": media_type,
|
||||
"tag_or_groups": tag_or_groups, "tag_exclude": tag_exclude,
|
||||
"platform": platform,
|
||||
"untagged": untagged, "no_artist": no_artist,
|
||||
"date_from": date_from, "date_to": date_to,
|
||||
"include_hidden": include_hidden,
|
||||
}
|
||||
return filters, sort
|
||||
|
||||
|
||||
@gallery_bp.route("/scroll", methods=["GET"])
|
||||
async def scroll():
|
||||
cursor = request.args.get("cursor") or None
|
||||
try:
|
||||
limit = int(request.args.get("limit", "50"))
|
||||
filters, sort = _parse_filters()
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid filter or limit parameter"}), 400
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
tag_id_raw = request.args.get("tag_id")
|
||||
tag_id = int(tag_id_raw) if tag_id_raw else None
|
||||
post_id_raw = request.args.get("post_id")
|
||||
post_id = int(post_id_raw) if post_id_raw else None
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = int(artist_id_raw) if artist_id_raw else None
|
||||
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
page = await svc.scroll(
|
||||
cursor=cursor, limit=limit, sort=sort, **filters,
|
||||
cursor=cursor, limit=limit, tag_id=tag_id,
|
||||
post_id=post_id, artist_id=artist_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
|
||||
return jsonify(
|
||||
{
|
||||
"images": [_image_json(i) for i in page.images],
|
||||
"images": [
|
||||
{
|
||||
"id": i.id,
|
||||
"sha256": i.sha256,
|
||||
"mime": i.mime,
|
||||
"width": i.width,
|
||||
"height": i.height,
|
||||
"created_at": i.created_at.isoformat(),
|
||||
"thumbnail_url": i.thumbnail_url,
|
||||
"artist": i.artist,
|
||||
}
|
||||
for i in page.images
|
||||
],
|
||||
"next_cursor": page.next_cursor,
|
||||
"date_groups": [
|
||||
{"year": y, "month": m, "image_ids": ids} for y, m, ids in page.date_groups
|
||||
@@ -134,66 +55,20 @@ async def scroll():
|
||||
)
|
||||
|
||||
|
||||
@gallery_bp.route("/similar", methods=["GET"])
|
||||
async def similar():
|
||||
"""Visual "more like this": images ranked by cosine distance to the
|
||||
`similar_to` image's embedding. Composes with the scope filters (AND) but
|
||||
ignores post_id and sort. Bounded top-N, no cursor."""
|
||||
try:
|
||||
similar_to = int(request.args["similar_to"])
|
||||
limit = int(request.args.get("limit", "100"))
|
||||
filters, _sort = _parse_filters()
|
||||
except (KeyError, ValueError):
|
||||
return jsonify({"error": "similar_to query param required"}), 400
|
||||
# Explore passes exclude_wip=1 to also drop work-in-progress from the
|
||||
# rabbit-hole; the gallery's own "similar" button omits it (keeps wip, #1274).
|
||||
exclude_wip = request.args.get("exclude_wip") in ("1", "true", "True")
|
||||
# Explore reach (#1476): 0 = nearest (gallery default), →1 reaches into farther
|
||||
# distance bands so the walk can escape a dense cluster. exclude_ids = the
|
||||
# breadcrumb, so already-walked images aren't re-served as neighbours.
|
||||
try:
|
||||
reach = max(0.0, min(1.0, float(request.args.get("reach", "0"))))
|
||||
except ValueError:
|
||||
reach = 0.0
|
||||
exclude_ids = [
|
||||
int(x) for x in request.args.get("exclude_ids", "").split(",")
|
||||
if x.strip().isdigit()
|
||||
] or None
|
||||
# post_id is the exclusive post-detail view — not a similarity scope.
|
||||
# include_hidden is a gallery-browse flag; similar() has its OWN presentation
|
||||
# exclusion (a similarity-quality concern, #1274), so drop it here (#141).
|
||||
scope = {
|
||||
k: v for k, v in filters.items() if k not in ("post_id", "include_hidden")
|
||||
}
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
images = await svc.similar(
|
||||
image_id=similar_to, limit=limit, exclude_wip=exclude_wip,
|
||||
reach=reach, exclude_ids=exclude_ids, **scope)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
if images is None:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
return jsonify(
|
||||
{
|
||||
"images": [_image_json(i) for i in images],
|
||||
"next_cursor": None,
|
||||
"date_groups": [],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@gallery_bp.route("/timeline", methods=["GET"])
|
||||
async def timeline():
|
||||
try:
|
||||
filters, _sort = _parse_filters()
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid filter parameter"}), 400
|
||||
tag_id_raw = request.args.get("tag_id")
|
||||
tag_id = int(tag_id_raw) if tag_id_raw else None
|
||||
post_id_raw = request.args.get("post_id")
|
||||
post_id = int(post_id_raw) if post_id_raw else None
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = int(artist_id_raw) if artist_id_raw else None
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
buckets = await svc.timeline(**filters)
|
||||
buckets = await svc.timeline(
|
||||
tag_id=tag_id, post_id=post_id, artist_id=artist_id
|
||||
)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify(
|
||||
@@ -201,144 +76,31 @@ async def timeline():
|
||||
)
|
||||
|
||||
|
||||
@gallery_bp.route("/facets", methods=["GET"])
|
||||
async def facets():
|
||||
try:
|
||||
filters, _sort = _parse_filters()
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid filter parameter"}), 400
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
f = await svc.facets(**filters)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify(
|
||||
{
|
||||
"total": f.total,
|
||||
"platforms": f.platforms,
|
||||
"untagged": f.untagged,
|
||||
"no_artist": f.no_artist,
|
||||
"date_min": f.date_min.isoformat() if f.date_min else None,
|
||||
"date_max": f.date_max.isoformat() if f.date_max else None,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@gallery_bp.route("/jump", methods=["GET"])
|
||||
async def jump():
|
||||
try:
|
||||
year = int(request.args["year"])
|
||||
month = int(request.args["month"])
|
||||
filters, sort = _parse_filters()
|
||||
except (KeyError, ValueError):
|
||||
return jsonify({"error": "year and month query params required"}), 400
|
||||
tag_id_raw = request.args.get("tag_id")
|
||||
tag_id = int(tag_id_raw) if tag_id_raw else None
|
||||
post_id_raw = request.args.get("post_id")
|
||||
post_id = int(post_id_raw) if post_id_raw else None
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
artist_id = int(artist_id_raw) if artist_id_raw else None
|
||||
async with get_session() as session:
|
||||
svc = GalleryService(session)
|
||||
try:
|
||||
cursor = await svc.jump_cursor(
|
||||
year=year, month=month, sort=sort, **filters,
|
||||
year=year, month=month, tag_id=tag_id,
|
||||
post_id=post_id, artist_id=artist_id,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
return jsonify({"cursor": cursor})
|
||||
|
||||
|
||||
# -- Hidden-view review (#141): auto-hidden chrome flagged "also looks like
|
||||
# content", surfaced in the gallery's Show-hidden review strip. -----------
|
||||
@gallery_bp.route("/hidden-review", methods=["GET"])
|
||||
async def hidden_review():
|
||||
"""Unresolved system-tag auto-apply review flags (chrome + process, #1464),
|
||||
most-concerning first (highest content score) — for the review strip. `mode`
|
||||
tells the client whether the flagged tag hid the image ('chrome') or left it
|
||||
visible ('process'), which decides the resolve labels (un-hide vs remove-tag)."""
|
||||
ptag = aliased(Tag)
|
||||
ctag = aliased(Tag)
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(
|
||||
PresentationReview.image_record_id,
|
||||
PresentationReview.tag_id,
|
||||
PresentationReview.conflict_tag_id,
|
||||
PresentationReview.conflict_score,
|
||||
PresentationReview.mode,
|
||||
ImageRecord.path, ImageRecord.thumbnail_path,
|
||||
ImageRecord.sha256, ImageRecord.mime,
|
||||
ptag.name.label("tag_name"),
|
||||
ctag.name.label("conflict_name"),
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.id == PresentationReview.image_record_id)
|
||||
.join(ptag, ptag.id == PresentationReview.tag_id)
|
||||
.outerjoin(ctag, ctag.id == PresentationReview.conflict_tag_id)
|
||||
.where(PresentationReview.resolved_at.is_(None))
|
||||
.order_by(PresentationReview.conflict_score.desc())
|
||||
)).all()
|
||||
return jsonify({"items": [
|
||||
{
|
||||
"image_id": r.image_record_id,
|
||||
"tag_id": r.tag_id,
|
||||
"tag_name": r.tag_name,
|
||||
"conflict_tag_id": r.conflict_tag_id,
|
||||
"conflict_name": r.conflict_name,
|
||||
"conflict_score": r.conflict_score,
|
||||
"mode": r.mode,
|
||||
"thumbnail_url": thumbnail_url(r.thumbnail_path, r.sha256, r.mime),
|
||||
"image_url": image_url(r.path),
|
||||
}
|
||||
for r in rows
|
||||
]})
|
||||
|
||||
|
||||
@gallery_bp.route(
|
||||
"/hidden-review/<int:image_id>/<int:tag_id>/keep", methods=["POST"]
|
||||
)
|
||||
async def hidden_review_keep(image_id, tag_id):
|
||||
"""Keep the auto-hide: resolve the flag; the tag stays applied (#141)."""
|
||||
async with get_session() as session:
|
||||
await session.execute(
|
||||
update(PresentationReview)
|
||||
.where(
|
||||
PresentationReview.image_record_id == image_id,
|
||||
PresentationReview.tag_id == tag_id,
|
||||
)
|
||||
.values(resolved_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@gallery_bp.route(
|
||||
"/hidden-review/<int:image_id>/<int:tag_id>/unhide", methods=["POST"]
|
||||
)
|
||||
async def hidden_review_unhide(image_id, tag_id):
|
||||
"""Un-hide: remove the presentation tag (image returns to the gallery), record
|
||||
a rejection so the head LEARNS it misfired, and resolve the flag (#141)."""
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
async with get_session() as session:
|
||||
await session.execute(
|
||||
delete(image_tag).where(
|
||||
image_tag.c.image_record_id == image_id,
|
||||
image_tag.c.tag_id == tag_id,
|
||||
)
|
||||
)
|
||||
await session.execute(
|
||||
pg_insert(TagSuggestionRejection)
|
||||
.values(image_record_id=image_id, tag_id=tag_id)
|
||||
.on_conflict_do_nothing()
|
||||
)
|
||||
await session.execute(
|
||||
update(PresentationReview)
|
||||
.where(
|
||||
PresentationReview.image_record_id == image_id,
|
||||
PresentationReview.tag_id == tag_id,
|
||||
)
|
||||
.values(resolved_at=datetime.now(UTC))
|
||||
)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@gallery_bp.route("/image/<int:image_id>", methods=["GET"])
|
||||
async def image_detail(image_id: int):
|
||||
async with get_session() as session:
|
||||
|
||||
@@ -1,469 +0,0 @@
|
||||
"""GPU-job API (#114): the HTTP surface the desktop agent pulls work from.
|
||||
|
||||
The agent stays HTTP-only — it leases jobs, fetches image pixels via the normal
|
||||
FC image URLs, and submits embeddings/regions back, all over this API. Redis and
|
||||
Postgres are never exposed. The agent endpoints are gated by a bearer token
|
||||
(Authorization: Bearer <token>) stored in AppSetting; the admin endpoints
|
||||
(token / backfill / status) ride the browser session like the rest of FC's
|
||||
homelab admin.
|
||||
"""
|
||||
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, or_, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import AppSetting, GpuJob, ImageRecord, MLSettings
|
||||
from ..services.gallery_service import image_url
|
||||
from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
|
||||
from ..services.ml.gpu_triage import classify_reason, recover_defective_image
|
||||
from ..services.ml.regions import RegionService
|
||||
from ..services.service_roster import touch_service
|
||||
|
||||
gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
|
||||
|
||||
# Same container mount the maintenance tasks use (tasks/admin.py) — recovery
|
||||
# deletes the defective original + thumbnail under it.
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
_TOKEN_KEY = "gpu_agent_token"
|
||||
|
||||
|
||||
def _bearer() -> str | None:
|
||||
h = request.headers.get("Authorization", "")
|
||||
return h[7:].strip() if h.startswith("Bearer ") else None
|
||||
|
||||
|
||||
async def _agent_authed(session) -> bool:
|
||||
supplied = _bearer()
|
||||
if not supplied:
|
||||
return False
|
||||
stored = (
|
||||
await session.execute(
|
||||
select(AppSetting.value).where(AppSetting.key == _TOKEN_KEY)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return stored is not None and secrets.compare_digest(supplied, stored)
|
||||
|
||||
|
||||
# --- Admin (browser): token + backfill + status -------------------------
|
||||
|
||||
@gpu_bp.route("/token", methods=["GET"])
|
||||
async def get_token():
|
||||
async with get_session() as session:
|
||||
tok = (
|
||||
await session.execute(
|
||||
select(AppSetting.value).where(AppSetting.key == _TOKEN_KEY)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
return jsonify({"token": tok, "configured": tok is not None})
|
||||
|
||||
|
||||
@gpu_bp.route("/token/rotate", methods=["POST"])
|
||||
async def rotate_token():
|
||||
token = secrets.token_urlsafe(32)
|
||||
async with get_session() as session:
|
||||
await session.execute(
|
||||
pg_insert(AppSetting)
|
||||
.values(key=_TOKEN_KEY, value=token)
|
||||
.on_conflict_do_update(index_elements=["key"], set_={"value": token})
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"token": token})
|
||||
|
||||
|
||||
@gpu_bp.route("/status", methods=["GET"])
|
||||
async def status():
|
||||
async with get_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(GpuJob.status, func.count()).group_by(GpuJob.status)
|
||||
)
|
||||
).all()
|
||||
counts = dict(rows)
|
||||
return jsonify({
|
||||
"pending": counts.get("pending", 0),
|
||||
"leased": counts.get("leased", 0),
|
||||
"done": counts.get("done", 0),
|
||||
"error": counts.get("error", 0),
|
||||
})
|
||||
|
||||
|
||||
@gpu_bp.route("/backfill", methods=["POST"])
|
||||
async def backfill():
|
||||
"""Enqueue a job for every image that doesn't already have one for `task`."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
task = str(body.get("task") or "ccip")
|
||||
from ..tasks.gpu_queue import enqueue_gpu_backfill
|
||||
|
||||
r = enqueue_gpu_backfill.delay(task)
|
||||
return jsonify({"celery_task_id": r.id, "task": task}), 202
|
||||
|
||||
|
||||
@gpu_bp.route("/reprocess", methods=["POST"])
|
||||
async def reprocess():
|
||||
"""Reset every done/error job of `task` back to pending so the agent re-runs
|
||||
the WHOLE library under the current pipeline (e.g. after adding crop
|
||||
detectors). Heavy — the back-catalogue is otherwise skipped by the backfills."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
task = str(body.get("task") or "ccip")
|
||||
from ..tasks.gpu_queue import reprocess_gpu_jobs
|
||||
|
||||
r = reprocess_gpu_jobs.delay(task)
|
||||
return jsonify({"celery_task_id": r.id, "task": task}), 202
|
||||
|
||||
|
||||
@gpu_bp.route("/retry_errors", methods=["POST"])
|
||||
async def retry_errors():
|
||||
"""Requeue every ERRORED job (all task types) back to pending — the scoped
|
||||
recovery after an agent-side fix (e.g. the short-video sampler), where
|
||||
/reprocess would needlessly re-run the whole done library too. Attempts and
|
||||
the stored error reset so each job gets its full retry budget under the
|
||||
fixed pipeline. Stale tombstones are pruned FIRST (loop-era duplicates and
|
||||
rows a later success made moot — the same statements the backfills run), so
|
||||
one failing file requeues as ONE job, never a fan-out of duplicates. Small
|
||||
row count (errors only) → inline statements; the response carries the
|
||||
counts for the UI toast. Triage-confirmed defects are NOT requeued (see
|
||||
the WHERE below) — they stay on the recovery surface."""
|
||||
async with get_session() as session:
|
||||
pruned = 0
|
||||
for stmt in error_dedupe_statements():
|
||||
pruned += (await session.execute(stmt)).rowcount or 0
|
||||
res = await session.execute(
|
||||
update(GpuJob)
|
||||
.where(
|
||||
GpuJob.status == "error",
|
||||
# Triage-confirmed DEFECTS stay errored: the integrity probe
|
||||
# already proved the FILE is bad, so re-running the job just
|
||||
# burns agent time re-minting the same tombstone — those go
|
||||
# through /errors/<id>/recover instead.
|
||||
or_(GpuJob.triage_status.is_(None),
|
||||
GpuJob.triage_status != "defect"),
|
||||
)
|
||||
.values(
|
||||
status="pending", attempts=0, error=None, lease_token=None,
|
||||
leased_at=None, lease_expires_at=None, triage_status=None,
|
||||
updated_at=func.now(),
|
||||
)
|
||||
)
|
||||
kept = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
await session.commit()
|
||||
return jsonify({
|
||||
"requeued": res.rowcount or 0, "pruned": pruned, "defects_kept": kept,
|
||||
})
|
||||
|
||||
|
||||
# --- Failure triage + recovery (#125) ------------------------------------
|
||||
|
||||
@gpu_bp.route("/errors", methods=["GET"])
|
||||
async def errors():
|
||||
"""The triage view of the error tombstones: every errored job joined with
|
||||
its image's integrity verdict, bucketed by reason for the overview. The
|
||||
probe sweep (triage_gpu_errors, 15-min beat) fills triage_status; 'defect'
|
||||
rows are the recovery surface's list."""
|
||||
async with get_session() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
GpuJob.id, GpuJob.image_record_id, GpuJob.task,
|
||||
GpuJob.error, GpuJob.triage_status, GpuJob.updated_at,
|
||||
ImageRecord.integrity_status, ImageRecord.mime,
|
||||
ImageRecord.path, ImageRecord.thumbnail_path,
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.id == GpuJob.image_record_id)
|
||||
.where(GpuJob.status == "error")
|
||||
.order_by(GpuJob.updated_at.desc())
|
||||
.limit(500)
|
||||
)
|
||||
).all()
|
||||
total = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(GpuJob)
|
||||
.where(GpuJob.status == "error")
|
||||
)
|
||||
).scalar_one()
|
||||
by_class: dict[str, int] = {}
|
||||
triage = {"defect": 0, "file_ok": 0, "unclassified": 0}
|
||||
items = []
|
||||
for r in rows:
|
||||
cls = classify_reason(r.error)
|
||||
by_class[cls] = by_class.get(cls, 0) + 1
|
||||
bucket = r.triage_status or "unclassified"
|
||||
triage[bucket] = triage.get(bucket, 0) + 1
|
||||
items.append({
|
||||
"job_id": r.id,
|
||||
"image_id": r.image_record_id,
|
||||
"task": r.task,
|
||||
"error": r.error,
|
||||
"reason_class": cls,
|
||||
"triage_status": r.triage_status,
|
||||
"integrity_status": r.integrity_status,
|
||||
"mime": r.mime,
|
||||
"image_url": image_url(r.path),
|
||||
"thumbnail_url": (
|
||||
image_url(r.thumbnail_path) if r.thumbnail_path else None
|
||||
),
|
||||
"updated_at": r.updated_at.isoformat() if r.updated_at else None,
|
||||
})
|
||||
return jsonify({
|
||||
"total": total, "by_class": by_class, "triage": triage, "items": items,
|
||||
})
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/triage", methods=["POST"])
|
||||
async def errors_triage():
|
||||
"""Run the probe sweep NOW (the card's button) instead of waiting out the
|
||||
15-minute beat cadence."""
|
||||
from ..tasks.maintenance import triage_gpu_errors
|
||||
|
||||
r = triage_gpu_errors.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@gpu_bp.route("/errors/<int:image_id>/recover", methods=["POST"])
|
||||
async def errors_recover(image_id: int):
|
||||
"""Recover a defect-triaged original: delete the bad copy + record and
|
||||
re-poll its subscription Source (a fresh fetch re-imports the file, which
|
||||
re-enters the GPU pipeline). Returns status 'no_source' when nothing
|
||||
pollable resolves — the file needs manual replacement there."""
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: recover_defective_image(
|
||||
s, image_id, images_root=_IMAGES_ROOT,
|
||||
)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
||||
|
||||
|
||||
def _accel_detail(body: dict) -> dict:
|
||||
"""The agent's own report of which runtime got the GPU, kept on its roster
|
||||
row so the System view can call a CPU-bound agent degraded (#4410).
|
||||
|
||||
Only a dict of {runtime: {device, error?}} is kept, and each value is
|
||||
reduced to those two short strings: this is written on every lease, by a
|
||||
client the server does not control. An agent that sends nothing (an
|
||||
older build) simply has no `accel`, which reads as not-yet-reported.
|
||||
"""
|
||||
raw = body.get("accel")
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
accel = {}
|
||||
for name, entry in list(raw.items())[:4]:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
clean = {"device": str(entry.get("device") or "")[:16]}
|
||||
if entry.get("error"):
|
||||
clean["error"] = str(entry["error"])[:200]
|
||||
accel[str(name)[:16]] = clean
|
||||
return {"accel": accel} if accel else {}
|
||||
|
||||
@gpu_bp.route("/jobs/lease", methods=["POST"])
|
||||
async def lease():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
agent_id = str(body.get("agent_id") or "agent")
|
||||
try:
|
||||
batch = min(max(int(body.get("batch_size", 8)), 1), 64)
|
||||
except (TypeError, ValueError):
|
||||
batch = 8
|
||||
async with get_session() as session:
|
||||
if not await _agent_authed(session):
|
||||
return jsonify({"error": "unauthorized"}), 401
|
||||
jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
|
||||
# The agent cannot be polled — it is HTTP-only and pulls from here, so
|
||||
# web never dials it. A lease IS the check-in, and until milestone 365
|
||||
# it was thrown away: an agent sitting idle with nothing to lease left
|
||||
# no trace at all and was indistinguishable from one switched off a
|
||||
# week ago. Recorded on the call that was already happening.
|
||||
await touch_service(
|
||||
session,
|
||||
key=f"agent:{agent_id}",
|
||||
kind="agent",
|
||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||
details={
|
||||
"agent_id": agent_id, "last_call": "lease", "leased": len(jobs),
|
||||
**_accel_detail(body),
|
||||
},
|
||||
)
|
||||
ml = await MLSettings.load(session)
|
||||
# image rows for url/mime in one shot
|
||||
ids = [j.image_record_id for j in jobs]
|
||||
imgs = {
|
||||
i.id: i for i in (
|
||||
await session.execute(
|
||||
select(ImageRecord).where(ImageRecord.id.in_(ids))
|
||||
)
|
||||
).scalars()
|
||||
} if ids else {}
|
||||
await session.commit()
|
||||
# Crop-proposer config, announced FROM THE SETTING like embed_model_name
|
||||
# (#134): the agent builds its detectors from this, rebuilding live when
|
||||
# it changes — so tuning is a DB/UI edit, never an agent restart. Same
|
||||
# block for every job in the batch (it's global), built once. An enabled
|
||||
# toggle off is carried through so the agent skips that proposer.
|
||||
detectors = {
|
||||
"person": {
|
||||
"enabled": ml.detector_person_enabled,
|
||||
"weights": ml.detector_person_weights,
|
||||
"conf": ml.detector_person_conf,
|
||||
},
|
||||
"anatomy": {
|
||||
"enabled": ml.detector_anatomy_enabled,
|
||||
"weights": ml.detector_anatomy_weights,
|
||||
"conf": ml.detector_anatomy_conf,
|
||||
},
|
||||
"panel": {
|
||||
"enabled": ml.detector_panel_enabled,
|
||||
"weights": ml.detector_panel_weights,
|
||||
"conf": ml.detector_panel_conf,
|
||||
},
|
||||
"max_figures": ml.detector_max_figures,
|
||||
"max_components": ml.detector_max_components,
|
||||
"max_panels": ml.detector_max_panels,
|
||||
"max_regions": ml.detector_max_regions,
|
||||
"dedupe_iou": ml.detector_dedupe_iou,
|
||||
}
|
||||
out = []
|
||||
for j in jobs:
|
||||
img = imgs.get(j.image_record_id)
|
||||
if img is None:
|
||||
continue
|
||||
out.append({
|
||||
"job_id": j.id,
|
||||
"image_id": j.image_record_id,
|
||||
"task": j.task,
|
||||
"mime": img.mime,
|
||||
"image_url": image_url(img.path),
|
||||
# For video/animated: the agent samples at this cadence.
|
||||
"frame_interval_seconds": ml.video_frame_interval_seconds,
|
||||
"max_frames": ml.video_max_frames,
|
||||
# The embedding model the agent must use for concept crops + the
|
||||
# whole-image 'embed' task, so its vectors land in the SAME space
|
||||
# the heads trained in. Server-announced FROM THE SETTING → the
|
||||
# agent stays model-agnostic; an operator swap is a setting + a
|
||||
# re-embed, never an agent change.
|
||||
"embed_model_name": ml.embedder_model_name,
|
||||
"embed_version": ml.embedder_model_version,
|
||||
"detectors": detectors,
|
||||
})
|
||||
return jsonify({"jobs": out})
|
||||
|
||||
|
||||
@gpu_bp.route("/jobs/heartbeat", methods=["POST"])
|
||||
async def heartbeat():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
agent_id = str(body.get("agent_id") or "agent")
|
||||
job_ids = [int(x) for x in (body.get("job_ids") or [])]
|
||||
async with get_session() as session:
|
||||
if not await _agent_authed(session):
|
||||
return jsonify({"error": "unauthorized"}), 401
|
||||
n = await GpuJobService(session).heartbeat(agent_id, job_ids)
|
||||
await touch_service(
|
||||
session,
|
||||
key=f"agent:{agent_id}",
|
||||
kind="agent",
|
||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||
details={
|
||||
"agent_id": agent_id, "last_call": "heartbeat", "extended": n,
|
||||
**_accel_detail(body),
|
||||
},
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"extended": n})
|
||||
|
||||
|
||||
@gpu_bp.route("/jobs/submit", methods=["POST"])
|
||||
async def submit():
|
||||
"""Store a job's regions + close it. regions: [{kind, bbox:[x,y,w,h],
|
||||
frame_time?, score?, *_version?, ccip_embedding?, siglip_embedding?}].
|
||||
replace_kinds defaults to the kinds present in the submitted regions."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
agent_id = str(body.get("agent_id") or "agent")
|
||||
job_id = body.get("job_id")
|
||||
regions = body.get("regions") or []
|
||||
if job_id is None:
|
||||
return jsonify({"error": "job_id required"}), 400
|
||||
kinds = body.get("replace_kinds") or sorted({r["kind"] for r in regions})
|
||||
async with get_session() as session:
|
||||
if not await _agent_authed(session):
|
||||
return jsonify({"error": "unauthorized"}), 401
|
||||
job = await session.get(GpuJob, int(job_id))
|
||||
if job is None or job.status != "leased" or job.lease_token != agent_id:
|
||||
return jsonify({"error": "lease_invalid"}), 409
|
||||
if kinds:
|
||||
await RegionService(session).replace_regions(
|
||||
job.image_record_id, kinds, regions
|
||||
)
|
||||
await GpuJobService(session).complete(agent_id, int(job_id))
|
||||
await session.commit()
|
||||
return jsonify({"ok": True, "stored": len(regions)})
|
||||
|
||||
|
||||
@gpu_bp.route("/jobs/submit_embedding", methods=["POST"])
|
||||
async def submit_embedding():
|
||||
"""Store a whole-image SigLIP embedding (the 'embed' task) on image_record +
|
||||
close the job. Body: {agent_id, job_id, embedding:[...], embedding_version}.
|
||||
This is how the GPU agent re-embeds the library under a new model (#1190) —
|
||||
much faster than the CPU ml-worker at higher resolutions."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
agent_id = str(body.get("agent_id") or "agent")
|
||||
job_id = body.get("job_id")
|
||||
embedding = body.get("embedding")
|
||||
version = body.get("embedding_version")
|
||||
if job_id is None or not embedding or not version:
|
||||
return jsonify({"error": "job_id, embedding, embedding_version required"}), 400
|
||||
async with get_session() as session:
|
||||
if not await _agent_authed(session):
|
||||
return jsonify({"error": "unauthorized"}), 401
|
||||
job = await session.get(GpuJob, int(job_id))
|
||||
if job is None or job.status != "leased" or job.lease_token != agent_id:
|
||||
return jsonify({"error": "lease_invalid"}), 409
|
||||
img = await session.get(ImageRecord, job.image_record_id)
|
||||
if img is not None:
|
||||
img.siglip_embedding = embedding
|
||||
img.siglip_model_version = version
|
||||
await GpuJobService(session).complete(agent_id, int(job_id))
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@gpu_bp.route("/jobs/fail", methods=["POST"])
|
||||
async def fail():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
agent_id = str(body.get("agent_id") or "agent")
|
||||
job_id = body.get("job_id")
|
||||
if job_id is None:
|
||||
return jsonify({"error": "job_id required"}), 400
|
||||
async with get_session() as session:
|
||||
if not await _agent_authed(session):
|
||||
return jsonify({"error": "unauthorized"}), 401
|
||||
ok = await GpuJobService(session).fail(
|
||||
agent_id, int(job_id), str(body.get("error") or "")
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"ok": ok})
|
||||
|
||||
|
||||
@gpu_bp.route("/jobs/release", methods=["POST"])
|
||||
async def release():
|
||||
"""Graceful stop: the agent hands its still-leased jobs back to pending so
|
||||
they're picked up immediately instead of waiting out the lease."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
agent_id = str(body.get("agent_id") or "agent")
|
||||
job_ids = [int(x) for x in (body.get("job_ids") or [])]
|
||||
async with get_session() as session:
|
||||
if not await _agent_authed(session):
|
||||
return jsonify({"error": "unauthorized"}), 401
|
||||
n = await GpuJobService(session).release(agent_id, job_ids)
|
||||
await session.commit()
|
||||
return jsonify({"released": n})
|
||||
@@ -1,285 +0,0 @@
|
||||
"""Heads API (#114): train + inspect the per-concept heads that power
|
||||
suggestions (replacing Camie + centroid).
|
||||
|
||||
POST /api/heads/train — (re)train all eligible heads (one run at a time).
|
||||
GET /api/heads — status: head count, last-trained, running run, the
|
||||
per-concept head table (strength + auto-apply ready),
|
||||
and recent training runs. The card rehydrates from
|
||||
here so status survives navigation.
|
||||
"""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import desc, func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import (
|
||||
HeadAutoApplyRun,
|
||||
HeadMetric,
|
||||
HeadMetricsSnapshot,
|
||||
HeadTrainingRun,
|
||||
Tag,
|
||||
TagHead,
|
||||
)
|
||||
from ..models.tag import image_tag
|
||||
from ..services.ml.heads import (
|
||||
HeadAutoApplyAlreadyRunning,
|
||||
HeadAutoApplyDisabled,
|
||||
HeadTrainingAlreadyRunning,
|
||||
start_head_auto_apply_run,
|
||||
start_head_training_run,
|
||||
)
|
||||
|
||||
heads_bp = Blueprint("heads", __name__, url_prefix="/api/heads")
|
||||
|
||||
|
||||
def _serialize_run(run: HeadTrainingRun) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
"params": run.params,
|
||||
"status": run.status,
|
||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||
"n_trained": run.n_trained,
|
||||
"n_skipped": run.n_skipped,
|
||||
"error": run.error,
|
||||
}
|
||||
|
||||
|
||||
@heads_bp.route("/train", methods=["POST"])
|
||||
async def train():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
params = body.get("params") or body or {}
|
||||
async with get_session() as session:
|
||||
try:
|
||||
run_id = await session.run_sync(
|
||||
lambda s: start_head_training_run(s, params)
|
||||
)
|
||||
except HeadTrainingAlreadyRunning as running:
|
||||
return jsonify({
|
||||
"error": "training_already_running",
|
||||
"running_id": int(running.args[0]),
|
||||
}), 409
|
||||
await session.commit()
|
||||
return jsonify({"run_id": run_id, "status": "running"}), 202
|
||||
|
||||
|
||||
@heads_bp.route("", methods=["GET"])
|
||||
async def status():
|
||||
async with get_session() as session:
|
||||
count, last_trained = (
|
||||
await session.execute(
|
||||
select(func.count(), func.max(TagHead.trained_at))
|
||||
)
|
||||
).one()
|
||||
graduated = (
|
||||
await session.execute(
|
||||
select(func.count()).where(
|
||||
TagHead.auto_apply_threshold.is_not(None)
|
||||
)
|
||||
)
|
||||
).scalar_one()
|
||||
running = (
|
||||
await session.execute(
|
||||
select(HeadTrainingRun.id)
|
||||
.where(HeadTrainingRun.status == "running")
|
||||
.order_by(HeadTrainingRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
runs = (
|
||||
await session.execute(
|
||||
select(HeadTrainingRun)
|
||||
.order_by(HeadTrainingRun.id.desc())
|
||||
.limit(10)
|
||||
)
|
||||
).scalars().all()
|
||||
# The per-concept table: strongest first, capped for the admin card.
|
||||
head_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
TagHead.tag_id, Tag.name, Tag.kind,
|
||||
TagHead.n_pos, TagHead.n_neg, TagHead.ap,
|
||||
TagHead.precision_cv, TagHead.recall,
|
||||
TagHead.auto_apply_threshold, TagHead.trained_at,
|
||||
)
|
||||
.join(Tag, Tag.id == TagHead.tag_id)
|
||||
.order_by(desc(TagHead.ap))
|
||||
.limit(500)
|
||||
)
|
||||
).all()
|
||||
heads = [
|
||||
{
|
||||
"tag_id": r.tag_id,
|
||||
"name": r.name,
|
||||
"category": r.kind.value if hasattr(r.kind, "value") else str(r.kind),
|
||||
"n_pos": r.n_pos,
|
||||
"n_neg": r.n_neg,
|
||||
"ap": r.ap,
|
||||
"precision": r.precision_cv,
|
||||
"recall": r.recall,
|
||||
"auto_apply": r.auto_apply_threshold is not None,
|
||||
"trained_at": r.trained_at.isoformat() if r.trained_at else None,
|
||||
}
|
||||
for r in head_rows
|
||||
]
|
||||
return jsonify({
|
||||
"head_count": count,
|
||||
"graduated_count": graduated,
|
||||
"last_trained_at": last_trained.isoformat() if last_trained else None,
|
||||
"running_id": running,
|
||||
"runs": [_serialize_run(r) for r in runs],
|
||||
"heads": heads,
|
||||
})
|
||||
|
||||
|
||||
def _serialize_apply_run(run: HeadAutoApplyRun) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
"dry_run": run.dry_run,
|
||||
"status": run.status,
|
||||
"started_at": run.started_at.isoformat() if run.started_at else None,
|
||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||
"n_applied": run.n_applied,
|
||||
"report": run.report,
|
||||
"error": run.error,
|
||||
}
|
||||
|
||||
|
||||
@heads_bp.route("/auto-apply", methods=["POST"])
|
||||
async def auto_apply():
|
||||
"""Trigger an earned-auto-apply sweep. {dry_run:true} previews (writes
|
||||
nothing); a real sweep needs head_auto_apply_enabled on."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
params = {"dry_run": bool(body.get("dry_run", False))}
|
||||
async with get_session() as session:
|
||||
try:
|
||||
run_id = await session.run_sync(
|
||||
lambda s: start_head_auto_apply_run(s, params)
|
||||
)
|
||||
except HeadAutoApplyAlreadyRunning as running:
|
||||
return jsonify({
|
||||
"error": "auto_apply_already_running",
|
||||
"running_id": int(running.args[0]),
|
||||
}), 409
|
||||
except HeadAutoApplyDisabled:
|
||||
return jsonify({"error": "auto_apply_disabled"}), 400
|
||||
await session.commit()
|
||||
return jsonify({"run_id": run_id, "status": "running"}), 202
|
||||
|
||||
|
||||
@heads_bp.route("/auto-apply", methods=["GET"])
|
||||
async def auto_apply_status():
|
||||
async with get_session() as session:
|
||||
running = (
|
||||
await session.execute(
|
||||
select(HeadAutoApplyRun.id)
|
||||
.where(HeadAutoApplyRun.status == "running")
|
||||
.order_by(HeadAutoApplyRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
runs = (
|
||||
await session.execute(
|
||||
select(HeadAutoApplyRun)
|
||||
.order_by(HeadAutoApplyRun.id.desc())
|
||||
.limit(10)
|
||||
)
|
||||
).scalars().all()
|
||||
return jsonify({
|
||||
"running_id": running,
|
||||
"runs": [_serialize_apply_run(r) for r in runs],
|
||||
})
|
||||
|
||||
|
||||
@heads_bp.route("/metrics", methods=["GET"])
|
||||
async def metrics():
|
||||
"""Auto-apply observability: per-concept current counts (volume, misfires,
|
||||
under-fires, realized misfire rate, head quality) + the daily time-series so
|
||||
the operator can tune the precision target + support floor from real data."""
|
||||
async with get_session() as session:
|
||||
head_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
TagHead.tag_id, Tag.name, TagHead.ap, TagHead.precision_cv,
|
||||
TagHead.recall, TagHead.auto_apply_threshold, TagHead.n_pos,
|
||||
).join(Tag, Tag.id == TagHead.tag_id)
|
||||
)
|
||||
).all()
|
||||
heads = {r.tag_id: r for r in head_rows}
|
||||
metric_rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
HeadMetric.tag_id, HeadMetric.n_misfires, HeadMetric.n_underfires
|
||||
)
|
||||
)
|
||||
).all()
|
||||
mets = {r.tag_id: r for r in metric_rows}
|
||||
applied = dict(
|
||||
(
|
||||
await session.execute(
|
||||
select(image_tag.c.tag_id, func.count())
|
||||
.where(image_tag.c.source == "head_auto")
|
||||
.group_by(image_tag.c.tag_id)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
names = {r.tag_id: r.name for r in head_rows}
|
||||
# Names for metric-only tags (head pruned but corrections recorded).
|
||||
missing = [t for t in mets if t not in names]
|
||||
if missing:
|
||||
for tid, nm in (
|
||||
await session.execute(
|
||||
select(Tag.id, Tag.name).where(Tag.id.in_(missing))
|
||||
)
|
||||
).all():
|
||||
names[tid] = nm
|
||||
|
||||
concepts = []
|
||||
for tid in set(heads) | set(mets):
|
||||
h = heads.get(tid)
|
||||
m = mets.get(tid)
|
||||
n_applied = applied.get(tid, 0)
|
||||
n_mis = m.n_misfires if m else 0
|
||||
denom = n_applied + n_mis
|
||||
concepts.append({
|
||||
"tag_id": tid,
|
||||
"name": names.get(tid, str(tid)),
|
||||
"n_auto_applied": n_applied,
|
||||
"n_misfires": n_mis,
|
||||
"n_underfires": m.n_underfires if m else 0,
|
||||
# Of everything this head ever auto-applied, the fraction you
|
||||
# removed — the misfire rate (null until something fired).
|
||||
"misfire_rate": round(n_mis / denom, 4) if denom else None,
|
||||
"ap": h.ap if h else None,
|
||||
"precision_cv": h.precision_cv if h else None,
|
||||
"recall": h.recall if h else None,
|
||||
"auto_apply": bool(h and h.auto_apply_threshold is not None),
|
||||
"n_pos": h.n_pos if h else None,
|
||||
})
|
||||
concepts.sort(key=lambda c: (c["n_misfires"], c["n_auto_applied"]), reverse=True)
|
||||
|
||||
snaps = (
|
||||
await session.execute(
|
||||
select(HeadMetricsSnapshot)
|
||||
.order_by(HeadMetricsSnapshot.snapshot_at.desc())
|
||||
.limit(1000)
|
||||
)
|
||||
).scalars().all()
|
||||
return jsonify({
|
||||
"concepts": concepts,
|
||||
"snapshots": [
|
||||
{
|
||||
"tag_id": s.tag_id,
|
||||
"name": s.name,
|
||||
"snapshot_at": s.snapshot_at.isoformat() if s.snapshot_at else None,
|
||||
"n_auto_applied": s.n_auto_applied,
|
||||
"n_misfires": s.n_misfires,
|
||||
"n_underfires": s.n_underfires,
|
||||
"ap": s.ap,
|
||||
"precision_cv": s.precision_cv,
|
||||
"recall": s.recall,
|
||||
"n_pos": s.n_pos,
|
||||
}
|
||||
for s in snaps
|
||||
],
|
||||
})
|
||||
@@ -1,20 +1,5 @@
|
||||
"""Health endpoint — no DB or Redis touch; liveness, plus the build's identity.
|
||||
|
||||
The identity rides here rather than on a route of its own because it answers
|
||||
at the same cost: two module constants, no I/O, nothing that can be slow or
|
||||
fail. It is also already fetched app-wide — TopNav calls `refreshHealth` on
|
||||
mount — so a separate endpoint would mean a second request for two strings.
|
||||
|
||||
Both fields are OMITTED when unset rather than sent empty. See build_info.
|
||||
"""
|
||||
|
||||
from ..build_info import FC_CHANNEL, FC_VERSION
|
||||
"""Health endpoint — no DB or Redis touch; just liveness."""
|
||||
|
||||
|
||||
async def get_health():
|
||||
body = {"status": "ok"}
|
||||
if FC_VERSION:
|
||||
body["version"] = FC_VERSION
|
||||
if FC_CHANNEL:
|
||||
body["channel"] = FC_CHANNEL
|
||||
return body, 200
|
||||
return {"status": "ok"}, 200
|
||||
|
||||
+15
-149
@@ -35,26 +35,10 @@ async def trigger_scan():
|
||||
@import_admin_bp.route("/status", methods=["GET"])
|
||||
async def status():
|
||||
async with get_session() as session:
|
||||
# Active batch = running batch that still has outstanding work.
|
||||
# Plain "most recent running" picks freshly-created scans that
|
||||
# enqueued zero new files and hides the older batch that's
|
||||
# actually being processed. Mirrors the EXISTS predicate
|
||||
# /api/system/stats already uses (api/settings.py:145-160).
|
||||
# Audit 2026-06-02 — /api/import/status and /api/system/stats
|
||||
# used to disagree on the active-batch predicate; the UI banner
|
||||
# said "Scanning…" indefinitely while the stats card said idle.
|
||||
active = (
|
||||
await session.execute(
|
||||
select(ImportBatch)
|
||||
.where(
|
||||
ImportBatch.status == "running",
|
||||
select(ImportTask.id)
|
||||
.where(
|
||||
ImportTask.batch_id == ImportBatch.id,
|
||||
ImportTask.status.in_(["pending", "queued", "processing"]),
|
||||
)
|
||||
.exists(),
|
||||
)
|
||||
.where(ImportBatch.status == "running")
|
||||
.order_by(ImportBatch.started_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
@@ -63,13 +47,10 @@ async def status():
|
||||
if active:
|
||||
payload["active_batch"] = {
|
||||
"id": active.id,
|
||||
"source_path": active.source_path,
|
||||
"scan_mode": active.scan_mode,
|
||||
"total_files": active.total_files,
|
||||
"imported": active.imported,
|
||||
"skipped": active.skipped,
|
||||
"failed": active.failed,
|
||||
"refreshed": active.refreshed,
|
||||
"started_at": active.started_at.isoformat(),
|
||||
}
|
||||
return jsonify(payload)
|
||||
@@ -119,139 +100,24 @@ async def list_tasks():
|
||||
|
||||
@import_admin_bp.route("/retry-failed", methods=["POST"])
|
||||
async def retry_failed():
|
||||
# Fold SELECT into UPDATE…WHERE…RETURNING — the prior SELECT-then-
|
||||
# UPDATE-WHERE-id-IN pattern blew past psycopg's 65535-parameter
|
||||
# ceiling once failed_ids exceeded ~65k rows.
|
||||
async with get_session() as session:
|
||||
result = await session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.status == "failed")
|
||||
.values(
|
||||
status="queued", error=None,
|
||||
started_at=None, finished_at=None,
|
||||
)
|
||||
.returning(ImportTask.id, ImportTask.task_type)
|
||||
)
|
||||
failed = result.all()
|
||||
if not failed:
|
||||
return jsonify({"retried": 0})
|
||||
await session.commit()
|
||||
|
||||
from ..tasks.import_file import enqueue_import
|
||||
for tid, task_type in failed:
|
||||
enqueue_import(tid, task_type)
|
||||
|
||||
return jsonify({"retried": len(failed)})
|
||||
|
||||
|
||||
@import_admin_bp.route("/tasks/<int:task_id>/refetch", methods=["POST"])
|
||||
async def refetch_task(task_id: int):
|
||||
"""Layer-2 one-shot re-download: delete the (corrupt) file behind a
|
||||
failed import task and re-run its source's downloader to fetch a
|
||||
fresh copy. Only works for files that resolve to an enabled,
|
||||
real-URL subscription Source; filesystem-only imports return
|
||||
no_source.
|
||||
|
||||
Returns one of: refetch_queued (+source_id) / no_source /
|
||||
already_refetched / not_found / not_failed.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(_refetch_task_sync, task_id)
|
||||
if result["status"] == "not_found":
|
||||
return jsonify(result), 404
|
||||
if result["status"] == "not_failed":
|
||||
return jsonify(result), 400
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _refetch_task_sync(session, task_id: int) -> dict:
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import ImportSettings
|
||||
from ..services.refetch_service import attempt_refetch
|
||||
|
||||
task = session.get(ImportTask, task_id)
|
||||
if task is None:
|
||||
return {"status": "not_found"}
|
||||
if task.status != "failed":
|
||||
return {"status": "not_failed"}
|
||||
settings = ImportSettings.load_sync(session)
|
||||
return attempt_refetch(session, task, Path(settings.import_scan_path))
|
||||
|
||||
|
||||
@import_admin_bp.route("/clear-stuck", methods=["POST"])
|
||||
async def clear_stuck():
|
||||
"""Force any non-terminal ImportTask (status in pending/queued/
|
||||
processing) to 'failed' AND finalize any ImportBatch that ends up
|
||||
with no active children. Escape hatch for the operator when the
|
||||
automatic recover_interrupted_tasks sweep keeps re-queueing the
|
||||
same stuck row forever (e.g., underlying file is genuinely broken
|
||||
and the import keeps OSError-looping at PIL load).
|
||||
|
||||
Idempotent + non-destructive: rows survive as 'failed' so the
|
||||
Retry-Failed button can re-attempt them once whatever was broken
|
||||
is fixed. Banked 2026-05-25 — operator hit 3 large PNGs that
|
||||
autoretry-looped for 2 days after a corrupt-data PIL OSError.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
# Fold SELECT into UPDATE…WHERE — see /retry-failed for the
|
||||
# 65535-parameter ceiling rationale. rowcount is enough here
|
||||
# because we don't need the ids afterward (no .delay()).
|
||||
clear_result = await session.execute(
|
||||
update(ImportTask)
|
||||
.where(
|
||||
ImportTask.status.in_(["pending", "queued", "processing"])
|
||||
)
|
||||
.values(
|
||||
status="failed",
|
||||
finished_at=datetime.now(UTC),
|
||||
error=(
|
||||
"manually cleared via /api/import/clear-stuck "
|
||||
"— stuck in non-terminal state; retry once "
|
||||
"underlying cause (corrupt file, missing model, "
|
||||
"etc.) is resolved"
|
||||
),
|
||||
)
|
||||
)
|
||||
tasks_failed = clear_result.rowcount or 0
|
||||
|
||||
# Finalize any 'running' ImportBatch that no longer has any
|
||||
# active children. The "Scanning..." banner is driven by
|
||||
# /api/import/status finding a running batch; left untouched,
|
||||
# it would persist forever after the stuck-task clear.
|
||||
running_batches = (
|
||||
await session.execute(
|
||||
select(ImportBatch.id).where(ImportBatch.status == "running")
|
||||
)
|
||||
failed_ids = (
|
||||
await session.execute(select(ImportTask.id).where(ImportTask.status == "failed"))
|
||||
).scalars().all()
|
||||
finalized_batches = 0
|
||||
for batch_id in running_batches:
|
||||
still_active = (
|
||||
await session.execute(
|
||||
select(ImportTask.id)
|
||||
.where(ImportTask.batch_id == batch_id)
|
||||
.where(ImportTask.status.in_(
|
||||
["pending", "queued", "processing"]
|
||||
))
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if still_active is None:
|
||||
await session.execute(
|
||||
update(ImportBatch)
|
||||
.where(ImportBatch.id == batch_id)
|
||||
.values(
|
||||
status="complete",
|
||||
finished_at=datetime.now(UTC),
|
||||
)
|
||||
)
|
||||
finalized_batches += 1
|
||||
if not failed_ids:
|
||||
return jsonify({"retried": 0})
|
||||
await session.execute(
|
||||
update(ImportTask)
|
||||
.where(ImportTask.id.in_(failed_ids))
|
||||
.values(status="queued", error=None, started_at=None, finished_at=None)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
return jsonify({
|
||||
"tasks_failed": tasks_failed,
|
||||
"batches_finalized": finalized_batches,
|
||||
})
|
||||
from ..tasks.import_file import import_media_file
|
||||
for tid in failed_ids:
|
||||
import_media_file.delay(tid)
|
||||
|
||||
return jsonify({"retried": len(failed_ids)})
|
||||
|
||||
|
||||
@import_admin_bp.route("/clear-completed", methods=["POST"])
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""FC-5: /api/migrate — trigger and poll migration runs.
|
||||
|
||||
Ingest kinds (gs_ingest, ir_ingest) accept multipart/form-data with an
|
||||
`export_file` field. All other kinds accept JSON. Apply-without-backup
|
||||
guard rejects non-dry-run ingests unless a pre_migration-tagged backup
|
||||
exists in the last 24h (override with body.force=true).
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import MigrationRun
|
||||
from ..tasks.migration import run_migration
|
||||
|
||||
migrate_bp = Blueprint("migrate", __name__, url_prefix="/api/migrate")
|
||||
|
||||
_VALID_KINDS = frozenset({
|
||||
"backup", "gs_ingest", "ir_ingest", "tag_apply",
|
||||
"ml_queue", "verify", "rollback", "cleanup",
|
||||
})
|
||||
_INGEST_KINDS = frozenset({"gs_ingest", "ir_ingest"})
|
||||
_APPLY_KINDS = frozenset({"gs_ingest", "ir_ingest", "tag_apply", "rollback", "cleanup"})
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, **extra):
|
||||
body = {"error": error}
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
def _has_recent_pre_migration_backup() -> bool:
|
||||
from ..services.migrators import backup as backup_mod
|
||||
images_root = Path("/images")
|
||||
manifest = backup_mod.find_latest_backup(images_root, tag="pre_migration")
|
||||
if manifest is None:
|
||||
return False
|
||||
created_at_str = manifest.get("created_at")
|
||||
if not created_at_str:
|
||||
return False
|
||||
created_at = datetime.fromisoformat(created_at_str)
|
||||
return (datetime.now(UTC) - created_at) < timedelta(hours=24)
|
||||
|
||||
|
||||
def _run_to_dict(run: MigrationRun) -> dict:
|
||||
return {
|
||||
"id": run.id,
|
||||
"kind": run.kind,
|
||||
"status": run.status,
|
||||
"dry_run": run.dry_run,
|
||||
"started_at": run.started_at.isoformat(),
|
||||
"finished_at": run.finished_at.isoformat() if run.finished_at else None,
|
||||
"counts": run.counts or {},
|
||||
"error": run.error,
|
||||
"metadata": run.metadata_ or {},
|
||||
}
|
||||
|
||||
|
||||
@migrate_bp.route("/<kind>", methods=["POST"])
|
||||
async def create_run(kind: str):
|
||||
if kind not in _VALID_KINDS:
|
||||
return _bad("unknown_kind", detail=f"kind must be one of {sorted(_VALID_KINDS)}")
|
||||
|
||||
# Ingest kinds accept multipart/form-data; everything else takes JSON.
|
||||
if kind in _INGEST_KINDS:
|
||||
form = await request.form
|
||||
files = await request.files
|
||||
if "export_file" not in files:
|
||||
return _bad("missing_export_file", detail="multipart export_file required")
|
||||
export_file = files["export_file"]
|
||||
try:
|
||||
raw = export_file.read()
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
return _bad("invalid_export_file", detail=str(exc))
|
||||
dry_run = str(form.get("dry_run", "false")).lower() in ("true", "1", "yes")
|
||||
force = str(form.get("force", "false")).lower() in ("true", "1", "yes")
|
||||
params: dict = {"data": data, "dry_run": dry_run}
|
||||
else:
|
||||
body = await request.get_json()
|
||||
if body is None:
|
||||
body = {}
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body")
|
||||
dry_run = bool(body.get("dry_run", False))
|
||||
force = bool(body.get("force", False))
|
||||
params = dict(body)
|
||||
|
||||
is_apply = (kind in _APPLY_KINDS) and not dry_run
|
||||
if is_apply and not force and not _has_recent_pre_migration_backup():
|
||||
return _bad(
|
||||
"no_backup",
|
||||
detail="apply action requires a pre_migration-tagged backup "
|
||||
"in the last 24h (or force=true).",
|
||||
)
|
||||
|
||||
if kind == "backup":
|
||||
params.setdefault("tag", "pre_migration")
|
||||
|
||||
async with get_session() as session:
|
||||
run = MigrationRun(kind=kind, status="pending", dry_run=dry_run)
|
||||
session.add(run)
|
||||
await session.commit()
|
||||
await session.refresh(run)
|
||||
run_id = run.id
|
||||
|
||||
run_migration.delay(run_id, kind, params)
|
||||
return jsonify({"run_id": run_id, "status": "pending"}), 202
|
||||
|
||||
|
||||
@migrate_bp.route("/runs/<int:run_id>", methods=["GET"])
|
||||
async def get_run(run_id: int):
|
||||
async with get_session() as session:
|
||||
run = (await session.execute(
|
||||
select(MigrationRun).where(MigrationRun.id == run_id)
|
||||
)).scalar_one_or_none()
|
||||
if run is None:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(_run_to_dict(run))
|
||||
|
||||
|
||||
@migrate_bp.route("/runs", methods=["GET"])
|
||||
async def list_runs():
|
||||
try:
|
||||
limit = int(request.args.get("limit", "10"))
|
||||
except ValueError:
|
||||
return _bad("invalid_limit")
|
||||
if limit < 1 or limit > 100:
|
||||
return _bad("invalid_limit")
|
||||
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(
|
||||
select(MigrationRun)
|
||||
.order_by(MigrationRun.id.desc())
|
||||
.limit(limit)
|
||||
)).scalars().all()
|
||||
return jsonify([_run_to_dict(r) for r in rows])
|
||||
+38
-170
@@ -1,206 +1,74 @@
|
||||
"""ML admin API: settings + backfill trigger."""
|
||||
"""ML admin API: settings, backfill trigger, centroid recompute trigger."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import MLSettings
|
||||
from ..services.ml.heads import AUTO_APPLY_THRESHOLD_MAX, AUTO_APPLY_THRESHOLD_MIN
|
||||
|
||||
ml_admin_bp = Blueprint("ml_admin", __name__, url_prefix="/api/ml")
|
||||
|
||||
|
||||
# Crop-proposer / detector config (#134). Announced to the GPU agent in the lease
|
||||
# → tunable here with no restart. weights = ultralytics name | URL | hf_repo::file
|
||||
# (empty, or enabled off, skips that proposer).
|
||||
_DETECTOR_FIELDS = (
|
||||
"detector_person_enabled",
|
||||
"detector_person_weights",
|
||||
"detector_person_conf",
|
||||
"detector_anatomy_enabled",
|
||||
"detector_anatomy_weights",
|
||||
"detector_anatomy_conf",
|
||||
"detector_panel_enabled",
|
||||
"detector_panel_weights",
|
||||
"detector_panel_conf",
|
||||
"detector_max_figures",
|
||||
"detector_max_components",
|
||||
"detector_max_panels",
|
||||
"detector_max_regions",
|
||||
"detector_dedupe_iou",
|
||||
)
|
||||
|
||||
_EDITABLE = (
|
||||
"cpu_embed_enabled",
|
||||
"video_frame_interval_seconds",
|
||||
"video_max_frames",
|
||||
"head_min_positives",
|
||||
"head_auto_apply_precision",
|
||||
"head_auto_apply_enabled",
|
||||
"head_auto_apply_min_positives",
|
||||
"ccip_match_threshold",
|
||||
"ccip_auto_apply_enabled",
|
||||
"ccip_auto_apply_threshold",
|
||||
"presentation_auto_apply_enabled",
|
||||
"presentation_auto_apply_threshold",
|
||||
"presentation_conflict_threshold",
|
||||
"process_auto_apply_enabled",
|
||||
"process_auto_apply_threshold",
|
||||
"process_conflict_threshold",
|
||||
"embedder_model_name",
|
||||
"embedder_model_version",
|
||||
# Discord drop grouping (#388 E2). Operator-facing because the quality bar
|
||||
# is a judgement no test can settle: too greedy merges distinct pieces, too
|
||||
# shy leaves a drop scattered.
|
||||
"discord_grouping_enabled",
|
||||
"discord_group_max_distance",
|
||||
"discord_group_window_minutes",
|
||||
# E3: how long a grouping stays open, and the anti-thrash rule that keeps
|
||||
# a growing one from monopolising the feed.
|
||||
"discord_group_close_after_hours",
|
||||
"discord_group_resurface_min_images",
|
||||
"discord_group_resurface_cooldown_hours",
|
||||
*_DETECTOR_FIELDS,
|
||||
"suggestion_threshold_artist",
|
||||
"suggestion_threshold_character",
|
||||
"suggestion_threshold_copyright",
|
||||
"suggestion_threshold_general",
|
||||
"centroid_similarity_threshold",
|
||||
"min_reference_images",
|
||||
)
|
||||
|
||||
|
||||
# Supported embedders for the Settings dropdown — all 1152-d so a swap is a
|
||||
# drop-in (re-embed + retrain, no schema change). Server-authoritative so the UI
|
||||
# never free-types a model name.
|
||||
SUPPORTED_EMBEDDERS = (
|
||||
{
|
||||
"name": "google/siglip2-so400m-patch16-512",
|
||||
"version": "siglip2-so400m-patch16-512",
|
||||
"label": "SigLIP 2 · so400m · 512px (recommended)",
|
||||
"dim": 1152,
|
||||
},
|
||||
{
|
||||
"name": "google/siglip2-so400m-patch16-384",
|
||||
"version": "siglip2-so400m-patch16-384",
|
||||
"label": "SigLIP 2 · so400m · 384px (faster)",
|
||||
"dim": 1152,
|
||||
},
|
||||
{
|
||||
"name": "google/siglip-so400m-patch14-384",
|
||||
"version": "siglip-so400m-patch14-384",
|
||||
"label": "SigLIP 1 · so400m · 384px (original)",
|
||||
"dim": 1152,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@ml_admin_bp.route("/embedder-models", methods=["GET"])
|
||||
async def embedder_models():
|
||||
return jsonify({"models": list(SUPPORTED_EMBEDDERS)})
|
||||
|
||||
|
||||
@ml_admin_bp.route("/settings", methods=["GET"])
|
||||
async def get_settings():
|
||||
from sqlalchemy import select
|
||||
|
||||
async with get_session() as session:
|
||||
s = await MLSettings.load(session)
|
||||
# Table-driven off _EDITABLE (which PATCH also writes) so a new settings field
|
||||
# can never be silently absent from GET — the split that historically dropped
|
||||
# fields. _EDITABLE already includes *_DETECTOR_FIELDS.
|
||||
return jsonify({f: getattr(s, f) for f in _EDITABLE})
|
||||
s = (
|
||||
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||
).scalar_one()
|
||||
return jsonify(
|
||||
{
|
||||
"suggestion_threshold_artist": s.suggestion_threshold_artist,
|
||||
"suggestion_threshold_character": s.suggestion_threshold_character,
|
||||
"suggestion_threshold_copyright": s.suggestion_threshold_copyright,
|
||||
"suggestion_threshold_general": s.suggestion_threshold_general,
|
||||
"centroid_similarity_threshold": s.centroid_similarity_threshold,
|
||||
"min_reference_images": s.min_reference_images,
|
||||
"tagger_model_version": s.tagger_model_version,
|
||||
"embedder_model_version": s.embedder_model_version,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@ml_admin_bp.route("/settings", methods=["PATCH"])
|
||||
async def patch_settings():
|
||||
from sqlalchemy import select
|
||||
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return jsonify({"error": "body must be an object"}), 400
|
||||
async with get_session() as session:
|
||||
s = await MLSettings.load(session)
|
||||
|
||||
# Merge the patch over current values, then validate the result as a
|
||||
# whole — the store-floor invariant couples three fields, so they
|
||||
# can't be checked one at a time.
|
||||
proposed = {f: getattr(s, f) for f in _EDITABLE}
|
||||
s = (
|
||||
await session.execute(select(MLSettings).where(MLSettings.id == 1))
|
||||
).scalar_one()
|
||||
for field in _EDITABLE:
|
||||
if field in body:
|
||||
proposed[field] = body[field]
|
||||
|
||||
err = _validate(proposed)
|
||||
if err is not None:
|
||||
return jsonify({"error": err}), 400
|
||||
|
||||
for field in _EDITABLE:
|
||||
setattr(s, field, proposed[field])
|
||||
setattr(s, field, body[field])
|
||||
await session.commit()
|
||||
return await get_settings()
|
||||
|
||||
|
||||
def _validate(p: dict) -> str | None:
|
||||
"""Returns an error string if the proposed settings are invalid, else None."""
|
||||
# Video embedding (#747).
|
||||
if p["video_frame_interval_seconds"] <= 0:
|
||||
return "video_frame_interval_seconds must be > 0"
|
||||
if p["video_max_frames"] < 1:
|
||||
return "video_max_frames must be >= 1"
|
||||
# Head training (#114).
|
||||
if int(p["head_min_positives"]) < 1:
|
||||
return "head_min_positives must be >= 1"
|
||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["head_auto_apply_precision"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
||||
return f"head_auto_apply_precision must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
||||
if int(p["head_auto_apply_min_positives"]) < 1:
|
||||
return "head_auto_apply_min_positives must be >= 1"
|
||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["ccip_match_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
||||
return f"ccip_match_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["ccip_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
||||
return f"ccip_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
||||
# Presentation chrome auto-hide (#141). Auto-apply runs high (hiding is
|
||||
# consequential); the conflict cut is a plain probability [0,1].
|
||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["presentation_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
||||
return f"presentation_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
||||
if not (0.0 <= float(p["presentation_conflict_threshold"]) <= 1.0):
|
||||
return "presentation_conflict_threshold must be between 0 and 1"
|
||||
# Process auto-apply (#1464). wip/editor stay VISIBLE so a false apply is
|
||||
# low-harm (excludes-from-training + a review flag), but keep the same bar.
|
||||
if not (AUTO_APPLY_THRESHOLD_MIN <= float(p["process_auto_apply_threshold"]) <= AUTO_APPLY_THRESHOLD_MAX):
|
||||
return f"process_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
||||
if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
|
||||
return "process_conflict_threshold must be between 0 and 1"
|
||||
# Discord drop grouping (#388 E2). max_distance is a cosine DISTANCE, so
|
||||
# unlike the *_threshold family above it is not on the auto-apply scale:
|
||||
# 0 is identical and 1 is unrelated, and both ends are legal. The upper
|
||||
# bound is 1.0 rather than AUTO_APPLY_THRESHOLD_MAX for that reason.
|
||||
if not (0.0 <= float(p["discord_group_max_distance"]) <= 1.0):
|
||||
return "discord_group_max_distance must be between 0 and 1"
|
||||
if float(p["discord_group_window_minutes"]) <= 0:
|
||||
return "discord_group_window_minutes must be > 0"
|
||||
# A group must stay open at least as long as the drop window it was cut
|
||||
# with, or the joiner could never reach a message the grouper deferred —
|
||||
# the two would fight, and the symptom (drops that never grow) would look
|
||||
# like the predicate failing rather than a settings contradiction.
|
||||
if float(p["discord_group_close_after_hours"]) * 60 < float(p["discord_group_window_minutes"]):
|
||||
return "discord_group_close_after_hours must be at least the drop window"
|
||||
if int(p["discord_group_resurface_min_images"]) < 1:
|
||||
return "discord_group_resurface_min_images must be >= 1"
|
||||
if float(p["discord_group_resurface_cooldown_hours"]) < 0:
|
||||
return "discord_group_resurface_cooldown_hours must be >= 0"
|
||||
# Embedder model swap (#1190): both must be non-empty. Changing them means a
|
||||
# different embedding space — the operator must re-embed + retrain after.
|
||||
for key in ("embedder_model_name", "embedder_model_version"):
|
||||
if not str(p[key]).strip():
|
||||
return f"{key} must not be empty"
|
||||
# Crop proposers (#134). Weights may be empty (that proposer is just off);
|
||||
# confidences are probabilities; caps are positive counts; IoU is [0,1].
|
||||
for key in ("detector_person_conf", "detector_anatomy_conf", "detector_panel_conf"):
|
||||
if not (0.0 <= float(p[key]) <= 1.0):
|
||||
return f"{key} must be between 0 and 1"
|
||||
for key in (
|
||||
"detector_max_figures", "detector_max_components",
|
||||
"detector_max_panels", "detector_max_regions",
|
||||
):
|
||||
if int(p[key]) < 1:
|
||||
return f"{key} must be >= 1"
|
||||
if not (0.0 <= float(p["detector_dedupe_iou"]) <= 1.0):
|
||||
return "detector_dedupe_iou must be between 0 and 1"
|
||||
return None
|
||||
|
||||
|
||||
@ml_admin_bp.route("/backfill", methods=["POST"])
|
||||
async def trigger_backfill():
|
||||
from ..tasks.ml import backfill
|
||||
|
||||
r = backfill.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@ml_admin_bp.route("/recompute-centroids", methods=["POST"])
|
||||
async def trigger_recompute():
|
||||
from ..tasks.ml import recompute_centroids
|
||||
|
||||
r = recompute_centroids.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
+8
-153
@@ -3,29 +3,18 @@
|
||||
from quart import Blueprint, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import ImportSettings, Post
|
||||
from ..services import interpreter_client as ic
|
||||
from ..services.post_association_service import PostAssociationService
|
||||
from ..services.post_association_service import rescan as association_rescan
|
||||
from ..services.post_feed_service import PostFeedService
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
from ..utils.text import html_to_plain
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
posts_bp = Blueprint("posts", __name__, url_prefix="/api/posts")
|
||||
|
||||
_TRANSLATION_OVERRIDES = ("auto", "force", "original")
|
||||
|
||||
|
||||
def _queue_for_sweep(post: Post) -> None:
|
||||
"""Mark a post untranslated (all translation columns NULL) so the periodic
|
||||
sweep re-runs it under its new override — used when Interpreter is down and we
|
||||
can't translate inline."""
|
||||
post.post_title_translated = None
|
||||
post.description_translated = None
|
||||
post.translated_source_lang = None
|
||||
post.translation_engine_version = None
|
||||
post.translated_at = None
|
||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
||||
body = {"error": error}
|
||||
if detail is not None:
|
||||
body["detail"] = detail
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
@posts_bp.route("", methods=["GET"])
|
||||
@@ -35,10 +24,7 @@ async def list_posts():
|
||||
cursor = args.get("cursor") or None
|
||||
artist_id_raw = args.get("artist_id")
|
||||
platform = args.get("platform") or None
|
||||
q = (args.get("q") or "").strip() or None
|
||||
limit_raw = args.get("limit", "24")
|
||||
direction = args.get("direction", "older")
|
||||
around_raw = args.get("around")
|
||||
|
||||
try:
|
||||
limit = int(limit_raw)
|
||||
@@ -47,16 +33,6 @@ async def list_posts():
|
||||
if limit < 1 or limit > 100:
|
||||
return _bad("invalid_limit", detail="limit must be between 1 and 100")
|
||||
|
||||
if direction not in ("older", "newer"):
|
||||
return _bad("invalid_direction", detail="direction must be 'older' or 'newer'")
|
||||
|
||||
around_id = None
|
||||
if around_raw is not None:
|
||||
try:
|
||||
around_id = int(around_raw)
|
||||
except ValueError:
|
||||
return _bad("invalid_around", detail="around must be an integer post id")
|
||||
|
||||
artist_id = None
|
||||
if artist_id_raw is not None:
|
||||
try:
|
||||
@@ -71,19 +47,10 @@ async def list_posts():
|
||||
)
|
||||
|
||||
async with get_session() as session:
|
||||
svc = PostFeedService(session)
|
||||
if around_id is not None:
|
||||
result = await svc.around(
|
||||
post_id=around_id, artist_id=artist_id,
|
||||
platform=platform, q=q, limit=limit,
|
||||
)
|
||||
if result is None:
|
||||
return _bad("not_found", status=404, detail=f"post id={around_id}")
|
||||
return jsonify(result)
|
||||
try:
|
||||
page = await svc.scroll(
|
||||
page = await PostFeedService(session).scroll(
|
||||
cursor=cursor, artist_id=artist_id,
|
||||
platform=platform, q=q, limit=limit, direction=direction,
|
||||
platform=platform, limit=limit,
|
||||
)
|
||||
except ValueError as exc:
|
||||
# Service raises ValueError for malformed cursors only;
|
||||
@@ -100,115 +67,3 @@ async def get_post(post_id: int):
|
||||
if item is None:
|
||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||
return jsonify(item)
|
||||
|
||||
|
||||
@posts_bp.route("/<int:post_id>/translation-override", methods=["POST"])
|
||||
async def set_translation_override(post_id: int):
|
||||
"""Sticky per-post translation override (milestone 155). Body:
|
||||
``{"override": "auto" | "force" | "original"}``.
|
||||
|
||||
'original' keeps the original (clears any stored translation now — no
|
||||
Interpreter needed). 'force'/'auto' translate the post immediately if the
|
||||
service is up (force bypasses the acceptance floor; auto re-runs the gate);
|
||||
if it's down we save the flag and mark the post untranslated so the next sweep
|
||||
applies it. The override persists, so the sweep + Re-translate-all keep
|
||||
honoring it. Returns the updated translation fields + an ``applied`` status."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
override = body.get("override")
|
||||
if override not in _TRANSLATION_OVERRIDES:
|
||||
return _bad(
|
||||
"invalid_override",
|
||||
detail=f"override must be one of {list(_TRANSLATION_OVERRIDES)}",
|
||||
)
|
||||
|
||||
# Lazy import (mirrors settings.py) so the API module doesn't pull the celery
|
||||
# task graph at import time.
|
||||
from ..tasks.translation import _store_translation, _translate_field
|
||||
|
||||
async with get_session() as session:
|
||||
post = await session.get(Post, post_id)
|
||||
if post is None:
|
||||
return _bad("not_found", status=404, detail=f"post id={post_id}")
|
||||
post.translation_override = override
|
||||
cfg = await ImportSettings.load(session)
|
||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
||||
|
||||
if override == "original":
|
||||
_store_translation(post, (None, None, None), (None, None, None), target)
|
||||
applied = "cleared"
|
||||
else:
|
||||
base_url = (cfg.interpreter_base_url or "").strip()
|
||||
if cfg.translation_enabled and base_url and ic.health(base_url):
|
||||
force = override == "force"
|
||||
title = (post.post_title or "").strip()
|
||||
desc = (html_to_plain(post.description) if post.description else "") or ""
|
||||
desc = desc.strip()
|
||||
mc = cfg.translation_min_confidence
|
||||
try:
|
||||
title_res = _translate_field(title, base_url, target, mc, force=force)
|
||||
desc_res = _translate_field(desc, base_url, target, mc, force=force)
|
||||
except ic.InterpreterUnavailable:
|
||||
_queue_for_sweep(post)
|
||||
applied = "queued"
|
||||
else:
|
||||
_store_translation(post, title_res, desc_res, target)
|
||||
applied = "translated"
|
||||
else:
|
||||
# Disabled / no URL / unhealthy → let the sweep apply it later.
|
||||
_queue_for_sweep(post)
|
||||
applied = "queued"
|
||||
|
||||
await session.commit()
|
||||
return jsonify({
|
||||
"id": post.id,
|
||||
"translation_override": post.translation_override,
|
||||
"post_title_translated": post.post_title_translated,
|
||||
"description_translated": post.description_translated,
|
||||
"translated_source_lang": post.translated_source_lang,
|
||||
"applied": applied,
|
||||
})
|
||||
|
||||
|
||||
# --- #388 E5: the announcement review queue -------------------------------
|
||||
#
|
||||
# Confirm-only, following the series-suggestion routes (api/tags.py). Nothing
|
||||
# here links anything on its own: the matcher proposes, the operator decides.
|
||||
|
||||
|
||||
@posts_bp.route("/associations", methods=["GET"])
|
||||
async def list_associations():
|
||||
async with get_session() as session:
|
||||
return jsonify({"items": await PostAssociationService(session).list_pending()})
|
||||
|
||||
|
||||
@posts_bp.route("/associations/<int:association_id>/accept", methods=["POST"])
|
||||
async def accept_association(association_id: int):
|
||||
async with get_session() as session:
|
||||
result = await PostAssociationService(session).accept(association_id)
|
||||
if result is None:
|
||||
return _bad("association not found", 404)
|
||||
await session.commit()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@posts_bp.route("/associations/<int:association_id>/dismiss", methods=["POST"])
|
||||
async def dismiss_association(association_id: int):
|
||||
async with get_session() as session:
|
||||
result = await PostAssociationService(session).dismiss(association_id)
|
||||
if result is None:
|
||||
return _bad("association not found", 404)
|
||||
await session.commit()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@posts_bp.route("/associations/rescan", methods=["POST"])
|
||||
async def rescan_associations():
|
||||
"""Manual re-scan. The beat sweep only looks at recent posts (a pair has to
|
||||
be within the window to exist at all); this is the button for a first run
|
||||
over a library that predates the feature."""
|
||||
async with get_session() as session:
|
||||
# full=True: the button reaches the whole history, which the hourly
|
||||
# sweep's 48-hour horizon never does.
|
||||
result = await association_rescan(session, full=True)
|
||||
await session.commit()
|
||||
return jsonify(result)
|
||||
|
||||
+23
-294
@@ -1,24 +1,12 @@
|
||||
"""Settings API: import filters, system stats."""
|
||||
|
||||
import asyncio
|
||||
import secrets
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, or_, select
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import (
|
||||
AppSetting,
|
||||
Artist,
|
||||
ImageRecord,
|
||||
ImportBatch,
|
||||
ImportSettings,
|
||||
ImportTask,
|
||||
Post,
|
||||
Tag,
|
||||
TaskRun,
|
||||
)
|
||||
from ..services import interpreter_client as ic
|
||||
from ..models import AppSetting, Artist, ImageRecord, ImportBatch, ImportSettings, ImportTask, Tag
|
||||
|
||||
settings_bp = Blueprint("settings", __name__, url_prefix="/api")
|
||||
|
||||
@@ -36,47 +24,31 @@ _EDITABLE_FIELDS = (
|
||||
"download_validate_files",
|
||||
"download_schedule_default_seconds",
|
||||
"download_event_retention_days",
|
||||
"download_revisit_days",
|
||||
"download_failure_warning_threshold",
|
||||
"series_suggest_enabled",
|
||||
"series_suggest_threshold",
|
||||
# #388 E5 — the announcement matcher (Patreon teaser ↔ Discord drop).
|
||||
"discord_link_enabled",
|
||||
"discord_link_threshold",
|
||||
"discord_link_window_hours",
|
||||
"discord_link_auto",
|
||||
"discord_link_fold_hours",
|
||||
"discord_family_window_days",
|
||||
"extdl_mega_enabled",
|
||||
"extdl_gdrive_enabled",
|
||||
"extdl_mediafire_enabled",
|
||||
"extdl_dropbox_enabled",
|
||||
"extdl_pixeldrain_enabled",
|
||||
"translation_enabled",
|
||||
"interpreter_base_url",
|
||||
"translation_target_lang",
|
||||
"translation_min_confidence",
|
||||
"wip_title_tagging_enabled",
|
||||
"wip_soft_title_tagging_enabled",
|
||||
)
|
||||
|
||||
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
||||
_EXTDL_TOGGLE_FIELDS = (
|
||||
"extdl_mega_enabled",
|
||||
"extdl_gdrive_enabled",
|
||||
"extdl_mediafire_enabled",
|
||||
"extdl_dropbox_enabled",
|
||||
"extdl_pixeldrain_enabled",
|
||||
)
|
||||
|
||||
|
||||
@settings_bp.route("/settings/import", methods=["GET"])
|
||||
async def get_import_settings():
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
# Table-driven off _EDITABLE_FIELDS (which PATCH also writes) so a new field
|
||||
# can't be silently absent from GET.
|
||||
return jsonify({f: getattr(row, f) for f in _EDITABLE_FIELDS})
|
||||
row = (
|
||||
await session.execute(select(ImportSettings).where(ImportSettings.id == 1))
|
||||
).scalar_one()
|
||||
return jsonify({
|
||||
"min_width": row.min_width,
|
||||
"min_height": row.min_height,
|
||||
"skip_transparent": row.skip_transparent,
|
||||
"transparency_threshold": row.transparency_threshold,
|
||||
"skip_single_color": row.skip_single_color,
|
||||
"single_color_threshold": row.single_color_threshold,
|
||||
"single_color_tolerance": row.single_color_tolerance,
|
||||
"phash_threshold": row.phash_threshold,
|
||||
"download_rate_limit_seconds": row.download_rate_limit_seconds,
|
||||
"download_validate_files": row.download_validate_files,
|
||||
"download_schedule_default_seconds": row.download_schedule_default_seconds,
|
||||
"download_event_retention_days": row.download_event_retention_days,
|
||||
"download_failure_warning_threshold": row.download_failure_warning_threshold,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/import", methods=["PATCH"])
|
||||
@@ -117,12 +89,6 @@ async def update_import_settings():
|
||||
v = body["download_schedule_default_seconds"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 60 or v > 86400:
|
||||
return _bad_int("download_schedule_default_seconds", 60, 86400)
|
||||
# 0 is a real value (revisit off), so the floor is 0, not 1 — and the
|
||||
# ceiling is a year, past which a "tick" is a backfill wearing a hat.
|
||||
if "download_revisit_days" in body:
|
||||
v = body["download_revisit_days"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 0 or v > 365:
|
||||
return _bad_int("download_revisit_days", 0, 365)
|
||||
if "download_event_retention_days" in body:
|
||||
v = body["download_event_retention_days"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 3650:
|
||||
@@ -132,79 +98,10 @@ async def update_import_settings():
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 1 or v > 100:
|
||||
return _bad_int("download_failure_warning_threshold", 1, 100)
|
||||
|
||||
if "series_suggest_enabled" in body and not isinstance(
|
||||
body["series_suggest_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "series_suggest_enabled must be a boolean"}
|
||||
), 400
|
||||
for tog in _EXTDL_TOGGLE_FIELDS:
|
||||
if tog in body and not isinstance(body[tog], bool):
|
||||
return jsonify({"error": f"{tog} must be a boolean"}), 400
|
||||
# Translation (#143): base URL may be empty (feature off until set — no
|
||||
# default host; the operator points it at their own Interpreter proxy).
|
||||
if "translation_enabled" in body and not isinstance(
|
||||
body["translation_enabled"], bool
|
||||
):
|
||||
return jsonify({"error": "translation_enabled must be a boolean"}), 400
|
||||
for key in ("interpreter_base_url", "translation_target_lang"):
|
||||
if key in body and not isinstance(body[key], str):
|
||||
return jsonify({"error": f"{key} must be a string"}), 400
|
||||
# Acceptance floor (milestone 155): latin-script translations below this
|
||||
# Interpreter confidence are kept as the original.
|
||||
if "translation_min_confidence" in body:
|
||||
v = body["translation_min_confidence"]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||
return jsonify(
|
||||
{"error": "translation_min_confidence must be a number in [0, 1]"}
|
||||
), 400
|
||||
if "series_suggest_threshold" in body:
|
||||
v = body["series_suggest_threshold"]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||
return jsonify(
|
||||
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
||||
), 400
|
||||
if "discord_link_enabled" in body and not isinstance(
|
||||
body["discord_link_enabled"], bool
|
||||
):
|
||||
return jsonify({"error": "discord_link_enabled must be a boolean"}), 400
|
||||
if "discord_link_auto" in body and not isinstance(
|
||||
body["discord_link_auto"], bool
|
||||
):
|
||||
return jsonify({"error": "discord_link_auto must be a boolean"}), 400
|
||||
if "discord_link_threshold" in body:
|
||||
v = body["discord_link_threshold"]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||
return jsonify(
|
||||
{"error": "discord_link_threshold must be a number in [0, 1]"}
|
||||
), 400
|
||||
if "discord_link_window_hours" in body:
|
||||
v = body["discord_link_window_hours"]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v <= 0:
|
||||
return jsonify(
|
||||
{"error": "discord_link_window_hours must be a positive number"}
|
||||
), 400
|
||||
# Zero is meaningful for both: fold nothing, or reference no variants.
|
||||
for key in ("discord_link_fold_hours", "discord_family_window_days"):
|
||||
if key in body:
|
||||
v = body[key]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0:
|
||||
return jsonify({"error": f"{key} must be a number >= 0"}), 400
|
||||
if "wip_title_tagging_enabled" in body and not isinstance(
|
||||
body["wip_title_tagging_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "wip_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
if "wip_soft_title_tagging_enabled" in body and not isinstance(
|
||||
body["wip_soft_title_tagging_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "wip_soft_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
row = (
|
||||
await session.execute(select(ImportSettings).where(ImportSettings.id == 1))
|
||||
).scalar_one()
|
||||
for field in _EDITABLE_FIELDS:
|
||||
if field in body:
|
||||
setattr(row, field, body[field])
|
||||
@@ -213,18 +110,6 @@ async def update_import_settings():
|
||||
return await get_import_settings()
|
||||
|
||||
|
||||
@settings_bp.route("/settings/wip-title/scan", methods=["POST"])
|
||||
async def wip_title_scan():
|
||||
"""Enqueue the back-catalogue WIP-title scan (task #1458 Settings button):
|
||||
apply the `wip` system tag to EXISTING posts whose title declares
|
||||
work-in-progress. New imports are tagged live by the importer; this catches
|
||||
the existing library. Returns the Celery task id (202)."""
|
||||
from ..tasks.maintenance import backfill_wip_title_tags
|
||||
|
||||
r = backfill_wip_title_tags.delay()
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@settings_bp.route("/system/stats", methods=["GET"])
|
||||
async def system_stats():
|
||||
async with get_session() as session:
|
||||
@@ -350,159 +235,3 @@ async def rotate_extension_api_key():
|
||||
row.value = new_value
|
||||
await session.commit()
|
||||
return jsonify({"key": new_value})
|
||||
|
||||
|
||||
# --- Translation (#143): live status + manual "Translate now" --------------
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/status", methods=["GET"])
|
||||
async def translation_status():
|
||||
"""For the Settings card: is it on, is a URL set, is the service reachable,
|
||||
and how many posts still await translation. Health runs the sync client in a
|
||||
thread so the event loop isn't blocked."""
|
||||
translation_tasks = (
|
||||
"backend.app.tasks.translation.translate_posts",
|
||||
"backend.app.tasks.translation.retranslate_posts",
|
||||
)
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
untranslated = (await session.execute(
|
||||
select(func.count(Post.id))
|
||||
.where(Post.translated_source_lang.is_(None))
|
||||
.where(or_(
|
||||
Post.post_title.is_not(None), Post.description.is_not(None),
|
||||
))
|
||||
)).scalar_one()
|
||||
# Live progress: is a sweep running now, and what did the last one do?
|
||||
# (run-until-done re-enqueues itself, so `active` stays true across a
|
||||
# bulk re-translate; `last_run` surfaces a completed run's outcome.)
|
||||
active = (await session.execute(
|
||||
select(func.count(TaskRun.id))
|
||||
.where(TaskRun.task_name.in_(translation_tasks))
|
||||
.where(TaskRun.status == "running")
|
||||
)).scalar_one()
|
||||
last = (await session.execute(
|
||||
select(TaskRun.task_name, TaskRun.status, TaskRun.finished_at)
|
||||
.where(TaskRun.task_name.in_(translation_tasks))
|
||||
.where(TaskRun.finished_at.is_not(None))
|
||||
.order_by(TaskRun.finished_at.desc())
|
||||
.limit(1)
|
||||
)).first()
|
||||
base_url = (cfg.interpreter_base_url or "").strip()
|
||||
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
||||
return jsonify({
|
||||
"enabled": cfg.translation_enabled,
|
||||
"base_url_set": bool(base_url),
|
||||
"healthy": healthy,
|
||||
"untranslated_count": int(untranslated),
|
||||
"active": int(active) > 0,
|
||||
"last_run": {
|
||||
"task": last[0].rsplit(".", 1)[-1],
|
||||
"status": last[1],
|
||||
"finished_at": last[2].isoformat() if last[2] else None,
|
||||
} if last else None,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/test", methods=["POST"])
|
||||
async def translation_test():
|
||||
"""On-demand reachability check for a GIVEN Interpreter base URL (the Settings
|
||||
'Test connection' button) — pings /v1/health without saving, so the operator
|
||||
can verify a URL before enabling. Health runs in a thread (sync client)."""
|
||||
body = await request.get_json()
|
||||
base_url = ""
|
||||
if isinstance(body, dict):
|
||||
base_url = (body.get("base_url") or "").strip()
|
||||
healthy = await asyncio.to_thread(ic.health, base_url) if base_url else False
|
||||
return jsonify({"healthy": healthy})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/probe", methods=["POST"])
|
||||
async def translation_probe():
|
||||
"""Diagnostic for the Settings 'Test translation' box: translate a pasted
|
||||
snippet WITHOUT saving anything, returning what Interpreter *detected*
|
||||
(language + confidence) alongside the result. Lets the operator see why a
|
||||
given string was (mis-)detected — e.g. a short English title flagged as
|
||||
another language — so a detection guard can be tuned from real numbers.
|
||||
Read-only: no post is touched. Uses the currently-saved base URL + target."""
|
||||
body = await request.get_json(silent=True)
|
||||
body = body if isinstance(body, dict) else {}
|
||||
text = (body.get("text") or "").strip()
|
||||
if not text:
|
||||
return jsonify({"error": "provide text to translate"}), 400
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
base_url = (cfg.interpreter_base_url or "").strip()
|
||||
if not base_url:
|
||||
return jsonify({"error": "no Interpreter base URL is set"}), 400
|
||||
target = (cfg.translation_target_lang or "en").strip() or "en"
|
||||
try:
|
||||
res = await asyncio.to_thread(
|
||||
ic.translate, [text], base_url=base_url, target=target,
|
||||
)
|
||||
except ic.InterpreterUnavailable as e:
|
||||
return jsonify({"error": f"Interpreter unavailable: {e}"}), 503
|
||||
except ic.InterpreterBadRequest as e:
|
||||
return jsonify({"error": f"Interpreter rejected the request: {e}"}), 400
|
||||
translations = res.get("translations") or []
|
||||
return jsonify({
|
||||
"target": target,
|
||||
"detected_lang": res.get("detected_lang"),
|
||||
"detected_confidence": res.get("detected_confidence"),
|
||||
"engine": res.get("engine"),
|
||||
"engine_version": res.get("engine_version"),
|
||||
"translated": translations[0] if translations else None,
|
||||
})
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/run", methods=["POST"])
|
||||
async def translation_run():
|
||||
"""Enqueue the translate sweep now (the Settings 'Translate now' button).
|
||||
Runs in drain mode — run-until-done — so one press chases the whole
|
||||
untranslated backlog to zero rather than a single 300-post chunk."""
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
|
||||
return jsonify(
|
||||
{"error": "translation is disabled or no base URL is set"}
|
||||
), 400
|
||||
from ..tasks.translation import translate_posts
|
||||
|
||||
r = translate_posts.delay(drain=True)
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
|
||||
@settings_bp.route("/settings/translation/retranslate", methods=["POST"])
|
||||
async def translation_retranslate():
|
||||
"""Re-translate stored translations after a model change (m146). Body:
|
||||
``{"artist_id": <int>}`` aims at one artist; ``{"all": true}`` re-runs every
|
||||
artist. ``all`` must be explicit so an empty/typo body can't wipe everything.
|
||||
Clears the scoped translations and enqueues the run-until-done retranslate
|
||||
sweep (the Interpreter cache re-translates on a changed model, is cache-fast
|
||||
otherwise). Same enabled + base-URL guard as 'Translate now'."""
|
||||
body = await request.get_json(silent=True)
|
||||
body = body if isinstance(body, dict) else {}
|
||||
artist_id = body.get("artist_id")
|
||||
do_all = bool(body.get("all"))
|
||||
if artist_id is None and not do_all:
|
||||
return jsonify(
|
||||
{"error": "provide artist_id, or all=true to re-translate everything"}
|
||||
), 400
|
||||
if artist_id is not None:
|
||||
try:
|
||||
artist_id = int(artist_id)
|
||||
except (TypeError, ValueError):
|
||||
return jsonify({"error": "artist_id must be an integer"}), 400
|
||||
|
||||
async with get_session() as session:
|
||||
cfg = await ImportSettings.load(session)
|
||||
if not cfg.translation_enabled or not (cfg.interpreter_base_url or "").strip():
|
||||
return jsonify(
|
||||
{"error": "translation is disabled or no base URL is set"}
|
||||
), 400
|
||||
from ..tasks.translation import retranslate_posts
|
||||
|
||||
# artist_id wins when both are sent; otherwise all=true → None (every artist).
|
||||
artist_ids = [artist_id] if artist_id is not None else None
|
||||
r = retranslate_posts.delay(artist_ids=artist_ids)
|
||||
return jsonify({"celery_task_id": r.id}), 202
|
||||
|
||||
+12
-321
@@ -1,16 +1,10 @@
|
||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
||||
from ..services.artist_membership_service import ArtistMembershipService
|
||||
from ..services.artist_membership_service import rescan as membership_rescan
|
||||
from ..services.artist_service import ArtistService
|
||||
from ..services.membership_reconcile import reconcile_all
|
||||
from ..services.membership_roster import roster_is_fresh, source_for_membership
|
||||
from ..services.scheduler_service import active_platform_cooldowns, scheduler_status
|
||||
from ..models import DownloadEvent, Source
|
||||
from ..services.source_service import (
|
||||
KNOWN_PLATFORMS,
|
||||
ArtistNotFoundError,
|
||||
@@ -20,11 +14,18 @@ from ..services.source_service import (
|
||||
SourceService,
|
||||
UnknownPlatformError,
|
||||
)
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
sources_bp = Blueprint("sources", __name__, url_prefix="/api/sources")
|
||||
|
||||
|
||||
def _bad(error: str, *, status: int = 400, detail: str | None = None, **extra):
|
||||
body = {"error": error}
|
||||
if detail is not None:
|
||||
body["detail"] = detail
|
||||
body.update(extra)
|
||||
return jsonify(body), status
|
||||
|
||||
|
||||
@sources_bp.route("", methods=["GET"])
|
||||
async def list_sources():
|
||||
artist_id_raw = request.args.get("artist_id")
|
||||
@@ -34,19 +35,11 @@ async def list_sources():
|
||||
artist_id = int(artist_id_raw)
|
||||
except ValueError:
|
||||
return _bad("invalid_artist_id", detail="artist_id must be an integer")
|
||||
failing = request.args.get("failing", "").lower() in ("1", "true", "yes")
|
||||
async with get_session() as session:
|
||||
records = await SourceService(session).list(artist_id=artist_id, failing=failing)
|
||||
records = await SourceService(session).list(artist_id=artist_id)
|
||||
return jsonify([r.to_dict() for r in records])
|
||||
|
||||
|
||||
@sources_bp.route("/schedule-status", methods=["GET"])
|
||||
async def schedule_status():
|
||||
"""FC-dashboards: scheduler health for the Subscriptions hub."""
|
||||
async with get_session() as session:
|
||||
return jsonify(await scheduler_status(session))
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>", methods=["GET"])
|
||||
async def get_source(source_id: int):
|
||||
async with get_session() as session:
|
||||
@@ -90,22 +83,6 @@ async def create_source():
|
||||
return _bad("empty_url", detail=str(exc))
|
||||
except DuplicateSourceError as exc:
|
||||
return _bad("duplicate", status=409, existing_id=exc.existing_id)
|
||||
|
||||
# Immediate kickoff: a new enabled source is armed for backfill (#693)
|
||||
# but would otherwise sit idle until the next scheduler tick (~60s).
|
||||
# Enqueue the first walk now, skipping only if the platform is in a
|
||||
# rate-limit cooldown (the scheduler picks it up when that clears).
|
||||
dispatch_id = None
|
||||
if record.enabled:
|
||||
cooldowns = await active_platform_cooldowns(session)
|
||||
if record.platform not in cooldowns:
|
||||
session.add(DownloadEvent(source_id=record.id, status="pending"))
|
||||
await session.commit()
|
||||
dispatch_id = record.id
|
||||
|
||||
if dispatch_id is not None:
|
||||
from ..tasks.download import download_source
|
||||
download_source.delay(dispatch_id)
|
||||
return jsonify(record.to_dict()), 201
|
||||
|
||||
|
||||
@@ -141,125 +118,12 @@ async def delete_source(source_id: int):
|
||||
return "", 204
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/reassign", methods=["POST"])
|
||||
async def reassign_source(source_id: int):
|
||||
"""Move this source (and the content it brought in) to another artist
|
||||
(#130). Files don't move — the slug is immutable — so this just re-attributes
|
||||
the source, its posts, and its images. Body: {target_artist_id}."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
target = body.get("target_artist_id")
|
||||
if not isinstance(target, int):
|
||||
return _bad("invalid_body", detail="target_artist_id (int) required")
|
||||
async with get_session() as session:
|
||||
try:
|
||||
record = await SourceService(session).reassign(source_id, target)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except ArtistNotFoundError:
|
||||
return _bad("artist_not_found", detail="target artist not found", status=404)
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/backfill", methods=["POST"])
|
||||
async def set_backfill(source_id: int):
|
||||
"""Plan #693/#697 + #830: start/stop a backfill, or start a recovery /
|
||||
recapture. Body: `{"action": "start" | "stop" | "recover" | "recapture"}`
|
||||
(default "start"). 'start' walks the full post history in time-boxed chunks
|
||||
until it reaches the bottom (then the source shows 'complete'); 'recover' is
|
||||
the same walk but bypasses the Patreon seen-ledger to re-fetch
|
||||
dropped-and-deleted near-dups under the current pHash threshold; 'recapture'
|
||||
re-grabs EVERY post's body + external links and localizes on-disk inline
|
||||
images WITHOUT re-downloading media; 'stop' cancels any back to tick mode.
|
||||
Returns the updated source dict (incl. backfill_state / backfill_chunks /
|
||||
backfill_bypass_seen / backfill_recapture)."""
|
||||
from pathlib import Path
|
||||
|
||||
from ..services.credential_service import CredentialService
|
||||
from ..services.download_backends import (
|
||||
uses_native_ingester,
|
||||
verify_source_credential,
|
||||
)
|
||||
from .credentials import _get_crypto
|
||||
|
||||
payload = await request.get_json(silent=True) or {}
|
||||
action = payload.get("action", "start")
|
||||
if action not in ("start", "stop", "recover", "recapture"):
|
||||
return _bad(
|
||||
"invalid_action",
|
||||
detail="action must be 'start', 'stop', 'recover', or 'recapture'",
|
||||
)
|
||||
|
||||
# Pre-flight (plan #703 #2): before arming a deep walk on a native-ingester
|
||||
# platform (where verify is one cheap API page), refuse if the credential is
|
||||
# DEFINITIVELY rejected — don't burn chunks against expired cookies. Proceed
|
||||
# on valid OR inconclusive (a network blip shouldn't block). Gated to native
|
||||
# platforms: gallery-dl verify is a slow --simulate subprocess, too heavy for
|
||||
# an arm action. The credential read happens in a session that's CLOSED
|
||||
# before the verify network call (don't hold a DB conn across the request).
|
||||
if action in ("start", "recover", "recapture"):
|
||||
async with get_session() as session:
|
||||
rec = await SourceService(session).get(source_id)
|
||||
if rec is None:
|
||||
return _bad("not_found", status=404)
|
||||
# A disabled source must not be armable for a deep walk — the same
|
||||
# rule /check has carried all along (see `source_disabled` below).
|
||||
# Arming one anyway is how #4279 happened: the membership sweep had
|
||||
# stopped Ebi77 as `former_patron`, a deep scan was armed twenty
|
||||
# minutes later, the walk could not complete without access, and
|
||||
# the recovery sweep stranded it with a failure count no surface
|
||||
# could clear — a disabled source is never scheduled again, and
|
||||
# Retry routes to /check, which refuses it.
|
||||
if not rec.enabled:
|
||||
return _bad("source_disabled", detail="enable the source first")
|
||||
native = uses_native_ingester(rec.platform)
|
||||
if native:
|
||||
cred = CredentialService(session, _get_crypto())
|
||||
cookies_path = await cred.get_cookies_path(rec.platform)
|
||||
auth_token = await cred.get_token(rec.platform)
|
||||
if native:
|
||||
ok, message = await verify_source_credential(
|
||||
platform=rec.platform,
|
||||
url=rec.url,
|
||||
artist_slug=rec.artist_slug,
|
||||
config_overrides=rec.config_overrides or {},
|
||||
cookies_path=str(cookies_path) if cookies_path else None,
|
||||
auth_token=auth_token,
|
||||
images_root=Path("/images"),
|
||||
)
|
||||
if ok is False:
|
||||
return _bad("credential_rejected", detail=message, status=409)
|
||||
|
||||
async with get_session() as session:
|
||||
try:
|
||||
svc = SourceService(session)
|
||||
if action == "start":
|
||||
record = await svc.start_backfill(source_id)
|
||||
elif action == "recover":
|
||||
record = await svc.start_recovery(source_id)
|
||||
elif action == "recapture":
|
||||
record = await svc.start_recapture(source_id)
|
||||
else:
|
||||
record = await svc.stop_backfill(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(record.to_dict())
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/check", methods=["POST"])
|
||||
async def check_source(source_id: int):
|
||||
"""FC-3c: enqueue a download for this source.
|
||||
|
||||
Returns 202 with the new DownloadEvent id. If a pending/running
|
||||
event already exists for this source, returns 409 with that id. If
|
||||
the source's platform is currently in a rate-limit cooldown, returns
|
||||
**202 with `{status: "deferred", cooldown_until, platform}`** and
|
||||
does NOT create an event or dispatch — the bulk retry path uses this
|
||||
to avoid bowling N sources right back into the rate limit the
|
||||
cooldown is preventing. Single-click "retry this one source" passes
|
||||
`?force=true` to override the cooldown (operator-explicit, useful
|
||||
for rapid auth-fix testing). The in-flight guard always applies.
|
||||
"""
|
||||
force = (request.args.get("force") or "").lower() in ("1", "true", "yes")
|
||||
event already exists for this source, returns 409 with that id."""
|
||||
async with get_session() as session:
|
||||
source = (await session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
@@ -269,19 +133,6 @@ async def check_source(source_id: int):
|
||||
if not source.enabled:
|
||||
return _bad("source_disabled", detail="enable the source first")
|
||||
|
||||
# Cooldown gate (unless explicitly overridden). Checked before the
|
||||
# in-flight guard because a deferred retry doesn't need to create
|
||||
# or check for an event at all.
|
||||
if not force:
|
||||
cooldowns = await active_platform_cooldowns(session)
|
||||
expires_at = cooldowns.get(source.platform)
|
||||
if expires_at is not None:
|
||||
return jsonify({
|
||||
"status": "deferred",
|
||||
"platform": source.platform,
|
||||
"cooldown_until": expires_at.isoformat(),
|
||||
}), 202
|
||||
|
||||
in_flight = (await session.execute(
|
||||
select(DownloadEvent.id).where(
|
||||
DownloadEvent.source_id == source_id,
|
||||
@@ -303,163 +154,3 @@ async def check_source(source_id: int):
|
||||
download_source.delay(source_id)
|
||||
|
||||
return jsonify({"download_event_id": event_id, "status": "pending"}), 202
|
||||
|
||||
|
||||
# --- #387 C3: the membership roster's sync state --------------------------
|
||||
#
|
||||
# Rule 164's visibility requirement lives here. A roster that failed to sync,
|
||||
# or never has, must be DISTINGUISHABLE from an account that subscribes to
|
||||
# nothing — otherwise the reconciliation this unlocks would tell the operator
|
||||
# to cancel sources they are actively paying for.
|
||||
|
||||
|
||||
@sources_bp.route("/membership-sync", methods=["GET"])
|
||||
async def membership_sync_status():
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(select(MembershipSync))).scalars().all()
|
||||
counts = dict(
|
||||
(await session.execute(
|
||||
select(PlatformMembership.platform, func.count())
|
||||
.group_by(PlatformMembership.platform)
|
||||
)).all()
|
||||
)
|
||||
return jsonify({"platforms": [
|
||||
{
|
||||
"platform": r.platform,
|
||||
"last_attempt_at": r.last_attempt_at.isoformat() if r.last_attempt_at else None,
|
||||
# NULL here means NEVER, and the UI must say so in words. Rendering
|
||||
# it as 0 or as "-" is the exact conflation this endpoint exists to
|
||||
# prevent.
|
||||
"last_success_at": r.last_success_at.isoformat() if r.last_success_at else None,
|
||||
"last_count": r.last_count,
|
||||
"last_error_type": r.last_error_type,
|
||||
"last_error_message": r.last_error_message,
|
||||
# Whether a CONCLUSION may be drawn from this roster — not merely
|
||||
# whether it looks recent. C4 gates on this, and it is computed
|
||||
# server-side so no caller can forget to.
|
||||
"fresh": roster_is_fresh(r),
|
||||
"known_memberships": counts.get(r.platform, 0),
|
||||
}
|
||||
for r in sorted(rows, key=lambda r: r.platform)
|
||||
]})
|
||||
|
||||
|
||||
@sources_bp.route("/membership-sync", methods=["POST"])
|
||||
async def trigger_membership_sync():
|
||||
"""Run the roster sweep now.
|
||||
|
||||
The beat schedule runs daily, which is right for a billing-cycle fact but
|
||||
far too slow when the operator has just connected a credential and wants to
|
||||
see whether it works. Queued rather than run inline: it crosses the network
|
||||
to an external service and the request path is not where that belongs.
|
||||
"""
|
||||
from ..tasks.maintenance import sync_memberships
|
||||
|
||||
sync_memberships.delay()
|
||||
return jsonify({"queued": True})
|
||||
|
||||
|
||||
# --- #388 E4: creator/membership suggestions ------------------------------
|
||||
#
|
||||
# Confirm-only. Accepting ADDS A SOURCE under the existing artist — it never
|
||||
# merges two artists, because adding a source is trivially undone and a wrong
|
||||
# merge silently mixes two creators' work with nothing left to separate them by.
|
||||
|
||||
|
||||
@sources_bp.route("/membership-suggestions", methods=["GET"])
|
||||
async def list_membership_suggestions():
|
||||
async with get_session() as session:
|
||||
return jsonify({"items": await ArtistMembershipService(session).list_pending()})
|
||||
|
||||
|
||||
@sources_bp.route("/membership-suggestions/<int:sid>/accept", methods=["POST"])
|
||||
async def accept_membership_suggestion(sid: int):
|
||||
async with get_session() as session:
|
||||
result = await ArtistMembershipService(session).accept(sid)
|
||||
if result is None:
|
||||
return _bad("suggestion_not_found", status=404)
|
||||
await session.commit()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@sources_bp.route("/membership-suggestions/<int:sid>/dismiss", methods=["POST"])
|
||||
async def dismiss_membership_suggestion(sid: int):
|
||||
async with get_session() as session:
|
||||
result = await ArtistMembershipService(session).dismiss(sid)
|
||||
if result is None:
|
||||
return _bad("suggestion_not_found", status=404)
|
||||
await session.commit()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@sources_bp.route("/membership-suggestions/rescan", methods=["POST"])
|
||||
async def rescan_membership_suggestions():
|
||||
async with get_session() as session:
|
||||
result = await membership_rescan(session)
|
||||
await session.commit()
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
# --- #387 C4: reconciling the roster against what FC actually tracks -------
|
||||
#
|
||||
# Asymmetric on purpose. The "you subscribe but FC doesn't follow it" direction
|
||||
# carries a per-row action, because adding a source is the reversible half. The
|
||||
# "FC follows it but your roster doesn't show it" direction is REPORT ONLY by
|
||||
# the operator's decision (2026-09-11): it says what it sees and links to the
|
||||
# Subscriptions row, and offers no one-click disable.
|
||||
|
||||
|
||||
@sources_bp.route("/reconciliation", methods=["GET"])
|
||||
async def reconciliation():
|
||||
async with get_session() as session:
|
||||
return jsonify(await reconcile_all(session))
|
||||
|
||||
|
||||
@sources_bp.route("/reconciliation/adopt", methods=["POST"])
|
||||
async def adopt_membership():
|
||||
"""Start tracking a creator the roster says the account already pays for.
|
||||
|
||||
One row, one click, never a sweep side effect: adding a source commits disk,
|
||||
worker time and rate budget, and unwinding it means deleting files.
|
||||
"""
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", status=400)
|
||||
membership_id = body.get("membership_id")
|
||||
if not isinstance(membership_id, int):
|
||||
return _bad("membership_id_required", status=400)
|
||||
|
||||
async with get_session() as session:
|
||||
membership = await session.get(PlatformMembership, membership_id)
|
||||
if membership is None:
|
||||
return _bad("membership_not_found", status=404)
|
||||
if not membership.url:
|
||||
return _bad("membership_has_no_url", status=400)
|
||||
|
||||
existing = await source_for_membership(session, membership)
|
||||
if existing is not None:
|
||||
# The operator got there by another route between the page load and
|
||||
# the click. That is them being ahead of us, not an error.
|
||||
return jsonify({"already_tracked": existing.id})
|
||||
|
||||
# The sweep already captured the creator's real display name, so the
|
||||
# artist gets its true name with NO lookup on the request path. Task
|
||||
# #1293 asked for `resolve_display_name` here; the roster satisfies that
|
||||
# concern earlier in the pipeline than #1293 expected, which also keeps
|
||||
# this route off the network entirely (rule 164). The vanity is the
|
||||
# fallback, never the preferred value.
|
||||
name = membership.display_name or membership.vanity_or_none()
|
||||
if not name:
|
||||
return _bad("membership_has_no_name", status=400)
|
||||
|
||||
artist, _created = await ArtistService(session).find_or_create(name)
|
||||
try:
|
||||
record = await SourceService(session).create(
|
||||
artist_id=artist.id,
|
||||
platform=membership.platform,
|
||||
url=membership.url,
|
||||
)
|
||||
except DuplicateSourceError as exc:
|
||||
return jsonify({"already_tracked": exc.existing_id})
|
||||
artist_id = artist.id
|
||||
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
||||
|
||||
@@ -11,22 +11,8 @@ suggestions_bp = Blueprint("suggestions", __name__, url_prefix="/api")
|
||||
|
||||
@suggestions_bp.route("/images/<int:image_id>/suggestions", methods=["GET"])
|
||||
async def get_suggestions(image_id: int):
|
||||
# ?min=<float> overrides the per-head suggest thresholds for INCLUSION. The
|
||||
# rail sends min=0 in its single per-image fetch to get EVERY head (each row
|
||||
# still carries above_threshold vs its natural cut), then derives the panel
|
||||
# (above_threshold) and the typed dropdown (all, filtered by text) client-side
|
||||
# — no second request. Omitted → only above-threshold rows.
|
||||
override = None
|
||||
raw_min = request.args.get("min")
|
||||
if raw_min is not None:
|
||||
try:
|
||||
override = min(1.0, max(0.0, float(raw_min)))
|
||||
except ValueError:
|
||||
return jsonify({"error": "min must be a float in [0,1]"}), 400
|
||||
async with get_session() as session:
|
||||
sl = await SuggestionService(session).for_image(
|
||||
image_id, threshold_override=override
|
||||
)
|
||||
sl = await SuggestionService(session).for_image(image_id)
|
||||
return jsonify(
|
||||
{
|
||||
"by_category": {
|
||||
@@ -37,19 +23,7 @@ async def get_suggestions(image_id: int):
|
||||
"category": s.category,
|
||||
"score": round(s.score, 4),
|
||||
"source": s.source,
|
||||
# whether the score cleared the head's own suggest cut.
|
||||
# The single min=0 fetch returns every head; the panel
|
||||
# shows above_threshold, the typed dropdown shows all and
|
||||
# annotates each match with its score.
|
||||
"above_threshold": s.above_threshold,
|
||||
# operator dismissed this tag for this image — surfaced
|
||||
# (not dropped) so the rail can show it rejected + offer
|
||||
# one-click un-reject.
|
||||
"rejected": s.rejected,
|
||||
# the crop region that produced this tag (#1206) —
|
||||
# {bbox,kind,detector} or null (whole-image won). Drives
|
||||
# the hover→overlay highlight.
|
||||
"grounding": s.grounding,
|
||||
"creates_new_tag": s.creates_new_tag,
|
||||
}
|
||||
for s in items
|
||||
]
|
||||
@@ -68,9 +42,36 @@ async def accept_suggestion(image_id: int):
|
||||
return jsonify({"error": "tag_id required"}), 400
|
||||
tag_id = body["tag_id"]
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).accept(image_id, tag_id)
|
||||
newly_added = await AllowlistService(session).accept(image_id, tag_id)
|
||||
await session.commit()
|
||||
return jsonify({"accepted": True, "tag_id": tag_id})
|
||||
if newly_added:
|
||||
from ..tasks.ml import apply_allowlist_tags
|
||||
|
||||
apply_allowlist_tags.delay(tag_id=tag_id)
|
||||
return "", 204
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
"/images/<int:image_id>/suggestions/alias", methods=["POST"]
|
||||
)
|
||||
async def alias_suggestion(image_id: int):
|
||||
body = await request.get_json()
|
||||
required = {"alias_string", "alias_category", "canonical_tag_id"}
|
||||
if not body or not required.issubset(body):
|
||||
return jsonify({"error": f"required: {sorted(required)}"}), 400
|
||||
async with get_session() as session:
|
||||
newly_added = await AllowlistService(session).add_alias_and_accept(
|
||||
image_id,
|
||||
body["alias_string"],
|
||||
body["alias_category"],
|
||||
body["canonical_tag_id"],
|
||||
)
|
||||
await session.commit()
|
||||
if newly_added:
|
||||
from ..tasks.ml import apply_allowlist_tags
|
||||
|
||||
apply_allowlist_tags.delay(tag_id=body["canonical_tag_id"])
|
||||
return "", 204
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
@@ -86,21 +87,6 @@ async def dismiss_suggestion(image_id: int):
|
||||
return "", 204
|
||||
|
||||
|
||||
@suggestions_bp.route(
|
||||
"/images/<int:image_id>/suggestions/undismiss", methods=["POST"]
|
||||
)
|
||||
async def undismiss_suggestion(image_id: int):
|
||||
"""Reverse a per-image dismissal (reject-recovery). Idempotent — undoing a
|
||||
tag that isn't rejected is a no-op delete."""
|
||||
body = await request.get_json()
|
||||
if not body or "tag_id" not in body:
|
||||
return jsonify({"error": "tag_id required"}), 400
|
||||
async with get_session() as session:
|
||||
await AllowlistService(session).undismiss(image_id, body["tag_id"])
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@suggestions_bp.route("/suggestions/bulk", methods=["POST"])
|
||||
async def bulk_suggestions():
|
||||
body = await request.get_json()
|
||||
|
||||
@@ -1,261 +0,0 @@
|
||||
"""FC-3i: system activity dashboard endpoints.
|
||||
|
||||
Read-only. Combines Redis-broker queue depths (LLEN per queue),
|
||||
Celery worker introspection (celery inspect), and the task_run DB
|
||||
history into the surfaces the SystemActivityTab UI consumes.
|
||||
|
||||
All filesystem/sync-client work goes through asyncio.to_thread per
|
||||
ASYNC230/240 (mirrors backend.app.api.extension's pattern).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import desc, func, select
|
||||
|
||||
from ..config import get_config
|
||||
from ..extensions import get_session
|
||||
from ..models import TaskRun
|
||||
from ..services.scheduler_service import scheduler_status
|
||||
from ..services.worker_lanes import LANES
|
||||
|
||||
system_activity_bp = Blueprint(
|
||||
"system_activity", __name__, url_prefix="/api/system/activity",
|
||||
)
|
||||
|
||||
# Every queue, grouped by the lane that consumes it. DERIVED from
|
||||
# `worker_lanes.LANES` (milestone 422 step 1) rather than written out:
|
||||
# this was a hand-kept third copy of "which queues exist", alongside
|
||||
# celery_app.task_routes and service_roster.ROLE_NAMES, and its own comment
|
||||
# admitted the coupling — "must match celery_app.task_routes".
|
||||
#
|
||||
# The rendered ORDER changes with this: lane order rather than the previous
|
||||
# hand-chosen one. That is the better grouping for a lane-oriented UI, and
|
||||
# queues with no LLEN response still show as null rather than absent.
|
||||
_QUEUE_NAMES = tuple(q for lane in LANES for q in lane.queues)
|
||||
|
||||
# Cache module-level so all requests share the cache between polls.
|
||||
# Tests can reset via direct dict mutation if needed.
|
||||
_QUEUE_CACHE: dict = {"ts": 0.0, "data": None}
|
||||
_WORKER_CACHE: dict = {"ts": 0.0, "data": None}
|
||||
_QUEUE_CACHE_TTL = 2.0
|
||||
_WORKER_CACHE_TTL = 5.0
|
||||
|
||||
|
||||
def _read_queues_sync() -> dict:
|
||||
"""Reads each queue's LLEN from the broker. Sync — caller wraps in
|
||||
asyncio.to_thread. Per-queue try/except returns None on failure so
|
||||
one bad queue doesn't break the whole response."""
|
||||
import redis # local import; only this endpoint needs it
|
||||
|
||||
cfg = get_config()
|
||||
client = redis.Redis.from_url(cfg.celery_broker_url)
|
||||
out: dict = {}
|
||||
for name in _QUEUE_NAMES:
|
||||
try:
|
||||
out[name] = int(client.llen(name))
|
||||
except Exception: # noqa: BLE001 — broker hiccup shouldn't break UI
|
||||
out[name] = None
|
||||
return {
|
||||
"queues": out,
|
||||
"fetched_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _read_workers_sync() -> dict:
|
||||
"""celery inspect active_queues + active. Returns per-worker info."""
|
||||
from ..celery_app import celery as celery_app
|
||||
|
||||
insp = celery_app.control.inspect(timeout=2.0)
|
||||
active_queues = insp.active_queues() or {}
|
||||
active_tasks = insp.active() or {}
|
||||
|
||||
workers: dict = {}
|
||||
for hostname, queues in active_queues.items():
|
||||
workers[hostname] = {
|
||||
"queues": sorted({q["name"] for q in queues}),
|
||||
"active_count": len(active_tasks.get(hostname, [])),
|
||||
}
|
||||
return {
|
||||
"workers": workers,
|
||||
"fetched_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
async def _queues_cached() -> dict:
|
||||
"""Per-queue Redis LLEN, cached 2s. Shared by /queues and /summary."""
|
||||
now = time.time()
|
||||
if _QUEUE_CACHE["data"] is None or (now - _QUEUE_CACHE["ts"]) > _QUEUE_CACHE_TTL:
|
||||
_QUEUE_CACHE["data"] = await asyncio.to_thread(_read_queues_sync)
|
||||
_QUEUE_CACHE["ts"] = now
|
||||
return _QUEUE_CACHE["data"]
|
||||
|
||||
|
||||
@system_activity_bp.route("/queues", methods=["GET"])
|
||||
async def get_queues():
|
||||
"""Per-queue Redis LLEN. Cached 2s.
|
||||
|
||||
Response: {queues: {name: depth_or_null}, fetched_at: iso8601}
|
||||
"""
|
||||
return jsonify(await _queues_cached())
|
||||
|
||||
|
||||
@system_activity_bp.route("/workers", methods=["GET"])
|
||||
async def get_workers():
|
||||
"""Live celery inspect. Cached 5s.
|
||||
|
||||
Response: {workers: {hostname: {queues, active_count}}, fetched_at}
|
||||
"""
|
||||
now = time.time()
|
||||
if _WORKER_CACHE["data"] is None or (now - _WORKER_CACHE["ts"]) > _WORKER_CACHE_TTL:
|
||||
_WORKER_CACHE["data"] = await asyncio.to_thread(_read_workers_sync)
|
||||
_WORKER_CACHE["ts"] = now
|
||||
return jsonify(_WORKER_CACHE["data"])
|
||||
|
||||
|
||||
@system_activity_bp.route("/summary", methods=["GET"])
|
||||
async def get_summary():
|
||||
"""One-call rollup for the always-on TopNav pipeline indicator:
|
||||
scheduler health, per-queue pending depths, currently-running count, and
|
||||
recent (24h) failure count. Cheap — cached queue LLENs + two TaskRun
|
||||
counts — so it's safe to poll app-wide."""
|
||||
queues_data = await _queues_cached()
|
||||
depths = queues_data.get("queues", {})
|
||||
queued_total = sum(v for v in depths.values() if isinstance(v, int))
|
||||
since = datetime.now(UTC) - timedelta(hours=24)
|
||||
async with get_session() as session:
|
||||
scheduler = await scheduler_status(session)
|
||||
running = (await session.execute(
|
||||
select(func.count(TaskRun.id)).where(TaskRun.status == "running")
|
||||
)).scalar_one()
|
||||
failing = (await session.execute(
|
||||
select(func.count(TaskRun.id))
|
||||
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||
.where(TaskRun.finished_at >= since)
|
||||
)).scalar_one()
|
||||
return jsonify({
|
||||
"scheduler": scheduler,
|
||||
"queues": depths,
|
||||
"queued_total": queued_total,
|
||||
"running": int(running),
|
||||
"failing": int(failing),
|
||||
})
|
||||
|
||||
|
||||
@system_activity_bp.route("/runs", methods=["GET"])
|
||||
async def list_runs():
|
||||
"""Paginated task_run history. Query params:
|
||||
queue=<name> filter to one queue
|
||||
status=<status> filter to one status (running/ok/error/timeout/retry)
|
||||
task=<substr> case-insensitive substring match on task_name
|
||||
limit=<int> default 50, max 200
|
||||
before_id=<int> cursor for keyset pagination
|
||||
|
||||
Response: {runs: [...], next_cursor: id|null}
|
||||
"""
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "50")), 200)
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
if limit < 1:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
|
||||
queue = request.args.get("queue")
|
||||
status = request.args.get("status")
|
||||
task = request.args.get("task")
|
||||
before_id_raw = request.args.get("before_id")
|
||||
before_id = int(before_id_raw) if before_id_raw else None
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = select(TaskRun).order_by(desc(TaskRun.id))
|
||||
if queue:
|
||||
stmt = stmt.where(TaskRun.queue == queue)
|
||||
if status:
|
||||
stmt = stmt.where(TaskRun.status == status)
|
||||
if task:
|
||||
# Task names contain literal underscores (download_source,
|
||||
# vacuum_analyze) — escape LIKE wildcards so a search for
|
||||
# "vacuum_analyze" doesn't treat "_" as a single-char match.
|
||||
stmt = stmt.where(TaskRun.task_name.ilike(f"%{_escape_like(task)}%", escape="\\"))
|
||||
if before_id is not None:
|
||||
stmt = stmt.where(TaskRun.id < before_id)
|
||||
stmt = stmt.limit(limit + 1)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return jsonify({
|
||||
"runs": [_row_to_dict(r) for r in rows],
|
||||
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||
})
|
||||
|
||||
|
||||
@system_activity_bp.route("/failures", methods=["GET"])
|
||||
async def list_failures():
|
||||
"""Recent failures across all lanes (24h window).
|
||||
|
||||
Response: {recent: [...], count_by_type: {ErrorClass: n}, since}
|
||||
"""
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "50")), 200)
|
||||
except ValueError:
|
||||
return jsonify({"error": "invalid_limit"}), 400
|
||||
|
||||
since = datetime.now(UTC) - timedelta(hours=24)
|
||||
|
||||
async with get_session() as session:
|
||||
recent_stmt = (
|
||||
select(TaskRun)
|
||||
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||
.where(TaskRun.finished_at >= since)
|
||||
.order_by(desc(TaskRun.finished_at))
|
||||
.limit(limit)
|
||||
)
|
||||
recent = (await session.execute(recent_stmt)).scalars().all()
|
||||
|
||||
count_stmt = (
|
||||
select(TaskRun.error_type, func.count(TaskRun.id))
|
||||
.where(TaskRun.status.in_(["error", "timeout"]))
|
||||
.where(TaskRun.finished_at >= since)
|
||||
.group_by(TaskRun.error_type)
|
||||
.order_by(desc(func.count(TaskRun.id)))
|
||||
)
|
||||
counts = (await session.execute(count_stmt)).all()
|
||||
|
||||
return jsonify({
|
||||
"recent": [_row_to_dict(r) for r in recent],
|
||||
"count_by_type": {
|
||||
(row[0] or "Unknown"): row[1]
|
||||
for row in counts
|
||||
},
|
||||
"since": since.isoformat(),
|
||||
})
|
||||
|
||||
|
||||
def _escape_like(value: str) -> str:
|
||||
"""Escape SQL LIKE/ILIKE metacharacters so user search text is matched
|
||||
literally. Pairs with `escape="\\"` on the .ilike() call."""
|
||||
return value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
def _row_to_dict(r: TaskRun) -> dict:
|
||||
return {
|
||||
"id": r.id,
|
||||
"queue": r.queue,
|
||||
"task_name": r.task_name,
|
||||
"target_id": r.target_id,
|
||||
"celery_task_id": r.celery_task_id,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"duration_ms": r.duration_ms,
|
||||
"status": r.status,
|
||||
"error_type": r.error_type,
|
||||
"error_message": r.error_message,
|
||||
"retry_count": r.retry_count,
|
||||
"worker_hostname": r.worker_hostname,
|
||||
"args_summary": r.args_summary,
|
||||
}
|
||||
@@ -1,255 +0,0 @@
|
||||
"""FC-3h: /api/system/backup — create/list/restore/delete/tag for
|
||||
DB + image backups.
|
||||
|
||||
Read endpoints are public on FC (operator-facing internal API; same
|
||||
posture as /api/system/activity). Write endpoints take a typed
|
||||
`confirm` body field that must match a server-generated token for
|
||||
that backup row, to prevent click-to-destroy by stale browser tabs
|
||||
or accidental cURL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import desc, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import BackupRun, ImportSettings
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
system_backup_bp = Blueprint(
|
||||
"system_backup", __name__, url_prefix="/api/system/backup",
|
||||
)
|
||||
|
||||
_KINDS = frozenset({"db", "images"})
|
||||
_TAG_MAX_LEN = 64
|
||||
_BACKUP_SETTINGS_FIELDS = (
|
||||
"backup_db_nightly_enabled",
|
||||
"backup_db_nightly_hour_utc",
|
||||
"backup_db_keep_last_n",
|
||||
"backup_images_keep_last_n",
|
||||
)
|
||||
|
||||
|
||||
def _row_to_dict(r: BackupRun) -> dict:
|
||||
return {
|
||||
"id": r.id,
|
||||
"kind": r.kind,
|
||||
"status": r.status,
|
||||
"tag": r.tag,
|
||||
"triggered_by": r.triggered_by,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"duration_seconds": (
|
||||
int((r.finished_at - r.started_at).total_seconds())
|
||||
if r.finished_at and r.started_at else None
|
||||
),
|
||||
"sql_path": r.sql_path,
|
||||
"tar_path": r.tar_path,
|
||||
"size_bytes": r.size_bytes,
|
||||
"error": r.error,
|
||||
"restored_from_id": r.restored_from_id,
|
||||
"manifest": r.manifest or {},
|
||||
}
|
||||
|
||||
|
||||
def _validate_tag(tag):
|
||||
if tag is None:
|
||||
return None
|
||||
if not isinstance(tag, str):
|
||||
return _bad("invalid_tag", detail="tag must be string or null")
|
||||
tag = tag.strip()
|
||||
if not tag:
|
||||
return None
|
||||
if len(tag) > _TAG_MAX_LEN:
|
||||
return _bad("invalid_tag", detail=f"tag too long (max {_TAG_MAX_LEN})")
|
||||
return tag
|
||||
|
||||
|
||||
def _validate_backup_settings_patch(body: dict):
|
||||
if "backup_db_nightly_enabled" in body and not isinstance(
|
||||
body["backup_db_nightly_enabled"], bool,
|
||||
):
|
||||
return _bad("invalid_value", detail="backup_db_nightly_enabled must be bool")
|
||||
if "backup_db_nightly_hour_utc" in body:
|
||||
v = body["backup_db_nightly_hour_utc"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (0 <= v <= 23):
|
||||
return _bad("invalid_value", detail="backup_db_nightly_hour_utc must be 0..23")
|
||||
if "backup_db_keep_last_n" in body:
|
||||
v = body["backup_db_keep_last_n"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 365):
|
||||
return _bad("invalid_value", detail="backup_db_keep_last_n must be 1..365")
|
||||
if "backup_images_keep_last_n" in body:
|
||||
v = body["backup_images_keep_last_n"]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or not (1 <= v <= 100):
|
||||
return _bad("invalid_value", detail="backup_images_keep_last_n must be 1..100")
|
||||
return None
|
||||
|
||||
|
||||
@system_backup_bp.route("/db", methods=["POST"])
|
||||
async def trigger_db_backup():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
tag = _validate_tag(body.get("tag"))
|
||||
if isinstance(tag, tuple):
|
||||
return tag
|
||||
from ..tasks.backup import backup_db_task
|
||||
backup_db_task.delay(tag=tag, triggered_by="manual")
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/images", methods=["POST"])
|
||||
async def trigger_images_backup():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
tag = _validate_tag(body.get("tag"))
|
||||
if isinstance(tag, tuple):
|
||||
return tag
|
||||
from ..tasks.backup import backup_images_task
|
||||
backup_images_task.delay(tag=tag, triggered_by="manual")
|
||||
return jsonify({"status": "dispatched"}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs", methods=["GET"])
|
||||
async def list_runs():
|
||||
try:
|
||||
limit = min(int(request.args.get("limit", "50")), 200)
|
||||
except ValueError:
|
||||
return _bad("invalid_limit")
|
||||
if limit < 1:
|
||||
return _bad("invalid_limit")
|
||||
kind = request.args.get("kind")
|
||||
if kind is not None and kind not in _KINDS:
|
||||
return _bad("invalid_kind", detail=f"kind must be one of {sorted(_KINDS)}")
|
||||
before_id_raw = request.args.get("before_id")
|
||||
before_id = int(before_id_raw) if before_id_raw else None
|
||||
|
||||
async with get_session() as session:
|
||||
stmt = select(BackupRun).order_by(desc(BackupRun.id))
|
||||
if kind:
|
||||
stmt = stmt.where(BackupRun.kind == kind)
|
||||
if before_id is not None:
|
||||
stmt = stmt.where(BackupRun.id < before_id)
|
||||
stmt = stmt.limit(limit + 1)
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
return jsonify({
|
||||
"runs": [_row_to_dict(r) for r in rows],
|
||||
"next_cursor": rows[-1].id if has_more and rows else None,
|
||||
})
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>", methods=["GET"])
|
||||
async def get_run(run_id: int):
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
return jsonify(_row_to_dict(row))
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>", methods=["PATCH"])
|
||||
async def patch_run(run_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
if "tag" not in body:
|
||||
return _bad("invalid_body", detail="tag required")
|
||||
tag = _validate_tag(body["tag"])
|
||||
if isinstance(tag, tuple):
|
||||
return tag
|
||||
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
row.tag = tag
|
||||
await session.commit()
|
||||
await session.refresh(row)
|
||||
return jsonify(_row_to_dict(row))
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>/restore", methods=["POST"])
|
||||
async def trigger_restore(run_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
supplied = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
if row.status != "ok":
|
||||
return _bad(
|
||||
"not_restorable",
|
||||
detail=f"source backup status={row.status!r}; only 'ok' rows are restorable",
|
||||
)
|
||||
expected = f"restore-{row.kind}-{row.id}"
|
||||
if supplied != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
kind = row.kind
|
||||
|
||||
if kind == "db":
|
||||
from ..tasks.backup import restore_db_task
|
||||
restore_db_task.delay(source_backup_run_id=run_id)
|
||||
else: # 'images' (the only other value _KINDS allows via the trigger path)
|
||||
from ..tasks.backup import restore_images_task
|
||||
restore_images_task.delay(source_backup_run_id=run_id)
|
||||
return jsonify({"status": "dispatched", "kind": kind}), 202
|
||||
|
||||
|
||||
@system_backup_bp.route("/runs/<int:run_id>", methods=["DELETE"])
|
||||
async def delete_run(run_id: int):
|
||||
body = await request.get_json(silent=True) or {}
|
||||
supplied = body.get("confirm", "")
|
||||
|
||||
async with get_session() as session:
|
||||
row = await session.get(BackupRun, run_id)
|
||||
if row is None:
|
||||
return _bad("not_found", status=404)
|
||||
expected = f"delete-{row.kind}-{row.id}"
|
||||
if supplied != expected:
|
||||
return _bad(
|
||||
"confirm_mismatch",
|
||||
detail=f"confirm must equal {expected!r}",
|
||||
expected=expected,
|
||||
)
|
||||
from ..services import backup_service
|
||||
backup_service.unlink_artifact_files(
|
||||
sql_path=row.sql_path, tar_path=row.tar_path,
|
||||
manifest_path=(row.manifest or {}).get("manifest_path"),
|
||||
)
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@system_backup_bp.route("/settings", methods=["GET"])
|
||||
async def get_settings():
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
return jsonify({
|
||||
"backup_db_nightly_enabled": row.backup_db_nightly_enabled,
|
||||
"backup_db_nightly_hour_utc": row.backup_db_nightly_hour_utc,
|
||||
"backup_db_keep_last_n": row.backup_db_keep_last_n,
|
||||
"backup_images_keep_last_n": row.backup_images_keep_last_n,
|
||||
})
|
||||
|
||||
|
||||
@system_backup_bp.route("/settings", methods=["PATCH"])
|
||||
async def patch_settings():
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
|
||||
err = _validate_backup_settings_patch(body)
|
||||
if err is not None:
|
||||
return err
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
for field in _BACKUP_SETTINGS_FIELDS:
|
||||
if field in body:
|
||||
setattr(row, field, body[field])
|
||||
await session.commit()
|
||||
return await get_settings()
|
||||
@@ -1,243 +0,0 @@
|
||||
"""Is every part of FabledCurator running? One verdict, one endpoint.
|
||||
|
||||
Milestone 365. The nav indicator and the System page both read this and
|
||||
nothing else — composing a verdict is this module's job, not the UI's.
|
||||
|
||||
## Two kinds of part, answered two different ways
|
||||
|
||||
**Learned** — celery roles and the GPU agent, from `service_seen`. The
|
||||
question is "how long since it checked in", and these are the parts that can
|
||||
be ABSENT, which is the whole point: `celery inspect` alone reports presence,
|
||||
so a dead worker is a shorter list rather than a red light.
|
||||
|
||||
**Probed live** — Postgres and Redis. Always expected, never learned, and a
|
||||
last-seen for them would be actively misleading: that Redis answered thirty
|
||||
seconds ago says nothing about now.
|
||||
|
||||
## This endpoint must never fail because something it checks has failed
|
||||
|
||||
The inversion is easy to write by accident and it destroys the feature exactly
|
||||
when it is needed — a 500 when Redis is down, instead of `redis: down`. Every
|
||||
probe is wrapped, every wait has a deadline (rule 156), and the roster refresh
|
||||
swallows its own errors. The worst case is a part reported `unknown`, which is
|
||||
a true statement.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from quart import Blueprint, jsonify
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from ..config import get_config
|
||||
from ..extensions import get_session
|
||||
from ..models import ServiceSeen
|
||||
from ..services.worker_lanes import SWEEP_PERIOD_SECONDS
|
||||
|
||||
system_health_bp = Blueprint("system_health", __name__, url_prefix="/api/system")
|
||||
|
||||
# How long a learned part may go quiet before it is doubted, then disbelieved.
|
||||
#
|
||||
# These are deliberately generous, and the reason is a deploy rather than a
|
||||
# worker: `docker compose up -d` rolls start-first, so a role is briefly served
|
||||
# by two containers and then by neither while the old one drains. Thresholds
|
||||
# tight enough to catch a crash in seconds would paint the page red every time
|
||||
# the stack is updated, and an alarm that cries wolf on every deploy is one
|
||||
# nobody reads. Tune down only after watching a real deploy pass through.
|
||||
STALE_AFTER_SECONDS = 90
|
||||
DOWN_AFTER_SECONDS = 300
|
||||
|
||||
# The celery roster is written by `size_worker_lanes` and by nothing else, so
|
||||
# these thresholds are only meaningful against ITS cadence. Asserted at import
|
||||
# rather than left to a reader, because this is precisely the comparison that
|
||||
# was never made for the GPU agent: its lease poll backed off to 900s while
|
||||
# the roster called it stopped at 300s, and both numbers were individually
|
||||
# correct, in different directions, in different files (lesson #4355).
|
||||
#
|
||||
# Two clear sweeps before a part is even called STALE. One missed tick is
|
||||
# routine — the sweep rides the maintenance queue and does an inspect that can
|
||||
# take eleven seconds — and must not turn the page yellow.
|
||||
_SWEEPS_BEFORE_STALE = 2
|
||||
assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, (
|
||||
f"a {SWEEP_PERIOD_SECONDS}s sweep cannot keep a roster fresh against a "
|
||||
f"{STALE_AFTER_SECONDS}s stale threshold: raise the threshold or shorten "
|
||||
f"the sweep"
|
||||
)
|
||||
|
||||
# Probes cross a process boundary, so they carry deadlines. A hung Postgres
|
||||
# must make this endpoint say "postgres: down", not hang alongside it.
|
||||
PROBE_TIMEOUT_SECONDS = 2.0
|
||||
|
||||
_OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
|
||||
# Checking in, but working at a fraction of its speed: a GPU agent whose
|
||||
# runtimes fell back to the CPU (#4410). Below stale — a part that may have
|
||||
# stopped is the more urgent question — and above unknown, because this one
|
||||
# IS known to be wrong.
|
||||
_DEGRADED = "degraded"
|
||||
|
||||
# Worst-first, so an overall verdict is just the max.
|
||||
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4}
|
||||
|
||||
|
||||
def _age_state(age_seconds: float) -> str:
|
||||
if age_seconds >= DOWN_AFTER_SECONDS:
|
||||
return _DOWN
|
||||
if age_seconds >= STALE_AFTER_SECONDS:
|
||||
return _STALE
|
||||
return _OK
|
||||
|
||||
|
||||
def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
|
||||
"""Say what the state MEANS. A red chip tells an operator less than a
|
||||
sentence does at the moment they are deciding whether to go and look."""
|
||||
if state == _OK:
|
||||
replicas = details.get("replicas")
|
||||
if replicas and replicas > 1:
|
||||
return f"{name} is running ({replicas} replicas)"
|
||||
return f"{name} is running"
|
||||
mins = int(age // 60)
|
||||
ago = f"{mins} min" if mins else f"{int(age)}s"
|
||||
if state == _STALE:
|
||||
return f"{name} has not checked in for {ago}"
|
||||
return f"{name} has not checked in for {ago} — treat it as stopped"
|
||||
|
||||
|
||||
def _cpu_runtimes(details: dict) -> list[str]:
|
||||
"""The runtimes an agent reported as NOT on the GPU, with why.
|
||||
|
||||
Both torch and onnxruntime fall back to the CPU without raising, so an
|
||||
agent in that state leases, works and checks in exactly like a healthy
|
||||
one. On 2026-09-24 one had been doing so since a driver update left a
|
||||
stale CDI spec; the only sign was a line in the agent's own log.
|
||||
"""
|
||||
accel = details.get("accel")
|
||||
if not isinstance(accel, dict):
|
||||
return []
|
||||
out = []
|
||||
for name, entry in sorted(accel.items()):
|
||||
if not isinstance(entry, dict) or entry.get("device") == "cuda":
|
||||
continue
|
||||
why = entry.get("error") or entry.get("device") or "unknown"
|
||||
out.append(f"{name} ({why})")
|
||||
return out
|
||||
|
||||
|
||||
def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]:
|
||||
"""A roster row's state and its sentence, degraded included."""
|
||||
if state == _OK:
|
||||
cpu = _cpu_runtimes(details)
|
||||
if cpu:
|
||||
return _DEGRADED, (
|
||||
f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. "
|
||||
"After a driver update, regenerate the agent host's CDI spec "
|
||||
"(agent README)."
|
||||
)
|
||||
return state, _describe_learned(name, state, age, details)
|
||||
|
||||
|
||||
async def _probe_postgres(session) -> dict:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
session.execute(text("SELECT 1")), timeout=PROBE_TIMEOUT_SECONDS
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — a probe reports, it never raises
|
||||
return {
|
||||
"key": "postgres", "kind": "datastore", "name": "PostgreSQL",
|
||||
"state": _DOWN, "detail": f"not answering: {type(exc).__name__}",
|
||||
}
|
||||
return {
|
||||
"key": "postgres", "kind": "datastore", "name": "PostgreSQL", "state": _OK,
|
||||
"detail": "answering", "latency_ms": round((time.monotonic() - started) * 1000, 1),
|
||||
}
|
||||
|
||||
|
||||
def _ping_redis_sync() -> None:
|
||||
import redis # local import; mirrors system_activity's pattern
|
||||
|
||||
client = redis.Redis.from_url(
|
||||
get_config().celery_broker_url,
|
||||
socket_connect_timeout=PROBE_TIMEOUT_SECONDS,
|
||||
socket_timeout=PROBE_TIMEOUT_SECONDS,
|
||||
)
|
||||
client.ping()
|
||||
|
||||
|
||||
async def _probe_redis() -> dict:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.to_thread(_ping_redis_sync), timeout=PROBE_TIMEOUT_SECONDS * 2
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {
|
||||
"key": "redis", "kind": "datastore", "name": "Redis",
|
||||
"state": _DOWN,
|
||||
"detail": f"not answering: {type(exc).__name__} — queues and workers "
|
||||
f"cannot be reached either",
|
||||
}
|
||||
return {
|
||||
"key": "redis", "kind": "datastore", "name": "Redis", "state": _OK,
|
||||
"detail": "answering", "latency_ms": round((time.monotonic() - started) * 1000, 1),
|
||||
}
|
||||
|
||||
|
||||
@system_health_bp.route("/health", methods=["GET"])
|
||||
async def system_health():
|
||||
"""Every part, its state, and one overall verdict.
|
||||
|
||||
Response: {overall, parts: [{key, kind, name, state, detail, last_seen_at,
|
||||
…}], checked_at}
|
||||
"""
|
||||
parts: list[dict] = []
|
||||
now = datetime.now(UTC)
|
||||
|
||||
async with get_session() as session:
|
||||
# Postgres first, and if it is unreachable nothing else can be read —
|
||||
# say so rather than failing, because "the database is down" is the
|
||||
# single most useful thing this endpoint can ever report.
|
||||
pg = await _probe_postgres(session)
|
||||
parts.append(pg)
|
||||
|
||||
if pg["state"] == _OK:
|
||||
# A PURE READ since 2026-09-23. This used to refresh the celery
|
||||
# roster here, rate-limited to once per 20s — so the roster only
|
||||
# advanced while somebody had a browser open, and a broadcast rode
|
||||
# on a request. `size_worker_lanes` writes it now, on a timer, and
|
||||
# the assertion below is what keeps that cadence honest.
|
||||
rows = (
|
||||
await session.execute(select(ServiceSeen).order_by(ServiceSeen.display_name))
|
||||
).scalars().all()
|
||||
for row in rows:
|
||||
age = (now - row.last_seen_at).total_seconds()
|
||||
state, detail = _learned_state(
|
||||
row.display_name, _age_state(age), age, row.details or {},
|
||||
)
|
||||
parts.append({
|
||||
"key": row.key,
|
||||
"kind": row.kind,
|
||||
"name": row.display_name,
|
||||
"state": state,
|
||||
"detail": detail,
|
||||
"last_seen_at": row.last_seen_at.isoformat(),
|
||||
"first_seen_at": row.first_seen_at.isoformat(),
|
||||
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
||||
})
|
||||
|
||||
parts.append(await _probe_redis())
|
||||
|
||||
overall = max((p["state"] for p in parts), key=lambda s: _SEVERITY[s], default=_UNKNOWN)
|
||||
return jsonify({
|
||||
"overall": overall,
|
||||
"parts": sorted(parts, key=lambda p: (-_SEVERITY[p["state"]], p["name"])),
|
||||
"checked_at": now.isoformat(),
|
||||
# So the UI can explain a `stale` without hard-coding the same numbers
|
||||
# in a second place.
|
||||
"thresholds": {
|
||||
"stale_after_seconds": STALE_AFTER_SECONDS,
|
||||
"down_after_seconds": DOWN_AFTER_SECONDS,
|
||||
},
|
||||
})
|
||||
+50
-493
@@ -1,28 +1,20 @@
|
||||
"""Tags API: autocomplete, create, list/add/remove for an image."""
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy import exists, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import Tag, TagHead, TagKind, TagPositiveConfirmation
|
||||
from ..models.tag import image_tag
|
||||
from ..models.tag_suggestion_rejection import TagSuggestionRejection
|
||||
from ..models import Tag, TagKind
|
||||
from ..models.tag_allowlist import TagAllowlist
|
||||
from ..services.bulk_tag_service import BulkTagService
|
||||
from ..services.ml.aliases import AliasService
|
||||
from ..services.ml.heads import ground_applied_tag
|
||||
from ..services.series_match_service import SeriesMatchService
|
||||
from ..services.series_service import SeriesError, SeriesService
|
||||
from ..services.tag_directory_service import TagDirectoryService
|
||||
from ..services.tag_query import serialize_tag
|
||||
from ..services.tag_service import (
|
||||
TagMergeConflict,
|
||||
TagService,
|
||||
TagValidationError,
|
||||
normalize_tag_name,
|
||||
)
|
||||
from ..utils.tag_prefix import parse_kind_prefix
|
||||
|
||||
tags_bp = Blueprint("tags", __name__, url_prefix="/api")
|
||||
|
||||
@@ -63,117 +55,6 @@ def _parse_bulk_ids(
|
||||
return ids, None
|
||||
|
||||
|
||||
# Application-source groupings (image_tag.source). HUMAN = operator signal;
|
||||
# AUTO = machine-applied (heads/CCIP, + legacy Camie ml_auto).
|
||||
_SOURCE_GROUPS = {
|
||||
"human": ("manual", "ml_accepted"),
|
||||
"manual": ("manual",),
|
||||
"accepted": ("ml_accepted",),
|
||||
"auto": ("head_auto", "ccip_auto", "ml_auto"),
|
||||
}
|
||||
|
||||
|
||||
@tags_bp.route("/tags/top", methods=["GET"])
|
||||
async def tags_top():
|
||||
"""Top tags by image count — a fast indexed aggregate for ANALYSIS (not the
|
||||
paged UI directory, which is alphabetical + builds previews). Params:
|
||||
?kind=general|character|fandom|… ?source=all|human|manual|accepted|auto
|
||||
?limit=50 (cap 500) ?min_count=N. → {tags:[{tag_id,name,kind,count}]} desc."""
|
||||
kind = _coerce_kind(request.args.get("kind"))
|
||||
try:
|
||||
limit = min(max(int(request.args.get("limit", "50")), 1), 500)
|
||||
except ValueError:
|
||||
return jsonify({"error": "limit must be an integer"}), 400
|
||||
min_count = None
|
||||
if "min_count" in request.args:
|
||||
try:
|
||||
min_count = int(request.args["min_count"])
|
||||
except ValueError:
|
||||
return jsonify({"error": "min_count must be an integer"}), 400
|
||||
src_vals = _SOURCE_GROUPS.get((request.args.get("source") or "all").lower())
|
||||
|
||||
cnt = func.count(image_tag.c.image_record_id)
|
||||
stmt = (
|
||||
select(Tag.id, Tag.name, Tag.kind, cnt.label("count"))
|
||||
.select_from(Tag)
|
||||
.join(image_tag, image_tag.c.tag_id == Tag.id)
|
||||
.group_by(Tag.id, Tag.name, Tag.kind)
|
||||
.order_by(cnt.desc(), Tag.name.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
if kind is not None:
|
||||
stmt = stmt.where(Tag.kind == kind)
|
||||
if src_vals is not None:
|
||||
stmt = stmt.where(image_tag.c.source.in_(src_vals))
|
||||
if min_count is not None:
|
||||
stmt = stmt.having(cnt >= min_count)
|
||||
async with get_session() as session:
|
||||
rows = (await session.execute(stmt)).all()
|
||||
return jsonify({"tags": [
|
||||
{
|
||||
"tag_id": r.id, "name": r.name,
|
||||
"kind": r.kind.value if hasattr(r.kind, "value") else str(r.kind),
|
||||
"count": r.count,
|
||||
}
|
||||
for r in rows
|
||||
]})
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:tag_id>/stats", methods=["GET"])
|
||||
async def tag_stats(tag_id: int):
|
||||
"""Per-tag dataset health: total + per-source application counts (human vs
|
||||
machine), rejection count, and whether a trained head exists. Read-only,
|
||||
analysis-shaped — backs concept-readiness + source-split decisions."""
|
||||
async with get_session() as session:
|
||||
tag = await session.get(Tag, tag_id)
|
||||
if tag is None:
|
||||
return jsonify({"error": "not found"}), 404
|
||||
by_source = dict(
|
||||
(
|
||||
await session.execute(
|
||||
select(image_tag.c.source, func.count())
|
||||
.where(image_tag.c.tag_id == tag_id)
|
||||
.group_by(image_tag.c.source)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
rejected = (
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(TagSuggestionRejection)
|
||||
.where(TagSuggestionRejection.tag_id == tag_id)
|
||||
)
|
||||
).scalar_one()
|
||||
has_head = (
|
||||
await session.execute(
|
||||
select(func.count())
|
||||
.select_from(TagHead)
|
||||
.where(TagHead.tag_id == tag_id)
|
||||
)
|
||||
).scalar_one() > 0
|
||||
human = by_source.get("manual", 0) + by_source.get("ml_accepted", 0)
|
||||
auto = (
|
||||
by_source.get("head_auto", 0)
|
||||
+ by_source.get("ccip_auto", 0)
|
||||
+ by_source.get("ml_auto", 0)
|
||||
)
|
||||
return jsonify({
|
||||
"tag_id": tag_id,
|
||||
"name": tag.name,
|
||||
"kind": tag.kind.value if hasattr(tag.kind, "value") else str(tag.kind),
|
||||
"count_total": sum(by_source.values()),
|
||||
"count_human": human,
|
||||
"count_manual": by_source.get("manual", 0),
|
||||
"count_accepted": by_source.get("ml_accepted", 0),
|
||||
"count_auto": auto,
|
||||
"count_head_auto": by_source.get("head_auto", 0),
|
||||
"count_ccip_auto": by_source.get("ccip_auto", 0),
|
||||
"count_rejected": rejected,
|
||||
"by_source": by_source,
|
||||
"has_head": has_head,
|
||||
})
|
||||
|
||||
|
||||
@tags_bp.route("/tags/autocomplete", methods=["GET"])
|
||||
async def autocomplete():
|
||||
q = request.args.get("q", "")
|
||||
@@ -188,7 +69,17 @@ async def autocomplete():
|
||||
hits = await svc.autocomplete(q, kind=kind, limit=limit)
|
||||
|
||||
return jsonify(
|
||||
[{**serialize_tag(h), "image_count": h.image_count} for h in hits]
|
||||
[
|
||||
{
|
||||
"id": h.id,
|
||||
"name": h.name,
|
||||
"kind": h.kind,
|
||||
"fandom_id": h.fandom_id,
|
||||
"fandom_name": h.fandom_name,
|
||||
"image_count": h.image_count,
|
||||
}
|
||||
for h in hits
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@@ -214,46 +105,15 @@ async def directory():
|
||||
|
||||
@tags_bp.route("/tags", methods=["POST"])
|
||||
async def create_tag():
|
||||
"""Create a tag. Two input shapes accepted:
|
||||
1. Explicit: {name, kind, fandom_id?} — caller already split, kind wins.
|
||||
2. IR-suffix: {name} where name = "kind:Name" (e.g. "artist:Eric").
|
||||
The server runs parse_kind_prefix(name) to derive kind; the colon
|
||||
and prefix are stripped from the stored tag name. If no recognized
|
||||
prefix is present, the kind defaults to `general`.
|
||||
Explicit kind ALWAYS wins (backward-compat for existing callers).
|
||||
"""
|
||||
body = await request.get_json()
|
||||
if not body or "name" not in body:
|
||||
return jsonify({"error": "name required"}), 400
|
||||
if not body or "name" not in body or "kind" not in body:
|
||||
return jsonify({"error": "name and kind required"}), 400
|
||||
name = body["name"]
|
||||
explicit_kind_raw = body.get("kind")
|
||||
|
||||
if explicit_kind_raw is not None:
|
||||
# Caller provided kind — honor it; don't re-parse.
|
||||
kind = _coerce_kind(explicit_kind_raw)
|
||||
if kind is None:
|
||||
return jsonify({"error": f"invalid kind {explicit_kind_raw!r}"}), 400
|
||||
else:
|
||||
# IR-style: parse "kind:Name" from the raw name.
|
||||
parsed_kind, parsed_name = parse_kind_prefix(name)
|
||||
if parsed_kind is not None:
|
||||
name = parsed_name
|
||||
kind = _coerce_kind(parsed_kind)
|
||||
# parse_kind_prefix only returns kinds from KNOWN_KINDS which
|
||||
# are all valid TagKind members, so _coerce_kind can't return
|
||||
# None here — but defensive.
|
||||
if kind is None:
|
||||
return jsonify({"error": f"invalid kind {parsed_kind!r}"}), 400
|
||||
else:
|
||||
kind = TagKind.general
|
||||
|
||||
kind = _coerce_kind(body["kind"])
|
||||
if kind is None:
|
||||
return jsonify({"error": f"invalid kind {body['kind']!r}"}), 400
|
||||
fandom_id = body.get("fandom_id")
|
||||
|
||||
# #701: Title-Case operator-entered tags. Only here (the explicit create
|
||||
# endpoint), NOT in the shared find_or_create — the ML tagger uses that path
|
||||
# and must keep the booru vocabulary's casing for allowlist matching.
|
||||
name = normalize_tag_name(name)
|
||||
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
try:
|
||||
@@ -271,7 +131,17 @@ async def list_tags_for_image(image_id: int):
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
tags = await svc.list_for_image(image_id)
|
||||
return jsonify([serialize_tag(t) for t in tags])
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"id": t.id,
|
||||
"name": t.name,
|
||||
"kind": t.kind.value,
|
||||
"fandom_id": t.fandom_id,
|
||||
}
|
||||
for t in tags
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@tags_bp.route("/images/<int:image_id>/tags", methods=["POST"])
|
||||
@@ -297,95 +167,15 @@ async def remove_tag_from_image(image_id: int, tag_id: int):
|
||||
return "", 204
|
||||
|
||||
|
||||
@tags_bp.route("/images/<int:image_id>/tags/<int:tag_id>/confirm", methods=["POST"])
|
||||
async def confirm_tag_on_image(image_id: int, tag_id: int):
|
||||
"""Operator affirmed an applied tag is correct ("keep" on a doubted positive).
|
||||
Idempotent; recorded so the eval's doubts list stops resurfacing it (#1130)."""
|
||||
async with get_session() as session:
|
||||
await session.execute(
|
||||
pg_insert(TagPositiveConfirmation)
|
||||
.values(image_record_id=image_id, tag_id=tag_id)
|
||||
.on_conflict_do_nothing(index_elements=["image_record_id", "tag_id"])
|
||||
)
|
||||
await session.commit()
|
||||
return "", 204
|
||||
|
||||
|
||||
@tags_bp.route(
|
||||
"/images/<int:image_id>/tags/<int:tag_id>/grounding", methods=["GET"]
|
||||
)
|
||||
async def tag_grounding(image_id: int, tag_id: int):
|
||||
"""Which crop region best explains an ALREADY-APPLIED tag on this image
|
||||
(#1206 Step 4). Powers the hover→overlay highlight on applied tag chips,
|
||||
mirroring the suggestion rail's live grounding. Computed on demand (applied
|
||||
tags aren't scored live). → {grounding: {bbox,kind,detector}|null,
|
||||
has_head: bool}; has_head False means the tag has no head to localize with,
|
||||
so the chip shows no overlay."""
|
||||
async with get_session() as session:
|
||||
grounding, has_head = await ground_applied_tag(session, image_id, tag_id)
|
||||
return jsonify({"grounding": grounding, "has_head": has_head})
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:tag_id>", methods=["GET"])
|
||||
async def get_tag(tag_id: int):
|
||||
"""Resolve a single tag (used by the gallery to label its active
|
||||
tag-filter chip)."""
|
||||
async with get_session() as session:
|
||||
tag = await session.get(Tag, tag_id)
|
||||
if tag is None:
|
||||
return jsonify({"error": "tag not found"}), 404
|
||||
return jsonify(
|
||||
{
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"kind": tag.kind.value,
|
||||
"fandom_id": tag.fandom_id,
|
||||
"is_system": tag.is_system,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:tag_id>/aliases", methods=["GET"])
|
||||
async def list_tag_aliases(tag_id: int):
|
||||
"""Model keys that fold into this tag (tag-side alias view). Remove via the
|
||||
shared DELETE /api/aliases/<string>/<category>."""
|
||||
async with get_session() as session:
|
||||
if await session.get(Tag, tag_id) is None:
|
||||
return jsonify({"error": "tag not found"}), 404
|
||||
rows = await AliasService(session).list_for_tag(tag_id)
|
||||
return jsonify(
|
||||
[
|
||||
{
|
||||
"alias_string": r.alias_string,
|
||||
"alias_category": r.alias_category,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@tags_bp.route("/tags/<int:tag_id>", methods=["PATCH"])
|
||||
async def update_tag(tag_id: int):
|
||||
"""Rename and/or re-fandom a tag. Body may carry `name` and/or
|
||||
`fandom_id` (a fandom tag id, or null to clear — character tags only).
|
||||
`merge: true` resolves a collision by merging into the existing tag.
|
||||
"""
|
||||
body = await request.get_json() or {}
|
||||
has_name = "name" in body
|
||||
has_fandom = "fandom_id" in body
|
||||
if not has_name and not has_fandom:
|
||||
return jsonify({"error": "name or fandom_id required"}), 400
|
||||
do_merge = bool(body.get("merge"))
|
||||
async def rename_tag(tag_id: int):
|
||||
body = await request.get_json()
|
||||
if not body or "name" not in body:
|
||||
return jsonify({"error": "name required"}), 400
|
||||
async with get_session() as session:
|
||||
svc = TagService(session)
|
||||
try:
|
||||
tag = None
|
||||
if has_name:
|
||||
tag = await svc.rename(tag_id, body["name"])
|
||||
if has_fandom:
|
||||
tag = await svc.set_fandom(
|
||||
tag_id, body["fandom_id"], merge=do_merge
|
||||
)
|
||||
tag = await svc.rename(tag_id, body["name"])
|
||||
except TagMergeConflict as exc:
|
||||
return jsonify(
|
||||
{
|
||||
@@ -402,13 +192,7 @@ async def update_tag(tag_id: int):
|
||||
return jsonify({"error": str(exc)}), 400
|
||||
await session.commit()
|
||||
return jsonify(
|
||||
{
|
||||
"id": tag.id,
|
||||
"name": tag.name,
|
||||
"kind": tag.kind.value,
|
||||
"fandom_id": tag.fandom_id,
|
||||
"is_system": tag.is_system,
|
||||
}
|
||||
{"id": tag.id, "name": tag.name, "kind": tag.kind.value}
|
||||
)
|
||||
|
||||
|
||||
@@ -427,6 +211,13 @@ async def merge_tag(source_id: int):
|
||||
status = 404 if "not found" in msg else 400
|
||||
return jsonify({"error": msg}), status
|
||||
await session.commit()
|
||||
target_allowlisted = await session.scalar(
|
||||
select(exists().where(TagAllowlist.tag_id == result.target_id))
|
||||
)
|
||||
if target_allowlisted:
|
||||
from ..tasks.ml import apply_allowlist_tags
|
||||
|
||||
apply_allowlist_tags.delay(tag_id=result.target_id)
|
||||
return jsonify(
|
||||
{
|
||||
"target": {
|
||||
@@ -502,31 +293,6 @@ def _series_err(exc: SeriesError):
|
||||
return jsonify({"error": msg}), status
|
||||
|
||||
|
||||
def _opt_int(body, key: str):
|
||||
"""(value, error) — value is None when absent, error is (json, status)."""
|
||||
if not body or body.get(key) is None:
|
||||
return None, None
|
||||
try:
|
||||
return int(body[key]), None
|
||||
except (TypeError, ValueError):
|
||||
return None, (jsonify({"error": f"{key} must be an integer"}), 400)
|
||||
|
||||
|
||||
def _parse_int_list(body, key: str, *, max_ids: int = 500):
|
||||
"""(list, error) for a required list of ints under `key`."""
|
||||
if not body or key not in body:
|
||||
return None, (jsonify({"error": f"{key} required"}), 400)
|
||||
raw = body[key]
|
||||
if not isinstance(raw, list) or not raw:
|
||||
return None, (jsonify({"error": f"{key} must be a non-empty list"}), 400)
|
||||
if len(raw) > max_ids:
|
||||
return None, (jsonify({"error": f"too many ids (max {max_ids})"}), 400)
|
||||
try:
|
||||
return [int(x) for x in raw], None
|
||||
except (TypeError, ValueError):
|
||||
return None, (jsonify({"error": f"{key} must be integers"}), 400)
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/pages", methods=["GET"])
|
||||
async def series_pages(tag_id: int):
|
||||
async with get_session() as session:
|
||||
@@ -567,26 +333,15 @@ async def series_remove(tag_id: int):
|
||||
return jsonify({"removed_count": n})
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/pages/number", methods=["POST"])
|
||||
async def series_set_page_number(tag_id: int):
|
||||
"""Set one placed page's number — the operator's value (sparse, gaps
|
||||
allowed); pass page_number: null to leave it unnumbered."""
|
||||
body = await request.get_json() or {}
|
||||
image_id, ierr = _opt_int(body, "image_id")
|
||||
if ierr:
|
||||
return ierr
|
||||
if image_id is None:
|
||||
return jsonify({"error": "image_id required"}), 400
|
||||
if "page_number" not in body:
|
||||
return jsonify({"error": "page_number required (may be null)"}), 400
|
||||
page_number, perr = _opt_int(body, "page_number")
|
||||
if perr:
|
||||
return perr
|
||||
@tags_bp.route("/series/<int:tag_id>/reorder", methods=["POST"])
|
||||
async def series_reorder(tag_id: int):
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body, max_ids=500)
|
||||
if err:
|
||||
return err
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SeriesService(session).set_page_number(
|
||||
tag_id, image_id, page_number
|
||||
)
|
||||
await SeriesService(session).reorder(tag_id, ids)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
@@ -609,201 +364,3 @@ async def series_cover(tag_id: int):
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
# ---- chapter dividers (FC-6.x) -------------------------------------------
|
||||
# A chapter is a cosmetic divider anchored to the page that begins it; it owns
|
||||
# no pages. Page ordering follows each page's operator-set number (the
|
||||
# /pages/number endpoint), so there is no per-chapter reorder/merge — those are
|
||||
# gone.
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/chapters", methods=["POST"])
|
||||
async def series_chapter_create(tag_id: int):
|
||||
body = await request.get_json() or {}
|
||||
anchor, aerr = _opt_int(body, "anchor_image_id")
|
||||
if aerr:
|
||||
return aerr
|
||||
if anchor is None:
|
||||
return jsonify({"error": "anchor_image_id required"}), 400
|
||||
title = body.get("title")
|
||||
if title is not None and not isinstance(title, str):
|
||||
return jsonify({"error": "title must be a string"}), 400
|
||||
part, perr = _opt_int(body, "stated_part")
|
||||
if perr:
|
||||
return perr
|
||||
async with get_session() as session:
|
||||
try:
|
||||
ch = await SeriesService(session).create_divider(
|
||||
tag_id, anchor, title=title, stated_part=part,
|
||||
)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify(ch)
|
||||
|
||||
|
||||
@tags_bp.route(
|
||||
"/series/<int:tag_id>/chapters/<int:chapter_id>", methods=["PATCH"]
|
||||
)
|
||||
async def series_chapter_update(tag_id: int, chapter_id: int):
|
||||
body = await request.get_json() or {}
|
||||
kwargs: dict = {}
|
||||
if "title" in body:
|
||||
if body["title"] is not None and not isinstance(body["title"], str):
|
||||
return jsonify({"error": "title must be a string"}), 400
|
||||
kwargs.update(set_title=True, title=body["title"])
|
||||
if "stated_part" in body:
|
||||
part, perr = _opt_int(body, "stated_part")
|
||||
if perr:
|
||||
return perr
|
||||
kwargs.update(set_part=True, stated_part=part)
|
||||
if "anchor_image_id" in body:
|
||||
anchor, aerr = _opt_int(body, "anchor_image_id")
|
||||
if aerr:
|
||||
return aerr
|
||||
if anchor is None:
|
||||
return jsonify(
|
||||
{"error": "anchor_image_id must be an integer"}
|
||||
), 400
|
||||
kwargs.update(set_anchor=True, anchor_image_id=anchor)
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SeriesService(session).update_divider(
|
||||
tag_id, chapter_id, **kwargs
|
||||
)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@tags_bp.route(
|
||||
"/series/<int:tag_id>/chapters/<int:chapter_id>", methods=["DELETE"]
|
||||
)
|
||||
async def series_chapter_delete(tag_id: int, chapter_id: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SeriesService(session).delete_divider(tag_id, chapter_id)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
# ---- browse list + post→series flows (FC-6.2) -----------------------------
|
||||
|
||||
|
||||
@tags_bp.route("/series", methods=["GET"])
|
||||
async def series_list():
|
||||
args = request.args
|
||||
sort = args.get("sort", "recent")
|
||||
if sort not in ("recent", "name", "size"):
|
||||
return jsonify({"error": "sort must be recent|name|size"}), 400
|
||||
artist_id = None
|
||||
if args.get("artist_id") is not None:
|
||||
try:
|
||||
artist_id = int(args["artist_id"])
|
||||
except ValueError:
|
||||
return jsonify({"error": "artist_id must be an integer"}), 400
|
||||
async with get_session() as session:
|
||||
rows = await SeriesService(session).list_series(
|
||||
sort=sort, artist_id=artist_id
|
||||
)
|
||||
return jsonify({"series": rows})
|
||||
|
||||
|
||||
@tags_bp.route("/series/from-post", methods=["POST"])
|
||||
async def series_from_post():
|
||||
body = await request.get_json()
|
||||
post_id, err = _opt_int(body, "post_id")
|
||||
if err:
|
||||
return err
|
||||
if post_id is None:
|
||||
return jsonify({"error": "post_id required"}), 400
|
||||
async with get_session() as session:
|
||||
try:
|
||||
out = await SeriesService(session).promote_post_to_series(post_id)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/add-post", methods=["POST"])
|
||||
async def series_add_post(tag_id: int):
|
||||
body = await request.get_json()
|
||||
post_id, err = _opt_int(body, "post_id")
|
||||
if err:
|
||||
return err
|
||||
if post_id is None:
|
||||
return jsonify({"error": "post_id required"}), 400
|
||||
async with get_session() as session:
|
||||
try:
|
||||
out = await SeriesService(session).add_post(tag_id, post_id)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
@tags_bp.route("/series/<int:tag_id>/pending/place", methods=["POST"])
|
||||
async def series_place_pending(tag_id: int):
|
||||
"""Place staged (pending) pages into the run, numbered sequentially from
|
||||
`start_page` in the given order (#789). start_page null → unnumbered."""
|
||||
body = await request.get_json()
|
||||
ids, err = _parse_bulk_ids(body, max_ids=500)
|
||||
if err:
|
||||
return err
|
||||
start, serr = _opt_int(body, "start_page")
|
||||
if serr:
|
||||
return serr
|
||||
async with get_session() as session:
|
||||
try:
|
||||
n = await SeriesService(session).place_pending(
|
||||
tag_id, ids, start_page=start
|
||||
)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"placed_count": n})
|
||||
|
||||
|
||||
# ---- suggestion queue (FC-6.3) --------------------------------------------
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions", methods=["GET"])
|
||||
async def series_suggestions_list():
|
||||
async with get_session() as session:
|
||||
rows = await SeriesMatchService(session).list_pending()
|
||||
return jsonify({"suggestions": rows})
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions/<int:sid>/accept", methods=["POST"])
|
||||
async def series_suggestion_accept(sid: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
out = await SeriesMatchService(session).accept(sid)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify(out)
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions/<int:sid>/dismiss", methods=["POST"])
|
||||
async def series_suggestion_dismiss(sid: int):
|
||||
async with get_session() as session:
|
||||
try:
|
||||
await SeriesMatchService(session).dismiss(sid)
|
||||
except SeriesError as exc:
|
||||
return _series_err(exc)
|
||||
await session.commit()
|
||||
return jsonify({"ok": True})
|
||||
|
||||
|
||||
@tags_bp.route("/series/suggestions/rescan", methods=["POST"])
|
||||
async def series_suggestions_rescan():
|
||||
from ..tasks.admin import rescan_series_suggestions_task
|
||||
|
||||
res = rescan_series_suggestions_task.delay()
|
||||
return jsonify({"task_id": res.id})
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
"""Thumbnail admin API: backfill trigger."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from quart import Blueprint, jsonify
|
||||
|
||||
thumbnails_bp = Blueprint("thumbnails", __name__, url_prefix="/api/thumbnails")
|
||||
|
||||
|
||||
@thumbnails_bp.route("/backfill", methods=["POST"])
|
||||
async def trigger_backfill():
|
||||
"""Run the backfill scan synchronously, return the counts. The actual
|
||||
thumbnail generation work is still off-loaded to the thumbnail Celery
|
||||
queue via `generate_thumbnail.delay()` per missing row — so this
|
||||
handler is fast even on a 100k-image library (a scan is just SELECT
|
||||
id, thumbnail_path + a file.stat() per row, no heavy work).
|
||||
|
||||
Operator-flagged 2026-06-01: the previous fire-and-forget shape
|
||||
returned `{celery_task_id}` only, so the admin UI had no idea whether
|
||||
backfill found 0 or 5000 candidates — \"found nothing\" was
|
||||
indistinguishable from \"the worker isn't picking up the task.\""""
|
||||
from ..tasks.thumbnail import _run_backfill_scan
|
||||
|
||||
# Sync scan inside an executor so we don't block the event loop.
|
||||
counts = await asyncio.get_running_loop().run_in_executor(
|
||||
None, _run_backfill_scan,
|
||||
)
|
||||
return jsonify(counts), 200
|
||||
@@ -1,170 +0,0 @@
|
||||
"""Worker lanes: what each is doing, and the dial that changes it.
|
||||
|
||||
Milestone 422 step 2. The write half of a surface `api/system_activity.py`
|
||||
only reads.
|
||||
|
||||
## Why this is a separate blueprint
|
||||
|
||||
`system_activity` says in its own first line that it is read-only, and it
|
||||
answers a different question: its `/workers` is keyed on celery HOSTNAME and
|
||||
reports which nodes answered. That stays as it is — the existing
|
||||
SystemActivityTab consumes it.
|
||||
|
||||
This is keyed on LANE, joins the stored cap to the live pool, and accepts
|
||||
writes. Two endpoints answering "which celery processes exist" and "how much
|
||||
work is each lane allowed to do" are not the same endpoint, and folding the
|
||||
second into the first would make a read-only module a write one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from functools import partial
|
||||
|
||||
from quart import Blueprint, current_app, jsonify, request
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..services.worker_control import (
|
||||
LaneUpdateRefused,
|
||||
lane_settings,
|
||||
lane_view,
|
||||
push_lane_cap,
|
||||
store_lane_cap,
|
||||
)
|
||||
from ..services.worker_lanes import (
|
||||
LANES_BY_NAME,
|
||||
SWEEP_PERIOD_SECONDS,
|
||||
Lane,
|
||||
derived_ceiling,
|
||||
)
|
||||
from ._responses import error_response as _bad
|
||||
|
||||
workers_bp = Blueprint("workers", __name__, url_prefix="/api/system/workers")
|
||||
|
||||
|
||||
@workers_bp.route("", methods=["GET"])
|
||||
async def list_lanes():
|
||||
"""Every lane: its cap, the ceiling above it, and what is live.
|
||||
|
||||
Response: {lanes: [...], fetched_at: iso8601}
|
||||
|
||||
One database read, and NO broker call. Operator, 2026-09-23: *"there is a
|
||||
repull every time this page loads — is there a reason this info isn't
|
||||
being tracked in the background and stored in some way?"*
|
||||
|
||||
It used to inspect the broker here, four broadcasts on an eleven-second
|
||||
budget, four times a minute per open tab — while `size_worker_lanes` was
|
||||
already inspecting on a timer and discarding the same numbers. The sweep
|
||||
stores them now (`worker_lane_sample`) and this reads them.
|
||||
|
||||
So the live figures are up to `SWEEP_PERIOD_SECONDS` old, and each lane
|
||||
carries the `measured_at` that says so. `sweep_period_seconds` is returned
|
||||
alongside, so the UI can explain the age without hard-coding the cadence
|
||||
in a second place.
|
||||
"""
|
||||
async with get_session() as session:
|
||||
settings = await lane_settings(session)
|
||||
return jsonify({
|
||||
"lanes": lane_view(settings),
|
||||
"fetched_at": datetime.now(UTC).isoformat(),
|
||||
"sweep_period_seconds": SWEEP_PERIOD_SECONDS,
|
||||
})
|
||||
|
||||
|
||||
@workers_bp.route("/<name>", methods=["POST"])
|
||||
async def update_lane(name: str):
|
||||
"""Set a lane's cap. Stores it, answers, and makes the lane follow after.
|
||||
|
||||
ONE field, since 2026-09-23. It used to take `slots`, `slots_cap`,
|
||||
`enabled` and `autoscale`; how many workers are running is now a
|
||||
measurement the sizing pass owns, and `enabled` is `cap > 0`.
|
||||
|
||||
## The reply does not wait for the lane
|
||||
|
||||
Operator, 2026-09-23: *"when the number is changed the change should be
|
||||
queued so that it isn't blocking of the webui or the system itself. we
|
||||
shouldn't have to wait for the validation live."*
|
||||
|
||||
So the request does exactly one thing that can be slow — a row update —
|
||||
and hands the broker work to a background task. Turning a lane off is
|
||||
four `cancel_consumer` messages and a resize; lowering a cap is an
|
||||
`inspect` on an eleven-second budget. Both used to happen between the
|
||||
click and the response, with the stepper disabled the whole time.
|
||||
|
||||
Nothing is lost by not waiting: the cap in the database is what the
|
||||
system obeys, the sizing pass re-reads it every minute, and the table
|
||||
polls, so the live columns catch up on their own. If the web process dies
|
||||
before the background task runs, that sweep is the backstop — which is
|
||||
the same guarantee the awaited version had, since a push could fail
|
||||
there too.
|
||||
|
||||
Refusals still happen inline, because they are decided from the value and
|
||||
the machine's ceiling alone and never touch the broker:
|
||||
|
||||
* **400** — the value is not allowed (negative, or above what this
|
||||
container can hold). Nothing was stored. The body carries `detail`,
|
||||
which is the sentence the UI shows; a refused control with no reason
|
||||
reads as a bug.
|
||||
"""
|
||||
lane = LANES_BY_NAME.get(name)
|
||||
if lane is None:
|
||||
return _bad("unknown_lane", detail=name, known=sorted(LANES_BY_NAME))
|
||||
|
||||
body = await request.get_json()
|
||||
if not isinstance(body, dict):
|
||||
return _bad("invalid_body", detail="body must be a JSON object")
|
||||
|
||||
if "slots_cap" not in body:
|
||||
return _bad("invalid_body", detail="give slots_cap")
|
||||
value = body["slots_cap"]
|
||||
# Rejected rather than coerced: `True` is an int in Python, and silently
|
||||
# reading it as a cap of 1 would be a control that appears to work and
|
||||
# sets something nobody asked for.
|
||||
if not isinstance(value, int) or isinstance(value, bool):
|
||||
return _bad("invalid_body", detail="slots_cap must be an integer")
|
||||
|
||||
# Store, close the session, THEN hand off. The session must not be held
|
||||
# across broker work — that is what made this page block the whole site
|
||||
# (see `worker_control.LaneSettings`) — and now the request does not wait
|
||||
# for that work either.
|
||||
async with get_session() as session:
|
||||
try:
|
||||
was_cap = await store_lane_cap(session, lane, value)
|
||||
except LaneUpdateRefused as exc:
|
||||
return _bad("refused", detail=str(exc))
|
||||
|
||||
_schedule_push(lane, value, was_cap)
|
||||
|
||||
return jsonify({
|
||||
"name": lane.name,
|
||||
"slots_cap": value,
|
||||
"ceiling": derived_ceiling(lane),
|
||||
"enabled": value > 0,
|
||||
# The value is stored; the live lane is being told separately. The UI
|
||||
# patches its row from this and lets the next poll bring the live
|
||||
# columns, rather than refetching and paying for an inspect it just
|
||||
# avoided.
|
||||
"queued": True,
|
||||
# Raising the cap off zero is what downloads the model (step 6), and
|
||||
# the background task does it. Reported here so the UI can say a
|
||||
# download has started rather than leaving the operator to wonder why
|
||||
# a lane they just turned on is busy.
|
||||
"fetching_models": value > 0 and was_cap == 0 and bool(lane.models),
|
||||
})
|
||||
|
||||
|
||||
def _schedule_push(lane: Lane, slots_cap: int, was_cap: int) -> None:
|
||||
"""Run the live push after the response has gone out.
|
||||
|
||||
A seam, not an abstraction: it is one call, and it exists so the tests can
|
||||
hold the push still — a background task that outlived a test's patches
|
||||
would reach the real broker during teardown.
|
||||
|
||||
Quart tracks the task on the app and awaits it at shutdown, so an
|
||||
in-flight push survives a graceful restart. `partial` rather than passing
|
||||
`was_cap=` through `add_background_task`, so nothing depends on how that
|
||||
forwards keyword arguments.
|
||||
"""
|
||||
current_app.add_background_task(
|
||||
partial(push_lane_cap, lane, slots_cap, was_cap=was_cap)
|
||||
)
|
||||
@@ -1,77 +0,0 @@
|
||||
"""Celery beat that remembers when each job last ran — from task_run, not a file.
|
||||
|
||||
Celery's default PersistentScheduler keeps its memory in a shelve file in the
|
||||
working directory. Nothing mounts that directory, so every container recreate
|
||||
forgets it, and a scheduler that remembers nothing seeds every entry with
|
||||
`last_run_at = now`: each job waits a FULL interval after startup. A daily job
|
||||
therefore needs 24 hours without a redeploy to fire. Since the one-container
|
||||
image (172e33d) every redeploy restarts beat, and on 2026-09-24 no daily or
|
||||
weekly job had run since the 21st (#4408).
|
||||
|
||||
task_run already records every task that starts (celery_signals), indexed on
|
||||
(task_name, started_at DESC), and prune_task_runs keeps the newest row of each
|
||||
task however old it is. So on startup each entry takes its last_run_at from
|
||||
there:
|
||||
- a job that is overdue runs at once;
|
||||
- a job that is not due waits only the remainder of its interval;
|
||||
- a job that has never run is due now.
|
||||
|
||||
Beat keeps last_run_at in memory from then on, as the default scheduler does;
|
||||
only the startup seed changes. If the database cannot be read, the entries keep
|
||||
Celery's own default rather than beat failing to start.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.beat import Scheduler
|
||||
from sqlalchemy import func, select
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Seed for a job with no recorded run: far enough back that any interval or
|
||||
# crontab reads as due.
|
||||
NEVER = datetime(2000, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def last_runs(session, task_names: list[str]) -> dict[str, datetime]:
|
||||
"""The latest recorded start of each task, by task name."""
|
||||
from .models import TaskRun
|
||||
|
||||
if not task_names:
|
||||
return {}
|
||||
rows = session.execute(
|
||||
select(TaskRun.task_name, func.max(TaskRun.started_at))
|
||||
.where(TaskRun.task_name.in_(sorted(set(task_names))))
|
||||
.group_by(TaskRun.task_name)
|
||||
).all()
|
||||
return dict(rows)
|
||||
|
||||
|
||||
def seed(entries, last: dict[str, datetime]) -> None:
|
||||
"""Set each entry's last_run_at from `last`; a task with none is due now."""
|
||||
for entry in entries:
|
||||
entry.last_run_at = last.get(entry.task, NEVER)
|
||||
|
||||
|
||||
class TaskRunScheduler(Scheduler):
|
||||
"""An in-memory beat seeded from task_run history at startup."""
|
||||
|
||||
def setup_schedule(self):
|
||||
super().setup_schedule()
|
||||
try:
|
||||
from .tasks._sync_engine import sync_session_factory
|
||||
|
||||
with sync_session_factory()() as session:
|
||||
last = last_runs(session, [e.task for e in self.schedule.values()])
|
||||
except Exception:
|
||||
log.exception("beat: could not read task_run; every job waits a full interval")
|
||||
return
|
||||
seed(self.schedule.values(), last)
|
||||
due = sum(1 for e in self.schedule.values() if e.is_due()[0])
|
||||
log.info(
|
||||
"beat: seeded %d job(s) from task_run, %d due now",
|
||||
len(self.schedule), due,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user