Compare commits
48
Commits
2529b516e6
...
dev
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d0b0458d27 | ||
|
|
235393c08b | ||
|
|
ba96ecfb2d | ||
|
|
1e45e2c56c | ||
|
|
7071c87cd6 | ||
|
|
73eeb7a377 | ||
|
|
4fe792b61c | ||
|
|
6b19012bb6 | ||
|
|
3f8306f705 | ||
|
|
a708f5e9db | ||
|
|
ecd72015a7 | ||
|
|
7ca6ee0666 | ||
|
|
6bb18050a4 | ||
|
|
7751715b83 | ||
|
|
173f4b00aa | ||
|
|
ad8392b790 | ||
|
|
5084ba666b | ||
|
|
fe4e0f2b71 | ||
|
|
dc8af8b1a7 | ||
|
|
131237143b | ||
|
|
59d27ef76e | ||
|
|
f630e50e75 | ||
|
|
86abaf0b94 | ||
|
|
4815040d74 | ||
|
|
81b7b6f308 | ||
|
|
bfa9fd678b | ||
|
|
24a2b70a5a | ||
|
|
2c88ad3efb | ||
|
|
bfc4f9cec9 | ||
|
|
b590d25f8f | ||
|
|
635138b0d1 | ||
|
|
c0370069e0 | ||
|
|
3590c478f5 | ||
|
|
8a4af589f1 | ||
|
|
aa71cbbdbf | ||
|
|
973db73221 | ||
|
|
bc4eba636d | ||
|
|
dbc4e8b0c6 | ||
|
|
08418d54a3 | ||
|
|
b1bd2531ad | ||
|
|
389afe2f7b | ||
|
|
b979062dd7 | ||
|
|
573228b9da | ||
|
|
d044e93bdb | ||
|
|
ed2b1adc2e | ||
|
|
5e1996e77f | ||
|
|
98b56330d0 | ||
|
|
6959e1220c |
+97
-18
@@ -1,24 +1,103 @@
|
|||||||
# Database
|
# FabledCurator configuration.
|
||||||
DB_USER=fabledcurator
|
#
|
||||||
DB_PASSWORD=changeme_use_a_real_password
|
# Copy to `.env` and edit before your first production start:
|
||||||
DB_HOST=postgres
|
#
|
||||||
DB_PORT=5432
|
# cp .env.example .env
|
||||||
DB_NAME=fabledcurator
|
#
|
||||||
|
# Only the two values under CHANGE THESE actually need your attention. The
|
||||||
|
# rest have working defaults baked into docker-compose.yml and are listed
|
||||||
|
# here so you know they exist, not because you have to set them.
|
||||||
|
#
|
||||||
|
# Almost nothing else lives here on purpose. FabledCurator is configured from
|
||||||
|
# its own Settings UI, backed by the database — no restart, no YAML. If you
|
||||||
|
# are looking for where to set an import path, a download schedule or an ML
|
||||||
|
# threshold, it is in the app, not in this file.
|
||||||
|
|
||||||
# Redis / Celery
|
|
||||||
CELERY_BROKER_URL=redis://redis:6379/0
|
|
||||||
CELERY_RESULT_BACKEND=redis://redis:6379/0
|
|
||||||
|
|
||||||
# App
|
# ---------------------------------------------------------------------------
|
||||||
# Generate with: openssl rand -hex 32
|
# CHANGE THESE
|
||||||
SECRET_KEY=changeme_32_byte_hex_secret
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
# Extension API key — used in FC-3, lands later but reserved now
|
# The Postgres password. docker-compose.yml falls back to a published default
|
||||||
# Generate with: openssl rand -hex 32
|
# (`fabledcurator_dev`) so that `docker compose up` works with no config at
|
||||||
EXTENSION_API_KEY=
|
# all — which is exactly why you must not leave it at that on a real install.
|
||||||
|
# It is the credential protecting your stored platform session cookies.
|
||||||
|
DB_PASSWORD=
|
||||||
|
|
||||||
# Logging
|
# Sets Quart's app.secret_key. Today it signs nothing: FabledCurator has no
|
||||||
|
# login and uses no session cookies, so no value here is protecting anything
|
||||||
|
# right now. Set it anyway. It is required at boot rather than defaulted so
|
||||||
|
# that the day something session-backed does land, no instance is already
|
||||||
|
# running on a value published in this file.
|
||||||
|
#
|
||||||
|
# openssl rand -hex 32
|
||||||
|
SECRET_KEY=
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# FIRST BOOT ONLY — then delete this line
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# FabledCurator encrypts your stored platform credentials with a Fernet key it
|
||||||
|
# keeps at /images/secrets/credential_key.b64 — inside the ./images bind mount,
|
||||||
|
# so it outlives the container. On a brand-new install that file does not exist
|
||||||
|
# yet, and the app REFUSES TO START rather than quietly create one:
|
||||||
|
#
|
||||||
|
# MissingCredentialKey: Fernet key file not found at
|
||||||
|
# /images/secrets/credential_key.b64
|
||||||
|
#
|
||||||
|
# That refusal is deliberate. Auto-creating a key is indistinguishable from the
|
||||||
|
# disaster case — a restore that brought the database back but lost
|
||||||
|
# ./images/secrets — and there it would mint a key that cannot decrypt anything,
|
||||||
|
# leaving an instance that looks healthy while every paywalled download fails.
|
||||||
|
# So the choice is yours to make explicitly, once.
|
||||||
|
#
|
||||||
|
# Set this for your first `up`, watch the container come up, then DELETE THE
|
||||||
|
# LINE. Leaving it set disarms the protection permanently, on an instance that
|
||||||
|
# by then has credentials worth protecting.
|
||||||
|
#
|
||||||
|
# BACK UP ./images/secrets/ ALONGSIDE YOUR DATABASE. The key is the only thing
|
||||||
|
# that can read your stored credentials; a database restored without it needs
|
||||||
|
# every credential re-entered by hand.
|
||||||
|
CURATOR_BOOTSTRAP_NEW_KEY=1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Optional — defaults are fine
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Host port the UI is published on. The container always listens on 8080;
|
||||||
|
# this is only the left-hand side of the port mapping.
|
||||||
|
PORT=8080
|
||||||
|
|
||||||
|
# DEBUG | INFO | WARNING | ERROR
|
||||||
LOG_LEVEL=INFO
|
LOG_LEVEL=INFO
|
||||||
|
|
||||||
# Deployment posture: plain HTTP (no TLS in the app; reverse proxy if needed)
|
# Postgres identity. Change these only if you are pointing at a database you
|
||||||
# See docs/superpowers/specs/2026-05-13-fabledcurator-merge-design.md §2.1
|
# manage yourself — the bundled postgres service is created with whatever is
|
||||||
|
# set here, so changing them after the first start will not rename anything.
|
||||||
|
DB_USER=fabledcurator
|
||||||
|
DB_NAME=fabledcurator
|
||||||
|
|
||||||
|
# Set by docker-compose.yml to reach the bundled services. Override only when
|
||||||
|
# running Postgres or Redis outside this stack.
|
||||||
|
# DB_HOST=postgres
|
||||||
|
# DB_PORT=5432
|
||||||
|
# CELERY_BROKER_URL=redis://redis:6379/0
|
||||||
|
# CELERY_RESULT_BACKEND=redis://redis:6379/0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# There is no authentication variable here, and that is not an omission
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# FabledCurator has no login, no accounts and no permission model. Anything
|
||||||
|
# that can reach PORT is an administrator and can read the platform session
|
||||||
|
# cookies the app stores for Patreon, SubscribeStar and Pixiv.
|
||||||
|
#
|
||||||
|
# Bind it to a trusted network. See "Before you expose it" in README.md and
|
||||||
|
# the deployment posture section of SECURITY.md.
|
||||||
|
#
|
||||||
|
# The Firefox extension's API key is NOT configured here — it is generated
|
||||||
|
# automatically on first use and shown under Settings → Maintenance, where you
|
||||||
|
# can also rotate it.
|
||||||
|
|||||||
+171
-35
@@ -1,42 +1,50 @@
|
|||||||
|
|
||||||
# TEMPORARY — milestone 328 steps 1-2. Delete once the baseline is stamped.
|
# TEMPORARY — milestone 328. Delete once the baseline has shipped and settled.
|
||||||
#
|
#
|
||||||
# Squashing 87 alembic revisions into one baseline has exactly one dangerous
|
# Collapsing 89 alembic revisions into one baseline has exactly one dangerous
|
||||||
# failure: the generated baseline does not reproduce the schema the chain
|
# failure: the baseline does not reproduce the schema the chain produced, and
|
||||||
# produced, `alembic stamp` writes a version string anyway (it validates
|
# the divergence surfaces later, on the operator's live data, in whatever
|
||||||
# NOTHING), and the divergence surfaces on the next real migration against the
|
# migration comes next.
|
||||||
# operator's live data.
|
|
||||||
#
|
#
|
||||||
# So this workflow does the comparison in CI, where a pgvector Postgres already
|
# So the comparison happens in CI, against a throwaway pgvector Postgres, where
|
||||||
# gets built from the chain on every integration run, and nothing is at risk.
|
# nothing is at risk. It answers one question: does `upgrade head` on the
|
||||||
# It answers one question: does `upgrade head` on the collapsed chain produce a
|
# collapsed tree produce the same schema as `upgrade head` on the full chain?
|
||||||
# byte-identical schema to `upgrade head` on the 87-revision chain?
|
|
||||||
#
|
#
|
||||||
# The chain is read from git rather than from the working tree, so this keeps
|
# The chain is read out of GIT, not the working tree, which is what lets this
|
||||||
# working AFTER the old revisions are deleted — `chain_ref` names a commit that
|
# keep working now that the revisions are deleted — `chain_ref` names a commit
|
||||||
# still has them. That is what makes this the proof for step 1 and the
|
# that still carries 0001..0089. That is the whole reason this is a workflow
|
||||||
# pre-flight for step 2, rather than a one-shot script.
|
# rather than a script someone ran once.
|
||||||
#
|
#
|
||||||
# While the chain is still present it also autogenerates a candidate baseline
|
# WHAT THIS CANNOT SEE, and it matters: the comparison is of SCHEMA. Migrations
|
||||||
# from the models and prints it. That is a starting point, NOT the answer:
|
# 0002 and 0003 also INSERTED rows (the import_settings and ml_settings
|
||||||
# autogenerate reads SQLAlchemy metadata, and three things here do not live
|
# singletons), and the application reads those with scalar_one(), which raises
|
||||||
# there —
|
# on an empty result. A baseline that omitted them would produce an identical
|
||||||
# * CREATE EXTENSION vector (0001)
|
# schema, pass this check with a perfect diff, and crash a fresh install on its
|
||||||
# * CREATE EXTENSION tsm_system_rows (0004)
|
# first settings access. Only running the app against a new database finds
|
||||||
# * the HNSW index on image_record.siglip_embedding, which is raw SQL
|
# that class of defect. Do not read a green run here as "the baseline is
|
||||||
# because alembic's create_index cannot express `USING hnsw (...)` (0036)
|
# correct" — read it as "the schema is correct".
|
||||||
# plus any CHECK constraint or server_default that a migration added without
|
#
|
||||||
# the model declaring it. Those must be hand-added, and the diff below is what
|
# Autogenerate now emits nearly all of the baseline unaided, which was NOT true
|
||||||
# proves none were missed.
|
# before #3275 put the previously migration-only objects onto the models — the
|
||||||
|
# HNSW index with its opclass, the COALESCE expression index, the partial
|
||||||
|
# unique indexes, 107 server_defaults, the enum CHECKs. An earlier attempt at
|
||||||
|
# this squash was reverted precisely because the generator dropped them all
|
||||||
|
# silently. What still needs hand-adding is only what cannot live in a model:
|
||||||
|
# the two CREATE EXTENSION statements, the two seed rows, and the pgvector
|
||||||
|
# import the generator forgets to write.
|
||||||
name: Alembic baseline
|
name: Alembic baseline
|
||||||
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
chain_ref:
|
chain_ref:
|
||||||
description: 'Commit/tag that still carries the full 0001..0087 chain'
|
description: 'Commit/tag carrying the full 0001..0089 chain (pinned: the tree no longer has it)'
|
||||||
type: string
|
type: string
|
||||||
default: '0a5bbe8'
|
default: '725bf15'
|
||||||
|
mode:
|
||||||
|
description: 'chain = compare against this tree''s migrations; models = compare against a schema built from the MODELS'
|
||||||
|
type: string
|
||||||
|
default: 'chain'
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
compare:
|
compare:
|
||||||
@@ -79,10 +87,21 @@ jobs:
|
|||||||
test -n "$PG_IP"
|
test -n "$PG_IP"
|
||||||
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
||||||
echo "DB_HOST=$PG_IP" >> "$GITHUB_ENV"
|
echo "DB_HOST=$PG_IP" >> "$GITHUB_ENV"
|
||||||
|
# Socket probe in python, not bash's /dev/tcp — these steps run under
|
||||||
|
# `sh -e`, where that path does not exist. Same fix and same reasoning
|
||||||
|
# as ci.yml's integration job; see the comment there.
|
||||||
|
pg_ready=""
|
||||||
for i in $(seq 1 60); do
|
for i in $(seq 1 60); do
|
||||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then
|
||||||
|
pg_ready=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
|
if [ -z "$pg_ready" ]; then
|
||||||
|
echo "postgres at $PG_IP:5432 did not accept a connection within 120s"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
if command -v uv >/dev/null 2>&1; then
|
if command -v uv >/dev/null 2>&1; then
|
||||||
uv pip install --system -r requirements.txt
|
uv pip install --system -r requirements.txt
|
||||||
else
|
else
|
||||||
@@ -96,10 +115,15 @@ jobs:
|
|||||||
- name: Build the schema the OLD chain produces
|
- name: Build the schema the OLD chain produces
|
||||||
env:
|
env:
|
||||||
CHAIN_REF: ${{ github.event.inputs.chain_ref }}
|
CHAIN_REF: ${{ github.event.inputs.chain_ref }}
|
||||||
|
THIS_SHA: ${{ github.sha }}
|
||||||
run: |
|
run: |
|
||||||
set -eux
|
set -eux
|
||||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_chain
|
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_chain
|
||||||
git worktree add /tmp/chain "$CHAIN_REF"
|
# Blank means "the chain in this ref", which is what you want while
|
||||||
|
# the chain is still intact — comparing the models against a PINNED
|
||||||
|
# older commit reports every migration written since as a difference.
|
||||||
|
# Pin it only after the collapse, when the tree no longer has them.
|
||||||
|
git worktree add /tmp/chain "${CHAIN_REF:-$THIS_SHA}"
|
||||||
ls /tmp/chain/alembic/versions/*.py | wc -l
|
ls /tmp/chain/alembic/versions/*.py | wc -l
|
||||||
cd /tmp/chain
|
cd /tmp/chain
|
||||||
DB_NAME=fc_chain alembic upgrade head
|
DB_NAME=fc_chain alembic upgrade head
|
||||||
@@ -107,6 +131,23 @@ jobs:
|
|||||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||||
--no-owner --no-privileges -d fc_chain > chain.sql
|
--no-owner --no-privileges -d fc_chain > chain.sql
|
||||||
wc -l chain.sql
|
wc -l chain.sql
|
||||||
|
# Emit the dump itself, checksummed, for local analysis. Reconciling
|
||||||
|
# the models against the deployed schema (#3275) needs the ACTUAL
|
||||||
|
# schema, not an inference from a diff — parsing table context out of
|
||||||
|
# unified-diff hunks drops every table whose CREATE TABLE line falls
|
||||||
|
# outside a hunk, which silently under-reports.
|
||||||
|
#
|
||||||
|
# base64 + sha256 for the same reason as the candidate: a plain cat
|
||||||
|
# of a file this size was truncated mid-line by the runner with the
|
||||||
|
# step still green (run 4964).
|
||||||
|
set +x
|
||||||
|
B64=$(base64 -w 120 chain.sql)
|
||||||
|
echo "===== BEGIN CHAIN SCHEMA (base64) ====="
|
||||||
|
echo "$B64"
|
||||||
|
echo "===== END CHAIN SCHEMA ====="
|
||||||
|
echo "chain-sha256: $(sha256sum chain.sql | cut -d' ' -f1)"
|
||||||
|
echo "chain-bytes: $(wc -c < chain.sql)"
|
||||||
|
set -x
|
||||||
|
|
||||||
# A candidate baseline, autogenerated from the models against an EMPTY
|
# A candidate baseline, autogenerated from the models against an EMPTY
|
||||||
# database so every table shows up as a create. Printed for a human to
|
# database so every table shows up as a create. Printed for a human to
|
||||||
@@ -157,20 +198,53 @@ jobs:
|
|||||||
echo "candidate-bytes: $(wc -c < "$F")"
|
echo "candidate-bytes: $(wc -c < "$F")"
|
||||||
echo "candidate-b64-lines: $(echo "$B64" | wc -l)"
|
echo "candidate-b64-lines: $(echo "$B64" | wc -l)"
|
||||||
set -x
|
set -x
|
||||||
|
mkdir -p /tmp/candidate
|
||||||
|
cp alembic/versions/*.py /tmp/candidate/
|
||||||
# Put the tree back exactly as it was; this job never mutates state.
|
# Put the tree back exactly as it was; this job never mutates state.
|
||||||
rm -f alembic/versions/*.py
|
rm -f alembic/versions/*.py
|
||||||
mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true
|
mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true
|
||||||
|
|
||||||
# DB 2: whatever the CURRENT tree's alembic/versions produces. Before the
|
# DB 2: what the CURRENT tree produces.
|
||||||
# squash that is the same 87 revisions and the diff is trivially clean —
|
#
|
||||||
# which is worth running once as a control, so a clean diff after the
|
# `mode: models` applies the candidate autogenerated from the MODELS
|
||||||
# squash means something.
|
# instead, which is what answers "do the models describe the schema?" —
|
||||||
|
# the question #3275 exists because nobody had ever asked it. Under that
|
||||||
|
# mode a clean diff means autogenerate is trustworthy again.
|
||||||
|
#
|
||||||
|
# The two extensions are created by hand first. They are database
|
||||||
|
# objects, not table metadata, so no model can carry them and their
|
||||||
|
# absence is not a model defect — it is simply outside what this
|
||||||
|
# comparison is asking about.
|
||||||
- name: Build the schema the CURRENT tree produces
|
- name: Build the schema the CURRENT tree produces
|
||||||
|
env:
|
||||||
|
MODE: ${{ github.event.inputs.mode }}
|
||||||
run: |
|
run: |
|
||||||
set -eux
|
set -eux
|
||||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base
|
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base
|
||||||
|
if [ "${MODE:-chain}" = "models" ]; then
|
||||||
|
docker exec "$PG_CONTAINER" psql -U fabledcurator -d fc_base \
|
||||||
|
-c "CREATE EXTENSION IF NOT EXISTS vector" \
|
||||||
|
-c "CREATE EXTENSION IF NOT EXISTS tsm_system_rows"
|
||||||
|
mkdir -p /tmp/held
|
||||||
|
mv alembic/versions/*.py /tmp/held/
|
||||||
|
cp /tmp/candidate/*.py alembic/versions/
|
||||||
|
# Autogenerate EMITS pgvector.sqlalchemy.vector.VECTOR(...) without
|
||||||
|
# importing it, so the file it writes cannot run:
|
||||||
|
# NameError: name 'pgvector' is not defined
|
||||||
|
# Observed on run 4988, which is the proof rather than the theory.
|
||||||
|
# This is a defect in the GENERATOR, not in the models, so it is
|
||||||
|
# repaired here rather than counted as a schema difference — the
|
||||||
|
# comparison is about whether the models describe the schema.
|
||||||
|
sed -i '0,/^import sqlalchemy as sa$/s//import sqlalchemy as sa\nimport pgvector.sqlalchemy.vector/' alembic/versions/*.py
|
||||||
|
grep -n 'import pgvector' alembic/versions/*.py
|
||||||
|
ls alembic/versions/*.py
|
||||||
|
DB_NAME=fc_base alembic upgrade head
|
||||||
|
rm -f alembic/versions/*.py
|
||||||
|
mv /tmp/held/*.py alembic/versions/
|
||||||
|
else
|
||||||
ls alembic/versions/*.py | wc -l
|
ls alembic/versions/*.py | wc -l
|
||||||
DB_NAME=fc_base alembic upgrade head
|
DB_NAME=fc_base alembic upgrade head
|
||||||
|
fi
|
||||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||||
--no-owner --no-privileges -d fc_base > baseline.sql
|
--no-owner --no-privileges -d fc_base > baseline.sql
|
||||||
wc -l baseline.sql
|
wc -l baseline.sql
|
||||||
@@ -191,6 +265,25 @@ jobs:
|
|||||||
# these two lines and nothing else. That control is what licenses this
|
# these two lines and nothing else. That control is what licenses this
|
||||||
# filter — it was observed to be the only false positive, rather than
|
# filter — it was observed to be the only false positive, rather than
|
||||||
# assumed to be one.
|
# assumed to be one.
|
||||||
|
# Column ORDER inside a CREATE TABLE is compared separately from column
|
||||||
|
# CONTENT, and only content is fatal.
|
||||||
|
#
|
||||||
|
# A table built by 87 migrations has its columns in ADD COLUMN order; the
|
||||||
|
# same table built in one shot has them in declaration order. That is a
|
||||||
|
# real and permanent difference which no baseline can erase — the
|
||||||
|
# operator's existing database keeps chain order forever, a fresh install
|
||||||
|
# gets model order — so a check that fails on it would never pass and
|
||||||
|
# would teach nothing. FC reaches every column through the ORM by name,
|
||||||
|
# and `SELECT *` ordering is not depended on anywhere.
|
||||||
|
#
|
||||||
|
# So the second pass SORTS the column lines within each CREATE TABLE
|
||||||
|
# rather than DELETING them. That distinction is the whole point: sorting
|
||||||
|
# cannot hide a column that exists on one side only, or one whose type,
|
||||||
|
# nullability or default differs — those still land in the diff. A filter
|
||||||
|
# could have hidden all three.
|
||||||
|
#
|
||||||
|
# Both diffs are reported. The ordered one is informational; the
|
||||||
|
# order-insensitive one is the verdict.
|
||||||
- name: Diff
|
- name: Diff
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
@@ -202,11 +295,54 @@ jobs:
|
|||||||
norm chain.sql > a.txt
|
norm chain.sql > a.txt
|
||||||
norm baseline.sql > b.txt
|
norm baseline.sql > b.txt
|
||||||
echo "normalised: chain=$(wc -l < a.txt) lines, current=$(wc -l < b.txt) lines"
|
echo "normalised: chain=$(wc -l < a.txt) lines, current=$(wc -l < b.txt) lines"
|
||||||
|
|
||||||
|
sort_table_columns() {
|
||||||
|
python3 - "$1" <<'PYEOF'
|
||||||
|
import re, sys
|
||||||
|
|
||||||
|
lines = open(sys.argv[1]).read().splitlines()
|
||||||
|
out, block = [], None
|
||||||
|
for line in lines:
|
||||||
|
if block is not None:
|
||||||
|
# ');' on its own closes the CREATE TABLE body.
|
||||||
|
if line.strip() == ");":
|
||||||
|
out.extend(sorted(block))
|
||||||
|
out.append(line)
|
||||||
|
block = None
|
||||||
|
else:
|
||||||
|
# Drop the list comma before sorting. Only the LAST
|
||||||
|
# column lacks one, so keeping it would make every
|
||||||
|
# reordering look like a content change as well — the
|
||||||
|
# comma is punctuation, and carries no schema meaning.
|
||||||
|
block.append(line.rstrip().rstrip(","))
|
||||||
|
continue
|
||||||
|
out.append(line)
|
||||||
|
if re.match(r"CREATE TABLE .*\($", line):
|
||||||
|
block = []
|
||||||
|
if block is not None: # unterminated body: emit it rather than drop it
|
||||||
|
out.extend(block)
|
||||||
|
print("\n".join(out))
|
||||||
|
PYEOF
|
||||||
|
}
|
||||||
|
sort_table_columns a.txt > a.sorted.txt
|
||||||
|
sort_table_columns b.txt > b.sorted.txt
|
||||||
|
test "$(wc -l < a.sorted.txt)" = "$(wc -l < a.txt)"
|
||||||
|
test "$(wc -l < b.sorted.txt)" = "$(wc -l < b.txt)"
|
||||||
|
|
||||||
if diff -u a.txt b.txt > schema.diff; then
|
if diff -u a.txt b.txt > schema.diff; then
|
||||||
echo "SCHEMAS IDENTICAL — the collapsed chain reproduces the old one."
|
echo "ORDERED DIFF: identical, column order included."
|
||||||
else
|
else
|
||||||
echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' schema.diff) changed lines:"
|
echo "ORDERED DIFF: $(grep -cE '^[+-]' schema.diff) changed lines (informational):"
|
||||||
cat schema.diff
|
cat schema.diff
|
||||||
|
fi
|
||||||
|
echo
|
||||||
|
echo "================================================================"
|
||||||
|
echo
|
||||||
|
if diff -u a.sorted.txt b.sorted.txt > sorted.diff; then
|
||||||
|
echo "SCHEMAS MATCH — every difference above is column ORDER alone."
|
||||||
|
else
|
||||||
|
echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' sorted.diff) changed lines that are NOT ordering:"
|
||||||
|
cat sorted.diff
|
||||||
echo
|
echo
|
||||||
echo "The baseline is wrong, not the database. Do not stamp."
|
echo "The baseline is wrong, not the database. Do not stamp."
|
||||||
exit 1
|
exit 1
|
||||||
|
|||||||
+565
-59
@@ -44,6 +44,10 @@ on:
|
|||||||
description: 'Rebuild every image even if the published revision matches'
|
description: 'Rebuild every image even if the published revision matches'
|
||||||
type: boolean
|
type: boolean
|
||||||
default: false
|
default: false
|
||||||
|
refresh:
|
||||||
|
description: 'Behave as the weekly base refresh: build main against fresh bases, publish through the candidate tag'
|
||||||
|
type: boolean
|
||||||
|
default: false
|
||||||
|
|
||||||
# The base-image refresh (milestone 326 step 4, #3154).
|
# The base-image refresh (milestone 326 step 4, #3154).
|
||||||
#
|
#
|
||||||
@@ -72,8 +76,43 @@ on:
|
|||||||
# Deriving it per job invites the two halves to disagree: sign-extension would
|
# Deriving it per job invites the two halves to disagree: sign-extension would
|
||||||
# derive dev's extension version while build-web bundled main's, and the
|
# derive dev's extension version while build-web bundled main's, and the
|
||||||
# release download would 404 on a version that exists perfectly well.
|
# release download would 404 on a version that exists perfectly well.
|
||||||
|
# IS THIS A BASE REFRESH? Asked in five places and previously spelled five
|
||||||
|
# ways — `github.event_name == 'schedule'` in an `if:`, `$GITHUB_EVENT_NAME` in
|
||||||
|
# one shell, an `EVENT:` env passed into another, and a bare expression on
|
||||||
|
# `pull:`. Five spellings of one fact is how half of them come to disagree
|
||||||
|
# after somebody adds a sixth trigger.
|
||||||
|
#
|
||||||
|
# The `refresh` dispatch input is here so this path can be EXERCISED. A weekly
|
||||||
|
# cron is otherwise testable once a week, which is not a cadence anything can
|
||||||
|
# be developed against — the same reason `force_build` exists (#3252, added to
|
||||||
|
# confirm #3190 was gone rather than wait for it to recur). It is also what
|
||||||
|
# makes the milestone-362 gate verifiable at all: a gate has to be watched
|
||||||
|
# rejecting something before anyone can believe it is wired up.
|
||||||
|
#
|
||||||
|
# The input is normalised through `format()` before it is compared, and that
|
||||||
|
# is not defensive styling — the direct comparison is WRONG and fails silently.
|
||||||
|
#
|
||||||
|
# `type: boolean` delivers a real boolean, and GitHub expression semantics cast
|
||||||
|
# operands to numbers when their types differ: `true == 'true'` compares 1
|
||||||
|
# against NaN and is FALSE. Measured on run 5270, whose own log says it —
|
||||||
|
#
|
||||||
|
# expression '(github.event_name == 'schedule'
|
||||||
|
# || github.event.inputs.refresh == 'true') && 'true' || 'false''
|
||||||
|
# evaluated to '%!t(string=false)'
|
||||||
|
# trigger: raw inputs refresh='true'
|
||||||
|
#
|
||||||
|
# — the input arrived as `true` and the expression still said false. The run
|
||||||
|
# then went green with every step skipped, because a refresh that evaluates
|
||||||
|
# false behaves exactly like an ordinary push. That is the whole hazard: the
|
||||||
|
# failure has no symptom.
|
||||||
|
#
|
||||||
|
# `force_build` never hit this because it never compares in an expression. It
|
||||||
|
# passes the raw value into an env var and tests it in the shell, where
|
||||||
|
# everything is already a string. `format('{0}', x)` buys the same thing here,
|
||||||
|
# where a step-level `if:` needs the answer before any shell runs.
|
||||||
env:
|
env:
|
||||||
BUILD_REF: ${{ github.event_name == 'schedule' && 'main' || github.ref }}
|
IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }}
|
||||||
|
BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.ref }}
|
||||||
|
|
||||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||||
@@ -143,7 +182,7 @@ jobs:
|
|||||||
# evaluate — this file already gates steps on it — so the guard cannot
|
# evaluate — this file already gates steps on it — so the guard cannot
|
||||||
# be disabled by the same uncertainty it exists to cover.
|
# be disabled by the same uncertainty it exists to cover.
|
||||||
- name: Guard — a scheduled run must have checked out main
|
- name: Guard — a scheduled run must have checked out main
|
||||||
if: github.event_name == 'schedule'
|
if: env.IS_REFRESH == 'true'
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||||
@@ -413,6 +452,18 @@ jobs:
|
|||||||
# to. Same source of truth; no double-store.
|
# to. Same source of truth; no double-store.
|
||||||
|
|
||||||
build-web:
|
build-web:
|
||||||
|
# Consumed by smoke-web's job-level `if:`. It cannot read `env` — the env
|
||||||
|
# context is available to STEP `if:` and step bodies, never to a job's own
|
||||||
|
# condition, and an unresolvable context there is empty rather than an
|
||||||
|
# error. `smoke-web` skipped silently on run 5290 for exactly that reason.
|
||||||
|
#
|
||||||
|
# Keying off the reuse step's own output is better than re-deriving the
|
||||||
|
# trigger anyway: it is the same single decision the build, the XPI
|
||||||
|
# download and the promote all take (build.yml's "one decision drives
|
||||||
|
# everything downstream"), and it says the thing smoke-web actually needs
|
||||||
|
# to know — a candidate was published — rather than restating why.
|
||||||
|
outputs:
|
||||||
|
candidate: ${{ steps.reuse.outputs.promote }}
|
||||||
# A plain `needs` — no `always()`. That expression existed to let a
|
# A plain `needs` — no `always()`. That expression existed to let a
|
||||||
# SKIPPED sign-extension through on a tag push while still blocking a
|
# SKIPPED sign-extension through on a tag push while still blocking a
|
||||||
# FAILED one. With no tag trigger, sign-extension always runs, so the
|
# FAILED one. With no tag trigger, sign-extension always runs, so the
|
||||||
@@ -437,7 +488,7 @@ jobs:
|
|||||||
|
|
||||||
# See sign-extension's copy for why this guard exists.
|
# See sign-extension's copy for why this guard exists.
|
||||||
- name: Guard — a scheduled run must have checked out main
|
- name: Guard — a scheduled run must have checked out main
|
||||||
if: github.event_name == 'schedule'
|
if: env.IS_REFRESH == 'true'
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||||
@@ -470,8 +521,18 @@ jobs:
|
|||||||
# the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
|
# the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
|
||||||
# * dev and main derive the same values for the same source.
|
# * dev and main derive the same values for the same source.
|
||||||
- name: Report the derived artifact version
|
- name: Report the derived artifact version
|
||||||
|
env:
|
||||||
|
# Diagnostic for the trigger normalisation. `refresh` is reported RAW
|
||||||
|
# as well as normalised, because the two disagreeing is the whole
|
||||||
|
# failure mode: a dispatch input whose type does not compare the way
|
||||||
|
# the expression assumes evaluates to false silently, and the only
|
||||||
|
# symptom is a refresh that quietly behaves like an ordinary push.
|
||||||
|
RAW_REFRESH: ${{ github.event.inputs.refresh }}
|
||||||
|
RAW_FORCE: ${{ github.event.inputs.force_build }}
|
||||||
run: |
|
run: |
|
||||||
set -u
|
set -u
|
||||||
|
echo "trigger: event=$GITHUB_EVENT_NAME IS_REFRESH='${IS_REFRESH:-<unset>}' BUILD_REF='${BUILD_REF:-<unset>}'"
|
||||||
|
echo "trigger: raw inputs refresh='${RAW_REFRESH:-<unset>}' force_build='${RAW_FORCE:-<unset>}'"
|
||||||
A=web
|
A=web
|
||||||
V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
|
V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
|
||||||
R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
|
R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
|
||||||
@@ -528,7 +589,7 @@ jobs:
|
|||||||
# Checked BEFORE the ref test, not after: a scheduled run's
|
# Checked BEFORE the ref test, not after: a scheduled run's
|
||||||
# GITHUB_REF is the default branch (dev), so the main test would
|
# GITHUB_REF is the default branch (dev), so the main test would
|
||||||
# never fire on it.
|
# never fire on it.
|
||||||
if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
|
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
||||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||||
@@ -628,7 +689,6 @@ jobs:
|
|||||||
# A scheduled refresh has to bypass reuse by construction: it
|
# A scheduled refresh has to bypass reuse by construction: it
|
||||||
# rebuilds the SAME source, so fc.revision always matches and the
|
# rebuilds the SAME source, so fc.revision always matches and the
|
||||||
# check would skip every refresh there has ever been.
|
# check would skip every refresh there has ever been.
|
||||||
EVENT: ${{ github.event_name }}
|
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
DERIVED=$(sh scripts/artifacts.sh revision web)
|
DERIVED=$(sh scripts/artifacts.sh revision web)
|
||||||
@@ -638,11 +698,60 @@ jobs:
|
|||||||
# adds no variability the reuse check would have to account for.
|
# adds no variability the reuse check would have to account for.
|
||||||
echo "version=$(sh scripts/artifacts.sh version web)" >> "$GITHUB_OUTPUT"
|
echo "version=$(sh scripts/artifacts.sh version web)" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
# The build clock, pinned to the same commit (#3265). Without it
|
||||||
|
# buildkit stamps the image config with the wall clock of the build,
|
||||||
|
# so identical layers republish under a new config blob and the
|
||||||
|
# channel tag gets a new manifest digest for no reason. Derived from
|
||||||
|
# `newest()` like revision and version, so all three name one commit
|
||||||
|
# and cannot drift apart.
|
||||||
|
echo "epoch=$(sh scripts/artifacts.sh epoch web)" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
# The moving tag for this channel. Which tag we ask IS the channel —
|
# The moving tag for this channel. Which tag we ask IS the channel —
|
||||||
# that is why the revision needs no -main/-dev qualifier any more.
|
# that is why the revision needs no -main/-dev qualifier any more.
|
||||||
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
|
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
|
||||||
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||||
|
# whether the channel then has to be written separately.
|
||||||
|
#
|
||||||
|
# On a push the build writes the channel tag directly: the bytes came
|
||||||
|
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
||||||
|
# it behind.
|
||||||
|
#
|
||||||
|
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
||||||
|
# refresh rebuilds against freshly resolved base images, and the web
|
||||||
|
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||||
|
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||||
|
# Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and
|
||||||
|
# install requirements.txt, and a base bump changes neither. So
|
||||||
|
# refreshed bytes have to be proven before :latest names them, and
|
||||||
|
# proving needs a moment between "built" and "published" to occupy.
|
||||||
|
# This is that moment; :latest goes on naming the build that works
|
||||||
|
# until something says otherwise.
|
||||||
|
#
|
||||||
|
# `:refresh-candidate` is one moving ref per image, overwritten in
|
||||||
|
# place, holding a build nobody is told to pull — the shape rule 145
|
||||||
|
# already allows for :buildcache, not the per-build tag family that
|
||||||
|
# milestone 318 withdrew.
|
||||||
|
#
|
||||||
|
# Decided HERE, beside `hit`, for the reason the force/schedule
|
||||||
|
# branch below gives: one step decides what this job does. A
|
||||||
|
# condition derived independently could disagree with the tag the
|
||||||
|
# build actually wrote.
|
||||||
|
#
|
||||||
|
# build-web additionally exposes this as `outputs.candidate`, which is
|
||||||
|
# what gates the `promote` job — a job's `if:` cannot read `env`, and
|
||||||
|
# one flag is enough because all three derive it from the same
|
||||||
|
# IS_REFRESH. ml and agent do not re-emit it; a second copy nothing
|
||||||
|
# reads is the kind of thing that later reads as load-bearing.
|
||||||
|
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
|
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "promote=true" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||||
|
echo "promote=false" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
||||||
# (run 4732): a missing key returns an empty string and exits 0, so
|
# (run 4732): a missing key returns an empty string and exits 0, so
|
||||||
# branching on the exit code would read "no label yet" as success.
|
# branching on the exit code would read "no label yet" as success.
|
||||||
@@ -675,7 +784,7 @@ jobs:
|
|||||||
if [ "${FORCE:-false}" = "true" ]; then
|
if [ "${FORCE:-false}" = "true" ]; then
|
||||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||||
echo "reuse: force_build set — building regardless"
|
echo "reuse: force_build set — building regardless"
|
||||||
elif [ "${EVENT:-}" = "schedule" ]; then
|
elif [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||||
echo "reuse: scheduled base refresh — building regardless"
|
echo "reuse: scheduled base refresh — building regardless"
|
||||||
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||||
@@ -764,6 +873,12 @@ jobs:
|
|||||||
|
|
||||||
- name: Build and push web image
|
- name: Build and push web image
|
||||||
if: steps.reuse.outputs.hit != 'true'
|
if: steps.reuse.outputs.hit != 'true'
|
||||||
|
# Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
|
||||||
|
# it normalises the image config's `created` field and the history
|
||||||
|
# timestamps rather than being consumed by the Dockerfile. See #3265
|
||||||
|
# and the reuse step's `epoch` output.
|
||||||
|
env:
|
||||||
|
SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }}
|
||||||
uses: docker/build-push-action@v5
|
uses: docker/build-push-action@v5
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
@@ -776,20 +891,17 @@ jobs:
|
|||||||
# invalidates, and the image genuinely rebuilds.
|
# invalidates, and the image genuinely rebuilds.
|
||||||
#
|
#
|
||||||
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
||||||
# did NOT move, the build is ~13s and every content step reports
|
# did NOT move, the build was ~13s with every content step CACHED —
|
||||||
# CACHED — but the channel tag STILL gets a new manifest digest.
|
# and the channel tag STILL got a new manifest digest, because
|
||||||
# buildkit mints a fresh image config each run, so identical layers
|
# buildkit stamps a fresh image config per run and republishes the
|
||||||
# are republished under a new config blob. All three images moved
|
# identical layers under it. All three images moved that way on
|
||||||
# that way on 2026-08-30 with nothing whatsoever changed in them.
|
# 2026-08-30 with nothing whatsoever changed in them.
|
||||||
#
|
#
|
||||||
# So a refresh currently rewrites :latest every Sunday whether or
|
# SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the
|
||||||
# not there is anything new in it, and :c-<sha> is handed a new
|
# content came from, the config is byte-identical across runs, so
|
||||||
# manifest to diverge from on the same cadence. Layers are shared,
|
# the manifest digest is too and the push is a registry no-op. A
|
||||||
# so the storage cost is a config blob; the cost that matters is
|
# digest change means the content changed again, which is the only
|
||||||
# that a digest change no longer MEANS anything. Tracked in #3265 —
|
# thing a digest is any use for.
|
||||||
# the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
|
|
||||||
# make "same source, same bytes" true and turn the no-op case into
|
|
||||||
# a genuine no-op.
|
|
||||||
#
|
#
|
||||||
# What `pull` does NOT catch either: a Debian package update inside
|
# What `pull` does NOT catch either: a Debian package update inside
|
||||||
# the `apt-get install` layer while the base tag itself stands
|
# the `apt-get install` layer while the base tag itself stands
|
||||||
@@ -799,14 +911,14 @@ jobs:
|
|||||||
# churn #3265 is about.
|
# churn #3265 is about.
|
||||||
#
|
#
|
||||||
# Only on the schedule. An ordinary push wants the cached base.
|
# Only on the schedule. An ordinary push wants the cached base.
|
||||||
pull: ${{ github.event_name == 'schedule' }}
|
pull: ${{ env.IS_REFRESH == 'true' }}
|
||||||
# ONE tag, the channel's. Every other tag is written by the step
|
# ONE tag, the channel's. Every other tag is written by the step
|
||||||
# below, registry-side. buildx here pushes the first tag to the
|
# below, registry-side. buildx here pushes the first tag to the
|
||||||
# registry and then re-pushes the rest through the DOCKER driver,
|
# registry and then re-pushes the rest through the DOCKER driver,
|
||||||
# out of a local image store a registry-direct build never filled —
|
# out of a local image store a registry-direct build never filled —
|
||||||
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
|
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
|
||||||
# published perfectly well.
|
# published perfectly well.
|
||||||
tags: ${{ steps.reuse.outputs.channel_ref }}
|
tags: ${{ steps.reuse.outputs.build_ref }}
|
||||||
# The reuse key. Read back off the channel tag on the next push to
|
# The reuse key. Read back off the channel tag on the next push to
|
||||||
# decide whether that push needs to build at all, so this is not
|
# decide whether that push needs to build at all, so this is not
|
||||||
# decoration — an unstamped image is one that will always rebuild.
|
# decoration — an unstamped image is one that will always rebuild.
|
||||||
@@ -937,6 +1049,282 @@ jobs:
|
|||||||
docker buildx imagetools create $ARGS "$SOURCE"
|
docker buildx imagetools create $ARGS "$SOURCE"
|
||||||
echo "repointed from $SOURCE:$ARGS"
|
echo "repointed from $SOURCE:$ARGS"
|
||||||
|
|
||||||
|
# Does the image a refresh just built still work?
|
||||||
|
#
|
||||||
|
# This is the gate the base refresh never had. `ci.yml` cannot be it: its
|
||||||
|
# lanes run on ci-python:3.14 and install requirements.txt, and a base bump
|
||||||
|
# changes neither — all five stay green through a refresh that breaks the
|
||||||
|
# product. What a refresh re-resolves is the Dockerfile's apt layer (ffmpeg,
|
||||||
|
# unar, libpq5, postgresql-client, zstd, megatools, libjpeg62-turbo,
|
||||||
|
# libwebp7, libpng16-16), unpinned, every build.
|
||||||
|
#
|
||||||
|
# So this runs the CANDIDATE IMAGE, against real Postgres and Redis. Not the
|
||||||
|
# source tree, and not a static inspection: `ffmpeg -version` exiting 0 would
|
||||||
|
# pass while a codec removal broke every thumbnail in the library.
|
||||||
|
#
|
||||||
|
# Refresh-only. On a push the bytes came from a commit, and a commit is what
|
||||||
|
# ci.yml already tests.
|
||||||
|
#
|
||||||
|
# Reports a verdict; it does not yet gate the promote (milestone 362 step 4).
|
||||||
|
# Landing the gate and the thing it gates in one change would mean the first
|
||||||
|
# time anyone saw this job run would also be the first time it could stop a
|
||||||
|
# publish.
|
||||||
|
smoke-web:
|
||||||
|
needs: [build-web]
|
||||||
|
if: needs.build-web.outputs.candidate == 'true'
|
||||||
|
runs-on: python-ci
|
||||||
|
container:
|
||||||
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
|
env:
|
||||||
|
DB_USER: fabledcurator
|
||||||
|
DB_PASSWORD: ci_smoke
|
||||||
|
DB_PORT: "5432"
|
||||||
|
DB_NAME: fabledcurator_smoke
|
||||||
|
SECRET_KEY: ci_smoke_placeholder
|
||||||
|
IMAGE: git.fabledsword.com/bvandeusen/fabledcurator
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: pgvector/pgvector:pg16
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: fabledcurator
|
||||||
|
POSTGRES_PASSWORD: ci_smoke
|
||||||
|
POSTGRES_DB: fabledcurator_smoke
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U fabledcurator"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 10
|
||||||
|
redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
options: >-
|
||||||
|
--health-cmd "redis-cli ping"
|
||||||
|
--health-interval 10s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 10
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
# The same ref the image was built from, so the smoke script matches
|
||||||
|
# the code inside the candidate.
|
||||||
|
ref: ${{ env.BUILD_REF }}
|
||||||
|
|
||||||
|
- name: Smoke the candidate image
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
|
ACTOR: ${{ github.actor }}
|
||||||
|
run: |
|
||||||
|
set -eux
|
||||||
|
# Service discovery mirrors ci.yml's integration lane: these jobs run
|
||||||
|
# in a container against a mounted docker socket, so the services are
|
||||||
|
# SIBLINGS reachable by IP, not by hostname.
|
||||||
|
PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||||
|
RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||||
|
test -n "$PG" && test -n "$RD"
|
||||||
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||||
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||||
|
test -n "$PG_IP" && test -n "$RD_IP"
|
||||||
|
|
||||||
|
# Socket probe in python, not bash's /dev/tcp — these steps run under
|
||||||
|
# `sh -e`, where that path does not exist. Same fix and reasoning as
|
||||||
|
# ci.yml's integration job; see the comment there.
|
||||||
|
pg_ready=""
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then
|
||||||
|
pg_ready=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
if [ -z "$pg_ready" ]; then
|
||||||
|
echo "postgres at $PG_IP:5432 did not accept a connection within 120s"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin
|
||||||
|
CANDIDATE="$IMAGE:refresh-candidate"
|
||||||
|
docker pull "$CANDIDATE"
|
||||||
|
|
||||||
|
ENVOPTS="-e DB_USER=$DB_USER -e DB_PASSWORD=$DB_PASSWORD -e DB_HOST=$PG_IP"
|
||||||
|
ENVOPTS="$ENVOPTS -e DB_PORT=5432 -e DB_NAME=$DB_NAME -e SECRET_KEY=$SECRET_KEY"
|
||||||
|
ENVOPTS="$ENVOPTS -e CELERY_BROKER_URL=redis://$RD_IP:6379/0"
|
||||||
|
ENVOPTS="$ENVOPTS -e CELERY_RESULT_BACKEND=redis://$RD_IP:6379/0"
|
||||||
|
# A throwaway CI instance IS first-time setup, which is the one case
|
||||||
|
# credential_crypto allows a key to be minted in. Without it the web
|
||||||
|
# role refuses to boot — deliberately, since silently generating a
|
||||||
|
# key on a restored-DB-but-lost-secrets deployment would leave every
|
||||||
|
# Credential row undecryptable (the 2026-06-02 audit). Discovered by
|
||||||
|
# this job on its first real run; see #3422 for the fact that no
|
||||||
|
# user-facing file mentions this variable at all.
|
||||||
|
ENVOPTS="$ENVOPTS -e CURATOR_BOOTSTRAP_NEW_KEY=1"
|
||||||
|
|
||||||
|
# 1. The schema builds from empty, using the image's OWN libpq and
|
||||||
|
# psycopg. This is the same call entrypoint.sh makes before it
|
||||||
|
# serves anything, so a failure here is a failure to boot.
|
||||||
|
echo "smoke: alembic upgrade head"
|
||||||
|
docker run --rm $ENVOPTS "$CANDIDATE" alembic upgrade head
|
||||||
|
|
||||||
|
# 2. The apt layer's binaries and the app's own thumbnail path, run
|
||||||
|
# inside the image. Piped over stdin rather than bind-mounted: the
|
||||||
|
# workspace is a docker VOLUME belonging to this job's container,
|
||||||
|
# so a host bind of $PWD would not resolve for a sibling.
|
||||||
|
echo "smoke: image-internal checks"
|
||||||
|
docker run --rm -i $ENVOPTS "$CANDIDATE" shell -c 'python3 -' < scripts/smoke_image.py
|
||||||
|
|
||||||
|
# 3. It actually serves. `docker run -d` then poll the container's own
|
||||||
|
# IP — no port publishing, because the job container reaches
|
||||||
|
# siblings directly and a published port would collide with
|
||||||
|
# whatever else the runner is hosting.
|
||||||
|
echo "smoke: web boots and answers /api/health"
|
||||||
|
CID=$(docker run -d $ENVOPTS "$CANDIDATE" web)
|
||||||
|
# Clean up the container however this ends, and dump its log ONLY
|
||||||
|
# on failure — a boot that never answers must fail with the reason
|
||||||
|
# visible rather than as a bare timeout (rule 156), while a green run
|
||||||
|
# has nothing to say. `exit $rc` preserves the real status, which a
|
||||||
|
# trap that ends on a successful `docker rm` would otherwise mask.
|
||||||
|
trap 'rc=$?; [ $rc -eq 0 ] || docker logs "$CID" 2>&1 | tail -40; docker rm -f "$CID" >/dev/null 2>&1 || true; exit $rc' EXIT
|
||||||
|
WEB_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$CID")
|
||||||
|
test -n "$WEB_IP"
|
||||||
|
healthy=""
|
||||||
|
for i in $(seq 1 60); do
|
||||||
|
if curl -fsS --max-time 5 "http://$WEB_IP:8080/api/health" >/dev/null 2>&1; then
|
||||||
|
healthy=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
# A container that has EXITED will never answer, so stop asking.
|
||||||
|
# Without this the loop spent 3m35s polling a dead container on
|
||||||
|
# this job's first run, and — because docker recycles the IP — got
|
||||||
|
# a confusing mix of connection-refused and 5s timeouts from
|
||||||
|
# whatever took the address next. The trap's log dump had the real
|
||||||
|
# answer the whole time; this just stops burying it.
|
||||||
|
if [ "$(docker inspect -f '{{.State.Running}}' "$CID" 2>/dev/null)" != "true" ]; then
|
||||||
|
echo "smoke: FAILED — the web container exited during boot." >&2
|
||||||
|
echo "smoke: its log follows; entrypoint runs alembic BEFORE" >&2
|
||||||
|
echo "smoke: serving, so a startup exception lands here." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
if [ -z "$healthy" ]; then
|
||||||
|
# 60 iterations of (up to 5s connect + 2s sleep) — up to ~7min, not
|
||||||
|
# the 120s an earlier version of this message claimed.
|
||||||
|
echo "smoke: FAILED — web is running but never answered" >&2
|
||||||
|
echo "smoke: /api/health. It is up, so look at hypercorn and the" >&2
|
||||||
|
echo "smoke: python base rather than at startup." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
curl -fsS --max-time 5 "http://$WEB_IP:8080/api/health"
|
||||||
|
echo
|
||||||
|
|
||||||
|
echo "smoke: all checks passed against $CANDIDATE"
|
||||||
|
|
||||||
|
# Move the channel tags — the whole point of the gate.
|
||||||
|
#
|
||||||
|
# Lives in its own job because the verdict it depends on cannot exist until
|
||||||
|
# after build-web has finished, and the promote used to run INSIDE build-web.
|
||||||
|
#
|
||||||
|
# `needs` on smoke-web is the gate. A failed smoke skips this job, so a
|
||||||
|
# refresh that broke something leaves :latest naming the build that works —
|
||||||
|
# "the refresh failed" and "production is broken" must not be the same event.
|
||||||
|
# A SKIPPED smoke also skips this job, which is the behaviour that matters
|
||||||
|
# most: on run 5290 the gate silently skipped itself, and a design where only
|
||||||
|
# a FAILED gate blocks would have published unverified images while reporting
|
||||||
|
# success. Not running is not the same as passing.
|
||||||
|
#
|
||||||
|
# All three images promote TOGETHER, or none do. They are one stack: build.yml
|
||||||
|
# already refuses to publish a :dev web image beside a stale :dev ml, because
|
||||||
|
# the mismatch only shows up as a runtime failure. A refresh that published ml
|
||||||
|
# and withheld web would be that same trap, arrived at through the gate.
|
||||||
|
#
|
||||||
|
# The gate covers the web image only (milestone 362 step 3 scoped it there),
|
||||||
|
# so ml and agent are being held to web's verdict rather than their own. That
|
||||||
|
# is deliberate and it is the conservative direction — they ship together, so
|
||||||
|
# the weakest evidence should govern all three — but it is not the same as
|
||||||
|
# having smoked them, and it should not be read as if it were.
|
||||||
|
promote:
|
||||||
|
needs: [build-web, build-ml, build-agent, smoke-web]
|
||||||
|
# Only a refresh publishes through a candidate; a push writes its channel
|
||||||
|
# tag directly from the build. Reads the same reuse-step decision the build
|
||||||
|
# took, via a job output — a job's `if:` cannot see the `env` context.
|
||||||
|
if: needs.build-web.outputs.candidate == 'true'
|
||||||
|
runs-on: python-ci
|
||||||
|
container:
|
||||||
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
|
steps:
|
||||||
|
- name: Point the channel tags at the smoked candidates
|
||||||
|
env:
|
||||||
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
|
ACTOR: ${{ github.actor }}
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
# `latest` is not a guess: a refresh always builds `main` (BUILD_REF),
|
||||||
|
# and the "must have checked out main" guard in every build job fails
|
||||||
|
# the run if that did not hold. So the channel is main's.
|
||||||
|
TAG=latest
|
||||||
|
FAILED=""
|
||||||
|
|
||||||
|
for NAME in fabledcurator fabledcurator-ml fabledcurator-agent; do
|
||||||
|
REPO="bvandeusen/$NAME"
|
||||||
|
echo "promote: $REPO"
|
||||||
|
|
||||||
|
# Registry auth is its own token exchange — `docker login`
|
||||||
|
# authenticates the docker client, not curl. Deadline on every call
|
||||||
|
# (rule 156): a registry that stops answering must fail this step,
|
||||||
|
# not hang the weekly refresh until the job times out.
|
||||||
|
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
|
||||||
|
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
|
||||||
|
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
|
||||||
|
|
||||||
|
# Ask for the IMAGE manifest media types only. Offering the index
|
||||||
|
# types too would let the registry hand back an index if one ever
|
||||||
|
# existed at this tag, and we would faithfully copy the thing this
|
||||||
|
# whole approach exists to avoid creating.
|
||||||
|
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
|
||||||
|
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
|
||||||
|
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||||
|
"https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
|
||||||
|
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
|
||||||
|
test -n "$CT"
|
||||||
|
SRC=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||||
|
echo "promote: candidate $SRC ($CT)"
|
||||||
|
|
||||||
|
# NOT `imagetools create`. That wraps its source in an INDEX, and
|
||||||
|
# `.Image.Config.Labels` does not resolve through one — the
|
||||||
|
# fc.revision the reuse check reads off the channel tag would come
|
||||||
|
# back empty, every later push would miss and rebuild, and nothing
|
||||||
|
# would go red (#3183, run 4751). A manifest PUT is what "make this
|
||||||
|
# tag name that image" means at the registry: same bytes, same media
|
||||||
|
# type, same digest, no layer transfer.
|
||||||
|
curl -fsS --max-time 120 -X PUT \
|
||||||
|
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
|
||||||
|
--data-binary @manifest.json \
|
||||||
|
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
|
||||||
|
|
||||||
|
# Read it back. A PUT that returned 2xx but landed something else is
|
||||||
|
# exactly the silent-and-plausible failure this pipeline keeps
|
||||||
|
# producing, and the check costs one request.
|
||||||
|
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
|
||||||
|
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||||
|
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
|
||||||
|
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||||
|
if [ "$NOW" != "$SRC" ]; then
|
||||||
|
echo "promote: FAILED — $NAME:$TAG is $NOW, expected $SRC" >&2
|
||||||
|
FAILED="$FAILED $NAME"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
echo "promote: $NAME:$TAG now names $NOW"
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ -n "$FAILED" ]; then
|
||||||
|
echo "" >&2
|
||||||
|
echo "promote: FAILED for:$FAILED" >&2
|
||||||
|
echo "promote: the channel tags are now INCONSISTENT — some images" >&2
|
||||||
|
echo "promote: moved and some did not. Re-run this refresh; the" >&2
|
||||||
|
echo "promote: candidates are still published and the promote is" >&2
|
||||||
|
echo "promote: idempotent." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "promote: all three channel tags moved"
|
||||||
|
|
||||||
build-ml:
|
build-ml:
|
||||||
runs-on: python-ci
|
runs-on: python-ci
|
||||||
container:
|
container:
|
||||||
@@ -957,7 +1345,7 @@ jobs:
|
|||||||
|
|
||||||
# See sign-extension's copy for why this guard exists.
|
# See sign-extension's copy for why this guard exists.
|
||||||
- name: Guard — a scheduled run must have checked out main
|
- name: Guard — a scheduled run must have checked out main
|
||||||
if: github.event_name == 'schedule'
|
if: env.IS_REFRESH == 'true'
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||||
@@ -990,8 +1378,18 @@ jobs:
|
|||||||
# the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
|
# the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
|
||||||
# * dev and main derive the same values for the same source.
|
# * dev and main derive the same values for the same source.
|
||||||
- name: Report the derived artifact version
|
- name: Report the derived artifact version
|
||||||
|
env:
|
||||||
|
# Diagnostic for the trigger normalisation. `refresh` is reported RAW
|
||||||
|
# as well as normalised, because the two disagreeing is the whole
|
||||||
|
# failure mode: a dispatch input whose type does not compare the way
|
||||||
|
# the expression assumes evaluates to false silently, and the only
|
||||||
|
# symptom is a refresh that quietly behaves like an ordinary push.
|
||||||
|
RAW_REFRESH: ${{ github.event.inputs.refresh }}
|
||||||
|
RAW_FORCE: ${{ github.event.inputs.force_build }}
|
||||||
run: |
|
run: |
|
||||||
set -u
|
set -u
|
||||||
|
echo "trigger: event=$GITHUB_EVENT_NAME IS_REFRESH='${IS_REFRESH:-<unset>}' BUILD_REF='${BUILD_REF:-<unset>}'"
|
||||||
|
echo "trigger: raw inputs refresh='${RAW_REFRESH:-<unset>}' force_build='${RAW_FORCE:-<unset>}'"
|
||||||
A=ml
|
A=ml
|
||||||
V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
|
V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
|
||||||
R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
|
R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
|
||||||
@@ -1008,7 +1406,7 @@ jobs:
|
|||||||
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||||
# Mirrors build-web's tag list and its schedule handling; see
|
# Mirrors build-web's tag list and its schedule handling; see
|
||||||
# the comments there.
|
# the comments there.
|
||||||
if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
|
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
||||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||||
@@ -1091,17 +1489,63 @@ jobs:
|
|||||||
# A scheduled refresh has to bypass reuse by construction: it
|
# A scheduled refresh has to bypass reuse by construction: it
|
||||||
# rebuilds the SAME source, so fc.revision always matches and the
|
# rebuilds the SAME source, so fc.revision always matches and the
|
||||||
# check would skip every refresh there has ever been.
|
# check would skip every refresh there has ever been.
|
||||||
EVENT: ${{ github.event_name }}
|
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
DERIVED=$(sh scripts/artifacts.sh revision ml)
|
DERIVED=$(sh scripts/artifacts.sh revision ml)
|
||||||
echo "revision=$DERIVED" >> "$GITHUB_OUTPUT"
|
echo "revision=$DERIVED" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
# The build clock, pinned to the same commit (#3265). Without it
|
||||||
|
# buildkit stamps the image config with the wall clock of the build,
|
||||||
|
# so identical layers republish under a new config blob and the
|
||||||
|
# channel tag gets a new manifest digest for no reason. Derived from
|
||||||
|
# `newest()` like revision and version, so all three name one commit
|
||||||
|
# and cannot drift apart.
|
||||||
|
echo "epoch=$(sh scripts/artifacts.sh epoch ml)" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
# The moving tag for this channel. Which tag we ask IS the channel —
|
# The moving tag for this channel. Which tag we ask IS the channel —
|
||||||
# that is why the revision needs no -main/-dev qualifier any more.
|
# that is why the revision needs no -main/-dev qualifier any more.
|
||||||
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
|
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
|
||||||
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||||
|
# whether the channel then has to be written separately.
|
||||||
|
#
|
||||||
|
# On a push the build writes the channel tag directly: the bytes came
|
||||||
|
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
||||||
|
# it behind.
|
||||||
|
#
|
||||||
|
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
||||||
|
# refresh rebuilds against freshly resolved base images, and the web
|
||||||
|
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||||
|
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||||
|
# Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and
|
||||||
|
# install requirements.txt, and a base bump changes neither. So
|
||||||
|
# refreshed bytes have to be proven before :latest names them, and
|
||||||
|
# proving needs a moment between "built" and "published" to occupy.
|
||||||
|
# This is that moment; :latest goes on naming the build that works
|
||||||
|
# until something says otherwise.
|
||||||
|
#
|
||||||
|
# `:refresh-candidate` is one moving ref per image, overwritten in
|
||||||
|
# place, holding a build nobody is told to pull — the shape rule 145
|
||||||
|
# already allows for :buildcache, not the per-build tag family that
|
||||||
|
# milestone 318 withdrew.
|
||||||
|
#
|
||||||
|
# Decided HERE, beside `hit`, for the reason the force/schedule
|
||||||
|
# branch below gives: one step decides what this job does. A
|
||||||
|
# condition derived independently could disagree with the tag the
|
||||||
|
# build actually wrote.
|
||||||
|
#
|
||||||
|
# build-web additionally exposes this as `outputs.candidate`, which is
|
||||||
|
# what gates the `promote` job — a job's `if:` cannot read `env`, and
|
||||||
|
# one flag is enough because all three derive it from the same
|
||||||
|
# IS_REFRESH. ml and agent do not re-emit it; a second copy nothing
|
||||||
|
# reads is the kind of thing that later reads as load-bearing.
|
||||||
|
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
|
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
||||||
# (run 4732): a missing key returns an empty string and exits 0, so
|
# (run 4732): a missing key returns an empty string and exits 0, so
|
||||||
# branching on the exit code would read "no label yet" as success.
|
# branching on the exit code would read "no label yet" as success.
|
||||||
@@ -1134,7 +1578,7 @@ jobs:
|
|||||||
if [ "${FORCE:-false}" = "true" ]; then
|
if [ "${FORCE:-false}" = "true" ]; then
|
||||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||||
echo "reuse: force_build set — building regardless"
|
echo "reuse: force_build set — building regardless"
|
||||||
elif [ "${EVENT:-}" = "schedule" ]; then
|
elif [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||||
echo "reuse: scheduled base refresh — building regardless"
|
echo "reuse: scheduled base refresh — building regardless"
|
||||||
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||||
@@ -1147,6 +1591,12 @@ jobs:
|
|||||||
|
|
||||||
- name: Build and push ml image
|
- name: Build and push ml image
|
||||||
if: steps.reuse.outputs.hit != 'true'
|
if: steps.reuse.outputs.hit != 'true'
|
||||||
|
# Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
|
||||||
|
# it normalises the image config's `created` field and the history
|
||||||
|
# timestamps rather than being consumed by the Dockerfile. See #3265
|
||||||
|
# and the reuse step's `epoch` output.
|
||||||
|
env:
|
||||||
|
SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }}
|
||||||
uses: docker/build-push-action@v5
|
uses: docker/build-push-action@v5
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
@@ -1159,20 +1609,17 @@ jobs:
|
|||||||
# invalidates, and the image genuinely rebuilds.
|
# invalidates, and the image genuinely rebuilds.
|
||||||
#
|
#
|
||||||
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
||||||
# did NOT move, the build is ~13s and every content step reports
|
# did NOT move, the build was ~13s with every content step CACHED —
|
||||||
# CACHED — but the channel tag STILL gets a new manifest digest.
|
# and the channel tag STILL got a new manifest digest, because
|
||||||
# buildkit mints a fresh image config each run, so identical layers
|
# buildkit stamps a fresh image config per run and republishes the
|
||||||
# are republished under a new config blob. All three images moved
|
# identical layers under it. All three images moved that way on
|
||||||
# that way on 2026-08-30 with nothing whatsoever changed in them.
|
# 2026-08-30 with nothing whatsoever changed in them.
|
||||||
#
|
#
|
||||||
# So a refresh currently rewrites :latest every Sunday whether or
|
# SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the
|
||||||
# not there is anything new in it, and :c-<sha> is handed a new
|
# content came from, the config is byte-identical across runs, so
|
||||||
# manifest to diverge from on the same cadence. Layers are shared,
|
# the manifest digest is too and the push is a registry no-op. A
|
||||||
# so the storage cost is a config blob; the cost that matters is
|
# digest change means the content changed again, which is the only
|
||||||
# that a digest change no longer MEANS anything. Tracked in #3265 —
|
# thing a digest is any use for.
|
||||||
# the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
|
|
||||||
# make "same source, same bytes" true and turn the no-op case into
|
|
||||||
# a genuine no-op.
|
|
||||||
#
|
#
|
||||||
# What `pull` does NOT catch either: a Debian package update inside
|
# What `pull` does NOT catch either: a Debian package update inside
|
||||||
# the `apt-get install` layer while the base tag itself stands
|
# the `apt-get install` layer while the base tag itself stands
|
||||||
@@ -1182,14 +1629,14 @@ jobs:
|
|||||||
# churn #3265 is about.
|
# churn #3265 is about.
|
||||||
#
|
#
|
||||||
# Only on the schedule. An ordinary push wants the cached base.
|
# Only on the schedule. An ordinary push wants the cached base.
|
||||||
pull: ${{ github.event_name == 'schedule' }}
|
pull: ${{ env.IS_REFRESH == 'true' }}
|
||||||
# ONE tag, the channel's. Every other tag is written by the step
|
# ONE tag, the channel's. Every other tag is written by the step
|
||||||
# below, registry-side. buildx here pushes the first tag to the
|
# below, registry-side. buildx here pushes the first tag to the
|
||||||
# registry and then re-pushes the rest through the DOCKER driver,
|
# registry and then re-pushes the rest through the DOCKER driver,
|
||||||
# out of a local image store a registry-direct build never filled —
|
# out of a local image store a registry-direct build never filled —
|
||||||
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
|
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
|
||||||
# published perfectly well.
|
# published perfectly well.
|
||||||
tags: ${{ steps.reuse.outputs.channel_ref }}
|
tags: ${{ steps.reuse.outputs.build_ref }}
|
||||||
# The reuse key. Read back off the channel tag on the next push to
|
# The reuse key. Read back off the channel tag on the next push to
|
||||||
# decide whether that push needs to build at all, so this is not
|
# decide whether that push needs to build at all, so this is not
|
||||||
# decoration — an unstamped image is one that will always rebuild.
|
# decoration — an unstamped image is one that will always rebuild.
|
||||||
@@ -1331,7 +1778,7 @@ jobs:
|
|||||||
|
|
||||||
# See sign-extension's copy for why this guard exists.
|
# See sign-extension's copy for why this guard exists.
|
||||||
- name: Guard — a scheduled run must have checked out main
|
- name: Guard — a scheduled run must have checked out main
|
||||||
if: github.event_name == 'schedule'
|
if: env.IS_REFRESH == 'true'
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||||
@@ -1364,8 +1811,18 @@ jobs:
|
|||||||
# the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
|
# the PREVIOUS XPI while the freshly signed one is orphaned (#3156).
|
||||||
# * dev and main derive the same values for the same source.
|
# * dev and main derive the same values for the same source.
|
||||||
- name: Report the derived artifact version
|
- name: Report the derived artifact version
|
||||||
|
env:
|
||||||
|
# Diagnostic for the trigger normalisation. `refresh` is reported RAW
|
||||||
|
# as well as normalised, because the two disagreeing is the whole
|
||||||
|
# failure mode: a dispatch input whose type does not compare the way
|
||||||
|
# the expression assumes evaluates to false silently, and the only
|
||||||
|
# symptom is a refresh that quietly behaves like an ordinary push.
|
||||||
|
RAW_REFRESH: ${{ github.event.inputs.refresh }}
|
||||||
|
RAW_FORCE: ${{ github.event.inputs.force_build }}
|
||||||
run: |
|
run: |
|
||||||
set -u
|
set -u
|
||||||
|
echo "trigger: event=$GITHUB_EVENT_NAME IS_REFRESH='${IS_REFRESH:-<unset>}' BUILD_REF='${BUILD_REF:-<unset>}'"
|
||||||
|
echo "trigger: raw inputs refresh='${RAW_REFRESH:-<unset>}' force_build='${RAW_FORCE:-<unset>}'"
|
||||||
A=agent
|
A=agent
|
||||||
V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
|
V=$(sh scripts/artifacts.sh version "$A" 2>&1 || echo UNAVAILABLE)
|
||||||
R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
|
R=$(sh scripts/artifacts.sh revision "$A" 2>&1 || echo UNAVAILABLE)
|
||||||
@@ -1377,7 +1834,7 @@ jobs:
|
|||||||
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||||
# Mirrors build-web's tag list and its schedule handling; see
|
# Mirrors build-web's tag list and its schedule handling; see
|
||||||
# the comments there.
|
# the comments there.
|
||||||
if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
|
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:latest" >> "$GITHUB_OUTPUT"
|
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:latest" >> "$GITHUB_OUTPUT"
|
||||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||||
@@ -1460,17 +1917,63 @@ jobs:
|
|||||||
# A scheduled refresh has to bypass reuse by construction: it
|
# A scheduled refresh has to bypass reuse by construction: it
|
||||||
# rebuilds the SAME source, so fc.revision always matches and the
|
# rebuilds the SAME source, so fc.revision always matches and the
|
||||||
# check would skip every refresh there has ever been.
|
# check would skip every refresh there has ever been.
|
||||||
EVENT: ${{ github.event_name }}
|
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
DERIVED=$(sh scripts/artifacts.sh revision agent)
|
DERIVED=$(sh scripts/artifacts.sh revision agent)
|
||||||
echo "revision=$DERIVED" >> "$GITHUB_OUTPUT"
|
echo "revision=$DERIVED" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
# The build clock, pinned to the same commit (#3265). Without it
|
||||||
|
# buildkit stamps the image config with the wall clock of the build,
|
||||||
|
# so identical layers republish under a new config blob and the
|
||||||
|
# channel tag gets a new manifest digest for no reason. Derived from
|
||||||
|
# `newest()` like revision and version, so all three name one commit
|
||||||
|
# and cannot drift apart.
|
||||||
|
echo "epoch=$(sh scripts/artifacts.sh epoch agent)" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
# The moving tag for this channel. Which tag we ask IS the channel —
|
# The moving tag for this channel. Which tag we ask IS the channel —
|
||||||
# that is why the revision needs no -main/-dev qualifier any more.
|
# that is why the revision needs no -main/-dev qualifier any more.
|
||||||
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
|
if [ "$CHANNEL" = "main" ]; then T=latest; else T=dev; fi
|
||||||
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
echo "channel_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||||
|
|
||||||
|
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||||
|
# whether the channel then has to be written separately.
|
||||||
|
#
|
||||||
|
# On a push the build writes the channel tag directly: the bytes came
|
||||||
|
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
||||||
|
# it behind.
|
||||||
|
#
|
||||||
|
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
||||||
|
# refresh rebuilds against freshly resolved base images, and the web
|
||||||
|
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||||
|
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||||
|
# Nothing in ci.yml can see that: its lanes run on ci-python:3.14 and
|
||||||
|
# install requirements.txt, and a base bump changes neither. So
|
||||||
|
# refreshed bytes have to be proven before :latest names them, and
|
||||||
|
# proving needs a moment between "built" and "published" to occupy.
|
||||||
|
# This is that moment; :latest goes on naming the build that works
|
||||||
|
# until something says otherwise.
|
||||||
|
#
|
||||||
|
# `:refresh-candidate` is one moving ref per image, overwritten in
|
||||||
|
# place, holding a build nobody is told to pull — the shape rule 145
|
||||||
|
# already allows for :buildcache, not the per-build tag family that
|
||||||
|
# milestone 318 withdrew.
|
||||||
|
#
|
||||||
|
# Decided HERE, beside `hit`, for the reason the force/schedule
|
||||||
|
# branch below gives: one step decides what this job does. A
|
||||||
|
# condition derived independently could disagree with the tag the
|
||||||
|
# build actually wrote.
|
||||||
|
#
|
||||||
|
# build-web additionally exposes this as `outputs.candidate`, which is
|
||||||
|
# what gates the `promote` job — a job's `if:` cannot read `env`, and
|
||||||
|
# one flag is enough because all three derive it from the same
|
||||||
|
# IS_REFRESH. ml and agent do not re-emit it; a second copy nothing
|
||||||
|
# reads is the kind of thing that later reads as load-bearing.
|
||||||
|
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
|
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||||
|
fi
|
||||||
|
|
||||||
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
||||||
# (run 4732): a missing key returns an empty string and exits 0, so
|
# (run 4732): a missing key returns an empty string and exits 0, so
|
||||||
# branching on the exit code would read "no label yet" as success.
|
# branching on the exit code would read "no label yet" as success.
|
||||||
@@ -1503,7 +2006,7 @@ jobs:
|
|||||||
if [ "${FORCE:-false}" = "true" ]; then
|
if [ "${FORCE:-false}" = "true" ]; then
|
||||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||||
echo "reuse: force_build set — building regardless"
|
echo "reuse: force_build set — building regardless"
|
||||||
elif [ "${EVENT:-}" = "schedule" ]; then
|
elif [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||||
echo "reuse: scheduled base refresh — building regardless"
|
echo "reuse: scheduled base refresh — building regardless"
|
||||||
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||||
@@ -1516,6 +2019,12 @@ jobs:
|
|||||||
|
|
||||||
- name: Build and push agent image
|
- name: Build and push agent image
|
||||||
if: steps.reuse.outputs.hit != 'true'
|
if: steps.reuse.outputs.hit != 'true'
|
||||||
|
# Read by buildx out of the ENVIRONMENT, not passed as a build-arg —
|
||||||
|
# it normalises the image config's `created` field and the history
|
||||||
|
# timestamps rather than being consumed by the Dockerfile. See #3265
|
||||||
|
# and the reuse step's `epoch` output.
|
||||||
|
env:
|
||||||
|
SOURCE_DATE_EPOCH: ${{ steps.reuse.outputs.epoch }}
|
||||||
uses: docker/build-push-action@v5
|
uses: docker/build-push-action@v5
|
||||||
with:
|
with:
|
||||||
context: agent
|
context: agent
|
||||||
@@ -1528,20 +2037,17 @@ jobs:
|
|||||||
# invalidates, and the image genuinely rebuilds.
|
# invalidates, and the image genuinely rebuilds.
|
||||||
#
|
#
|
||||||
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
||||||
# did NOT move, the build is ~13s and every content step reports
|
# did NOT move, the build was ~13s with every content step CACHED —
|
||||||
# CACHED — but the channel tag STILL gets a new manifest digest.
|
# and the channel tag STILL got a new manifest digest, because
|
||||||
# buildkit mints a fresh image config each run, so identical layers
|
# buildkit stamps a fresh image config per run and republishes the
|
||||||
# are republished under a new config blob. All three images moved
|
# identical layers under it. All three images moved that way on
|
||||||
# that way on 2026-08-30 with nothing whatsoever changed in them.
|
# 2026-08-30 with nothing whatsoever changed in them.
|
||||||
#
|
#
|
||||||
# So a refresh currently rewrites :latest every Sunday whether or
|
# SOURCE_DATE_EPOCH (below) is the fix: pinned to the commit the
|
||||||
# not there is anything new in it, and :c-<sha> is handed a new
|
# content came from, the config is byte-identical across runs, so
|
||||||
# manifest to diverge from on the same cadence. Layers are shared,
|
# the manifest digest is too and the push is a registry no-op. A
|
||||||
# so the storage cost is a config blob; the cost that matters is
|
# digest change means the content changed again, which is the only
|
||||||
# that a digest change no longer MEANS anything. Tracked in #3265 —
|
# thing a digest is any use for.
|
||||||
# the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
|
|
||||||
# make "same source, same bytes" true and turn the no-op case into
|
|
||||||
# a genuine no-op.
|
|
||||||
#
|
#
|
||||||
# What `pull` does NOT catch either: a Debian package update inside
|
# What `pull` does NOT catch either: a Debian package update inside
|
||||||
# the `apt-get install` layer while the base tag itself stands
|
# the `apt-get install` layer while the base tag itself stands
|
||||||
@@ -1551,14 +2057,14 @@ jobs:
|
|||||||
# churn #3265 is about.
|
# churn #3265 is about.
|
||||||
#
|
#
|
||||||
# Only on the schedule. An ordinary push wants the cached base.
|
# Only on the schedule. An ordinary push wants the cached base.
|
||||||
pull: ${{ github.event_name == 'schedule' }}
|
pull: ${{ env.IS_REFRESH == 'true' }}
|
||||||
# ONE tag, the channel's. Every other tag is written by the step
|
# ONE tag, the channel's. Every other tag is written by the step
|
||||||
# below, registry-side. buildx here pushes the first tag to the
|
# below, registry-side. buildx here pushes the first tag to the
|
||||||
# registry and then re-pushes the rest through the DOCKER driver,
|
# registry and then re-pushes the rest through the DOCKER driver,
|
||||||
# out of a local image store a registry-direct build never filled —
|
# out of a local image store a registry-direct build never filled —
|
||||||
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
|
# #3190, which cost `main` its :c-<sha> on 2026-08-29 while :latest
|
||||||
# published perfectly well.
|
# published perfectly well.
|
||||||
tags: ${{ steps.reuse.outputs.channel_ref }}
|
tags: ${{ steps.reuse.outputs.build_ref }}
|
||||||
# The reuse key. Read back off the channel tag on the next push to
|
# The reuse key. Read back off the channel tag on the next push to
|
||||||
# decide whether that push needs to build at all, so this is not
|
# decide whether that push needs to build at all, so this is not
|
||||||
# decoration — an unstamped image is one that will always rebuild.
|
# decoration — an unstamped image is one that will always rebuild.
|
||||||
|
|||||||
@@ -255,10 +255,27 @@ jobs:
|
|||||||
export DB_HOST="$PG_IP"
|
export DB_HOST="$PG_IP"
|
||||||
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
export CELERY_BROKER_URL="redis://$RD_IP:6379/0"
|
||||||
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
export CELERY_RESULT_BACKEND="redis://$RD_IP:6379/0"
|
||||||
|
# These steps run under `sh -e`, not bash, so bash's /dev/tcp magic
|
||||||
|
# path does not exist here — the probe this loop used to run could
|
||||||
|
# never succeed and simply burned the full 120s on every run, green
|
||||||
|
# or red, then continued without having established anything. Python
|
||||||
|
# is in the image and needs no installed package for a socket
|
||||||
|
# connect, so it is the probe. Exhausting the budget is now a named
|
||||||
|
# failure rather than a silent fall-through (rule 156): if Postgres
|
||||||
|
# is genuinely not up, that is what the log should say, instead of
|
||||||
|
# whatever the first query happens to raise two minutes later.
|
||||||
|
pg_ready=""
|
||||||
for i in $(seq 1 60); do
|
for i in $(seq 1 60); do
|
||||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('$PG_IP', 5432)) == 0 else 1)"; then
|
||||||
|
pg_ready=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
sleep 2
|
sleep 2
|
||||||
done
|
done
|
||||||
|
if [ -z "$pg_ready" ]; then
|
||||||
|
echo "postgres at $PG_IP:5432 did not accept a connection within 120s"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
if command -v uv >/dev/null 2>&1; then
|
if command -v uv >/dev/null 2>&1; then
|
||||||
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
uv pip install --system -r requirements.txt pytest pytest-asyncio
|
||||||
else
|
else
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
# Contributing
|
||||||
|
|
||||||
|
FabledCurator is developed by a single maintainer for their own use, and
|
||||||
|
published because it may be useful to others. That shapes what contribution
|
||||||
|
looks like here.
|
||||||
|
|
||||||
|
**Issues are welcome** — bug reports, and questions about running it, are
|
||||||
|
genuinely useful and often the fastest way to find out that something is
|
||||||
|
broken outside the one environment it was built in.
|
||||||
|
|
||||||
|
**Open an issue before writing a pull request.** Not as a formality: the
|
||||||
|
project has opinions that are not obvious from the code, and it is unpleasant
|
||||||
|
for everyone when a finished patch turns out to conflict with one. A short
|
||||||
|
issue first costs you nothing and may save you an evening.
|
||||||
|
|
||||||
|
**Contributions are licensed under the AGPL-3.0**, like the rest of the
|
||||||
|
project. By submitting one you agree it ships under that licence. There is no
|
||||||
|
CLA and no copyright assignment.
|
||||||
|
|
||||||
|
## Running it for development
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d # UI on http://localhost:8080
|
||||||
|
```
|
||||||
|
|
||||||
|
The dev override (`docker-compose.override.yml`) is auto-merged and builds the
|
||||||
|
app images locally from source, so this needs no `.env` and no registry
|
||||||
|
access. Postgres and Redis ports are exposed on the host.
|
||||||
|
|
||||||
|
## What CI checks
|
||||||
|
|
||||||
|
Every push runs these, and they are the definition of done for a change:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ruff check backend/ tests/ alembic/ agent/ scripts/ # lint (and import order)
|
||||||
|
pytest tests/ -m "not integration" # backend unit tests
|
||||||
|
pytest tests/ -m integration # needs pgvector + redis
|
||||||
|
cd frontend && npm run test:unit && npm run build # frontend
|
||||||
|
```
|
||||||
|
|
||||||
|
The integration lane builds its schema by running the real migrations
|
||||||
|
(`alembic upgrade head`), never from ORM metadata — so a migration that does
|
||||||
|
not apply cleanly fails CI rather than being discovered later.
|
||||||
|
|
||||||
|
Note for the linter: ruff's isort runs with `order-by-type`, which sorts
|
||||||
|
ALL-CAPS names ahead of CamelCase. `from sqlalchemy import JSON, DateTime, ...`
|
||||||
|
is correct; putting `JSON` alphabetically between `Integer` and `String` is
|
||||||
|
not. This catches people out.
|
||||||
|
|
||||||
|
## Database changes
|
||||||
|
|
||||||
|
The ORM models and the migration chain must agree. This is enforced, and it is
|
||||||
|
enforced because they silently diverged for a long time and nobody noticed
|
||||||
|
until they were compared: the models were missing indexes, defaults and
|
||||||
|
uniqueness guarantees that only ever existed inside a migration, which made
|
||||||
|
`alembic revision --autogenerate` actively unsafe to run.
|
||||||
|
|
||||||
|
So: if you change a model, write the migration; if you write a migration,
|
||||||
|
change the model to match. Both, in the same commit.
|
||||||
|
|
||||||
|
Adding a value to a CHECK-constrained column means swapping the constraint in
|
||||||
|
the same change — the constraint is not documentation, and a new value without
|
||||||
|
it fails at insert time.
|
||||||
|
|
||||||
|
## Branch model
|
||||||
|
|
||||||
|
`dev` is where work happens. `main` is production and is only reached by a
|
||||||
|
merge from `dev`, never pushed to directly. If you are sending a pull request,
|
||||||
|
target `dev`.
|
||||||
|
|
||||||
|
## Style
|
||||||
|
|
||||||
|
Match the surrounding code. The one convention worth stating explicitly is
|
||||||
|
that comments here explain *why*, especially where a choice looks wrong at a
|
||||||
|
glance — a comment recording which migration a constraint came from, or why a
|
||||||
|
default is a `text()` rather than a string, is the kind that has repeatedly
|
||||||
|
turned out to be worth its space.
|
||||||
@@ -0,0 +1,661 @@
|
|||||||
|
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||||
|
Version 3, 19 November 2007
|
||||||
|
|
||||||
|
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies
|
||||||
|
of this license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
Preamble
|
||||||
|
|
||||||
|
The GNU Affero General Public License is a free, copyleft license for
|
||||||
|
software and other kinds of works, specifically designed to ensure
|
||||||
|
cooperation with the community in the case of network server software.
|
||||||
|
|
||||||
|
The licenses for most software and other practical works are designed
|
||||||
|
to take away your freedom to share and change the works. By contrast,
|
||||||
|
our General Public Licenses are intended to guarantee your freedom to
|
||||||
|
share and change all versions of a program--to make sure it remains free
|
||||||
|
software for all its users.
|
||||||
|
|
||||||
|
When we speak of free software, we are referring to freedom, not
|
||||||
|
price. Our General Public Licenses are designed to make sure that you
|
||||||
|
have the freedom to distribute copies of free software (and charge for
|
||||||
|
them if you wish), that you receive source code or can get it if you
|
||||||
|
want it, that you can change the software or use pieces of it in new
|
||||||
|
free programs, and that you know you can do these things.
|
||||||
|
|
||||||
|
Developers that use our General Public Licenses protect your rights
|
||||||
|
with two steps: (1) assert copyright on the software, and (2) offer
|
||||||
|
you this License which gives you legal permission to copy, distribute
|
||||||
|
and/or modify the software.
|
||||||
|
|
||||||
|
A secondary benefit of defending all users' freedom is that
|
||||||
|
improvements made in alternate versions of the program, if they
|
||||||
|
receive widespread use, become available for other developers to
|
||||||
|
incorporate. Many developers of free software are heartened and
|
||||||
|
encouraged by the resulting cooperation. However, in the case of
|
||||||
|
software used on network servers, this result may fail to come about.
|
||||||
|
The GNU General Public License permits making a modified version and
|
||||||
|
letting the public access it on a server without ever releasing its
|
||||||
|
source code to the public.
|
||||||
|
|
||||||
|
The GNU Affero General Public License is designed specifically to
|
||||||
|
ensure that, in such cases, the modified source code becomes available
|
||||||
|
to the community. It requires the operator of a network server to
|
||||||
|
provide the source code of the modified version running there to the
|
||||||
|
users of that server. Therefore, public use of a modified version, on
|
||||||
|
a publicly accessible server, gives the public access to the source
|
||||||
|
code of the modified version.
|
||||||
|
|
||||||
|
An older license, called the Affero General Public License and
|
||||||
|
published by Affero, was designed to accomplish similar goals. This is
|
||||||
|
a different license, not a version of the Affero GPL, but Affero has
|
||||||
|
released a new version of the Affero GPL which permits relicensing under
|
||||||
|
this license.
|
||||||
|
|
||||||
|
The precise terms and conditions for copying, distribution and
|
||||||
|
modification follow.
|
||||||
|
|
||||||
|
TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
0. Definitions.
|
||||||
|
|
||||||
|
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||||
|
|
||||||
|
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||||
|
works, such as semiconductor masks.
|
||||||
|
|
||||||
|
"The Program" refers to any copyrightable work licensed under this
|
||||||
|
License. Each licensee is addressed as "you". "Licensees" and
|
||||||
|
"recipients" may be individuals or organizations.
|
||||||
|
|
||||||
|
To "modify" a work means to copy from or adapt all or part of the work
|
||||||
|
in a fashion requiring copyright permission, other than the making of an
|
||||||
|
exact copy. The resulting work is called a "modified version" of the
|
||||||
|
earlier work or a work "based on" the earlier work.
|
||||||
|
|
||||||
|
A "covered work" means either the unmodified Program or a work based
|
||||||
|
on the Program.
|
||||||
|
|
||||||
|
To "propagate" a work means to do anything with it that, without
|
||||||
|
permission, would make you directly or secondarily liable for
|
||||||
|
infringement under applicable copyright law, except executing it on a
|
||||||
|
computer or modifying a private copy. Propagation includes copying,
|
||||||
|
distribution (with or without modification), making available to the
|
||||||
|
public, and in some countries other activities as well.
|
||||||
|
|
||||||
|
To "convey" a work means any kind of propagation that enables other
|
||||||
|
parties to make or receive copies. Mere interaction with a user through
|
||||||
|
a computer network, with no transfer of a copy, is not conveying.
|
||||||
|
|
||||||
|
An interactive user interface displays "Appropriate Legal Notices"
|
||||||
|
to the extent that it includes a convenient and prominently visible
|
||||||
|
feature that (1) displays an appropriate copyright notice, and (2)
|
||||||
|
tells the user that there is no warranty for the work (except to the
|
||||||
|
extent that warranties are provided), that licensees may convey the
|
||||||
|
work under this License, and how to view a copy of this License. If
|
||||||
|
the interface presents a list of user commands or options, such as a
|
||||||
|
menu, a prominent item in the list meets this criterion.
|
||||||
|
|
||||||
|
1. Source Code.
|
||||||
|
|
||||||
|
The "source code" for a work means the preferred form of the work
|
||||||
|
for making modifications to it. "Object code" means any non-source
|
||||||
|
form of a work.
|
||||||
|
|
||||||
|
A "Standard Interface" means an interface that either is an official
|
||||||
|
standard defined by a recognized standards body, or, in the case of
|
||||||
|
interfaces specified for a particular programming language, one that
|
||||||
|
is widely used among developers working in that language.
|
||||||
|
|
||||||
|
The "System Libraries" of an executable work include anything, other
|
||||||
|
than the work as a whole, that (a) is included in the normal form of
|
||||||
|
packaging a Major Component, but which is not part of that Major
|
||||||
|
Component, and (b) serves only to enable use of the work with that
|
||||||
|
Major Component, or to implement a Standard Interface for which an
|
||||||
|
implementation is available to the public in source code form. A
|
||||||
|
"Major Component", in this context, means a major essential component
|
||||||
|
(kernel, window system, and so on) of the specific operating system
|
||||||
|
(if any) on which the executable work runs, or a compiler used to
|
||||||
|
produce the work, or an object code interpreter used to run it.
|
||||||
|
|
||||||
|
The "Corresponding Source" for a work in object code form means all
|
||||||
|
the source code needed to generate, install, and (for an executable
|
||||||
|
work) run the object code and to modify the work, including scripts to
|
||||||
|
control those activities. However, it does not include the work's
|
||||||
|
System Libraries, or general-purpose tools or generally available free
|
||||||
|
programs which are used unmodified in performing those activities but
|
||||||
|
which are not part of the work. For example, Corresponding Source
|
||||||
|
includes interface definition files associated with source files for
|
||||||
|
the work, and the source code for shared libraries and dynamically
|
||||||
|
linked subprograms that the work is specifically designed to require,
|
||||||
|
such as by intimate data communication or control flow between those
|
||||||
|
subprograms and other parts of the work.
|
||||||
|
|
||||||
|
The Corresponding Source need not include anything that users
|
||||||
|
can regenerate automatically from other parts of the Corresponding
|
||||||
|
Source.
|
||||||
|
|
||||||
|
The Corresponding Source for a work in source code form is that
|
||||||
|
same work.
|
||||||
|
|
||||||
|
2. Basic Permissions.
|
||||||
|
|
||||||
|
All rights granted under this License are granted for the term of
|
||||||
|
copyright on the Program, and are irrevocable provided the stated
|
||||||
|
conditions are met. This License explicitly affirms your unlimited
|
||||||
|
permission to run the unmodified Program. The output from running a
|
||||||
|
covered work is covered by this License only if the output, given its
|
||||||
|
content, constitutes a covered work. This License acknowledges your
|
||||||
|
rights of fair use or other equivalent, as provided by copyright law.
|
||||||
|
|
||||||
|
You may make, run and propagate covered works that you do not
|
||||||
|
convey, without conditions so long as your license otherwise remains
|
||||||
|
in force. You may convey covered works to others for the sole purpose
|
||||||
|
of having them make modifications exclusively for you, or provide you
|
||||||
|
with facilities for running those works, provided that you comply with
|
||||||
|
the terms of this License in conveying all material for which you do
|
||||||
|
not control copyright. Those thus making or running the covered works
|
||||||
|
for you must do so exclusively on your behalf, under your direction
|
||||||
|
and control, on terms that prohibit them from making any copies of
|
||||||
|
your copyrighted material outside their relationship with you.
|
||||||
|
|
||||||
|
Conveying under any other circumstances is permitted solely under
|
||||||
|
the conditions stated below. Sublicensing is not allowed; section 10
|
||||||
|
makes it unnecessary.
|
||||||
|
|
||||||
|
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||||
|
|
||||||
|
No covered work shall be deemed part of an effective technological
|
||||||
|
measure under any applicable law fulfilling obligations under article
|
||||||
|
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||||
|
similar laws prohibiting or restricting circumvention of such
|
||||||
|
measures.
|
||||||
|
|
||||||
|
When you convey a covered work, you waive any legal power to forbid
|
||||||
|
circumvention of technological measures to the extent such circumvention
|
||||||
|
is effected by exercising rights under this License with respect to
|
||||||
|
the covered work, and you disclaim any intention to limit operation or
|
||||||
|
modification of the work as a means of enforcing, against the work's
|
||||||
|
users, your or third parties' legal rights to forbid circumvention of
|
||||||
|
technological measures.
|
||||||
|
|
||||||
|
4. Conveying Verbatim Copies.
|
||||||
|
|
||||||
|
You may convey verbatim copies of the Program's source code as you
|
||||||
|
receive it, in any medium, provided that you conspicuously and
|
||||||
|
appropriately publish on each copy an appropriate copyright notice;
|
||||||
|
keep intact all notices stating that this License and any
|
||||||
|
non-permissive terms added in accord with section 7 apply to the code;
|
||||||
|
keep intact all notices of the absence of any warranty; and give all
|
||||||
|
recipients a copy of this License along with the Program.
|
||||||
|
|
||||||
|
You may charge any price or no price for each copy that you convey,
|
||||||
|
and you may offer support or warranty protection for a fee.
|
||||||
|
|
||||||
|
5. Conveying Modified Source Versions.
|
||||||
|
|
||||||
|
You may convey a work based on the Program, or the modifications to
|
||||||
|
produce it from the Program, in the form of source code under the
|
||||||
|
terms of section 4, provided that you also meet all of these conditions:
|
||||||
|
|
||||||
|
a) The work must carry prominent notices stating that you modified
|
||||||
|
it, and giving a relevant date.
|
||||||
|
|
||||||
|
b) The work must carry prominent notices stating that it is
|
||||||
|
released under this License and any conditions added under section
|
||||||
|
7. This requirement modifies the requirement in section 4 to
|
||||||
|
"keep intact all notices".
|
||||||
|
|
||||||
|
c) You must license the entire work, as a whole, under this
|
||||||
|
License to anyone who comes into possession of a copy. This
|
||||||
|
License will therefore apply, along with any applicable section 7
|
||||||
|
additional terms, to the whole of the work, and all its parts,
|
||||||
|
regardless of how they are packaged. This License gives no
|
||||||
|
permission to license the work in any other way, but it does not
|
||||||
|
invalidate such permission if you have separately received it.
|
||||||
|
|
||||||
|
d) If the work has interactive user interfaces, each must display
|
||||||
|
Appropriate Legal Notices; however, if the Program has interactive
|
||||||
|
interfaces that do not display Appropriate Legal Notices, your
|
||||||
|
work need not make them do so.
|
||||||
|
|
||||||
|
A compilation of a covered work with other separate and independent
|
||||||
|
works, which are not by their nature extensions of the covered work,
|
||||||
|
and which are not combined with it such as to form a larger program,
|
||||||
|
in or on a volume of a storage or distribution medium, is called an
|
||||||
|
"aggregate" if the compilation and its resulting copyright are not
|
||||||
|
used to limit the access or legal rights of the compilation's users
|
||||||
|
beyond what the individual works permit. Inclusion of a covered work
|
||||||
|
in an aggregate does not cause this License to apply to the other
|
||||||
|
parts of the aggregate.
|
||||||
|
|
||||||
|
6. Conveying Non-Source Forms.
|
||||||
|
|
||||||
|
You may convey a covered work in object code form under the terms
|
||||||
|
of sections 4 and 5, provided that you also convey the
|
||||||
|
machine-readable Corresponding Source under the terms of this License,
|
||||||
|
in one of these ways:
|
||||||
|
|
||||||
|
a) Convey the object code in, or embodied in, a physical product
|
||||||
|
(including a physical distribution medium), accompanied by the
|
||||||
|
Corresponding Source fixed on a durable physical medium
|
||||||
|
customarily used for software interchange.
|
||||||
|
|
||||||
|
b) Convey the object code in, or embodied in, a physical product
|
||||||
|
(including a physical distribution medium), accompanied by a
|
||||||
|
written offer, valid for at least three years and valid for as
|
||||||
|
long as you offer spare parts or customer support for that product
|
||||||
|
model, to give anyone who possesses the object code either (1) a
|
||||||
|
copy of the Corresponding Source for all the software in the
|
||||||
|
product that is covered by this License, on a durable physical
|
||||||
|
medium customarily used for software interchange, for a price no
|
||||||
|
more than your reasonable cost of physically performing this
|
||||||
|
conveying of source, or (2) access to copy the
|
||||||
|
Corresponding Source from a network server at no charge.
|
||||||
|
|
||||||
|
c) Convey individual copies of the object code with a copy of the
|
||||||
|
written offer to provide the Corresponding Source. This
|
||||||
|
alternative is allowed only occasionally and noncommercially, and
|
||||||
|
only if you received the object code with such an offer, in accord
|
||||||
|
with subsection 6b.
|
||||||
|
|
||||||
|
d) Convey the object code by offering access from a designated
|
||||||
|
place (gratis or for a charge), and offer equivalent access to the
|
||||||
|
Corresponding Source in the same way through the same place at no
|
||||||
|
further charge. You need not require recipients to copy the
|
||||||
|
Corresponding Source along with the object code. If the place to
|
||||||
|
copy the object code is a network server, the Corresponding Source
|
||||||
|
may be on a different server (operated by you or a third party)
|
||||||
|
that supports equivalent copying facilities, provided you maintain
|
||||||
|
clear directions next to the object code saying where to find the
|
||||||
|
Corresponding Source. Regardless of what server hosts the
|
||||||
|
Corresponding Source, you remain obligated to ensure that it is
|
||||||
|
available for as long as needed to satisfy these requirements.
|
||||||
|
|
||||||
|
e) Convey the object code using peer-to-peer transmission, provided
|
||||||
|
you inform other peers where the object code and Corresponding
|
||||||
|
Source of the work are being offered to the general public at no
|
||||||
|
charge under subsection 6d.
|
||||||
|
|
||||||
|
A separable portion of the object code, whose source code is excluded
|
||||||
|
from the Corresponding Source as a System Library, need not be
|
||||||
|
included in conveying the object code work.
|
||||||
|
|
||||||
|
A "User Product" is either (1) a "consumer product", which means any
|
||||||
|
tangible personal property which is normally used for personal, family,
|
||||||
|
or household purposes, or (2) anything designed or sold for incorporation
|
||||||
|
into a dwelling. In determining whether a product is a consumer product,
|
||||||
|
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||||
|
product received by a particular user, "normally used" refers to a
|
||||||
|
typical or common use of that class of product, regardless of the status
|
||||||
|
of the particular user or of the way in which the particular user
|
||||||
|
actually uses, or expects or is expected to use, the product. A product
|
||||||
|
is a consumer product regardless of whether the product has substantial
|
||||||
|
commercial, industrial or non-consumer uses, unless such uses represent
|
||||||
|
the only significant mode of use of the product.
|
||||||
|
|
||||||
|
"Installation Information" for a User Product means any methods,
|
||||||
|
procedures, authorization keys, or other information required to install
|
||||||
|
and execute modified versions of a covered work in that User Product from
|
||||||
|
a modified version of its Corresponding Source. The information must
|
||||||
|
suffice to ensure that the continued functioning of the modified object
|
||||||
|
code is in no case prevented or interfered with solely because
|
||||||
|
modification has been made.
|
||||||
|
|
||||||
|
If you convey an object code work under this section in, or with, or
|
||||||
|
specifically for use in, a User Product, and the conveying occurs as
|
||||||
|
part of a transaction in which the right of possession and use of the
|
||||||
|
User Product is transferred to the recipient in perpetuity or for a
|
||||||
|
fixed term (regardless of how the transaction is characterized), the
|
||||||
|
Corresponding Source conveyed under this section must be accompanied
|
||||||
|
by the Installation Information. But this requirement does not apply
|
||||||
|
if neither you nor any third party retains the ability to install
|
||||||
|
modified object code on the User Product (for example, the work has
|
||||||
|
been installed in ROM).
|
||||||
|
|
||||||
|
The requirement to provide Installation Information does not include a
|
||||||
|
requirement to continue to provide support service, warranty, or updates
|
||||||
|
for a work that has been modified or installed by the recipient, or for
|
||||||
|
the User Product in which it has been modified or installed. Access to a
|
||||||
|
network may be denied when the modification itself materially and
|
||||||
|
adversely affects the operation of the network or violates the rules and
|
||||||
|
protocols for communication across the network.
|
||||||
|
|
||||||
|
Corresponding Source conveyed, and Installation Information provided,
|
||||||
|
in accord with this section must be in a format that is publicly
|
||||||
|
documented (and with an implementation available to the public in
|
||||||
|
source code form), and must require no special password or key for
|
||||||
|
unpacking, reading or copying.
|
||||||
|
|
||||||
|
7. Additional Terms.
|
||||||
|
|
||||||
|
"Additional permissions" are terms that supplement the terms of this
|
||||||
|
License by making exceptions from one or more of its conditions.
|
||||||
|
Additional permissions that are applicable to the entire Program shall
|
||||||
|
be treated as though they were included in this License, to the extent
|
||||||
|
that they are valid under applicable law. If additional permissions
|
||||||
|
apply only to part of the Program, that part may be used separately
|
||||||
|
under those permissions, but the entire Program remains governed by
|
||||||
|
this License without regard to the additional permissions.
|
||||||
|
|
||||||
|
When you convey a copy of a covered work, you may at your option
|
||||||
|
remove any additional permissions from that copy, or from any part of
|
||||||
|
it. (Additional permissions may be written to require their own
|
||||||
|
removal in certain cases when you modify the work.) You may place
|
||||||
|
additional permissions on material, added by you to a covered work,
|
||||||
|
for which you have or can give appropriate copyright permission.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, for material you
|
||||||
|
add to a covered work, you may (if authorized by the copyright holders of
|
||||||
|
that material) supplement the terms of this License with terms:
|
||||||
|
|
||||||
|
a) Disclaiming warranty or limiting liability differently from the
|
||||||
|
terms of sections 15 and 16 of this License; or
|
||||||
|
|
||||||
|
b) Requiring preservation of specified reasonable legal notices or
|
||||||
|
author attributions in that material or in the Appropriate Legal
|
||||||
|
Notices displayed by works containing it; or
|
||||||
|
|
||||||
|
c) Prohibiting misrepresentation of the origin of that material, or
|
||||||
|
requiring that modified versions of such material be marked in
|
||||||
|
reasonable ways as different from the original version; or
|
||||||
|
|
||||||
|
d) Limiting the use for publicity purposes of names of licensors or
|
||||||
|
authors of the material; or
|
||||||
|
|
||||||
|
e) Declining to grant rights under trademark law for use of some
|
||||||
|
trade names, trademarks, or service marks; or
|
||||||
|
|
||||||
|
f) Requiring indemnification of licensors and authors of that
|
||||||
|
material by anyone who conveys the material (or modified versions of
|
||||||
|
it) with contractual assumptions of liability to the recipient, for
|
||||||
|
any liability that these contractual assumptions directly impose on
|
||||||
|
those licensors and authors.
|
||||||
|
|
||||||
|
All other non-permissive additional terms are considered "further
|
||||||
|
restrictions" within the meaning of section 10. If the Program as you
|
||||||
|
received it, or any part of it, contains a notice stating that it is
|
||||||
|
governed by this License along with a term that is a further
|
||||||
|
restriction, you may remove that term. If a license document contains
|
||||||
|
a further restriction but permits relicensing or conveying under this
|
||||||
|
License, you may add to a covered work material governed by the terms
|
||||||
|
of that license document, provided that the further restriction does
|
||||||
|
not survive such relicensing or conveying.
|
||||||
|
|
||||||
|
If you add terms to a covered work in accord with this section, you
|
||||||
|
must place, in the relevant source files, a statement of the
|
||||||
|
additional terms that apply to those files, or a notice indicating
|
||||||
|
where to find the applicable terms.
|
||||||
|
|
||||||
|
Additional terms, permissive or non-permissive, may be stated in the
|
||||||
|
form of a separately written license, or stated as exceptions;
|
||||||
|
the above requirements apply either way.
|
||||||
|
|
||||||
|
8. Termination.
|
||||||
|
|
||||||
|
You may not propagate or modify a covered work except as expressly
|
||||||
|
provided under this License. Any attempt otherwise to propagate or
|
||||||
|
modify it is void, and will automatically terminate your rights under
|
||||||
|
this License (including any patent licenses granted under the third
|
||||||
|
paragraph of section 11).
|
||||||
|
|
||||||
|
However, if you cease all violation of this License, then your
|
||||||
|
license from a particular copyright holder is reinstated (a)
|
||||||
|
provisionally, unless and until the copyright holder explicitly and
|
||||||
|
finally terminates your license, and (b) permanently, if the copyright
|
||||||
|
holder fails to notify you of the violation by some reasonable means
|
||||||
|
prior to 60 days after the cessation.
|
||||||
|
|
||||||
|
Moreover, your license from a particular copyright holder is
|
||||||
|
reinstated permanently if the copyright holder notifies you of the
|
||||||
|
violation by some reasonable means, this is the first time you have
|
||||||
|
received notice of violation of this License (for any work) from that
|
||||||
|
copyright holder, and you cure the violation prior to 30 days after
|
||||||
|
your receipt of the notice.
|
||||||
|
|
||||||
|
Termination of your rights under this section does not terminate the
|
||||||
|
licenses of parties who have received copies or rights from you under
|
||||||
|
this License. If your rights have been terminated and not permanently
|
||||||
|
reinstated, you do not qualify to receive new licenses for the same
|
||||||
|
material under section 10.
|
||||||
|
|
||||||
|
9. Acceptance Not Required for Having Copies.
|
||||||
|
|
||||||
|
You are not required to accept this License in order to receive or
|
||||||
|
run a copy of the Program. Ancillary propagation of a covered work
|
||||||
|
occurring solely as a consequence of using peer-to-peer transmission
|
||||||
|
to receive a copy likewise does not require acceptance. However,
|
||||||
|
nothing other than this License grants you permission to propagate or
|
||||||
|
modify any covered work. These actions infringe copyright if you do
|
||||||
|
not accept this License. Therefore, by modifying or propagating a
|
||||||
|
covered work, you indicate your acceptance of this License to do so.
|
||||||
|
|
||||||
|
10. Automatic Licensing of Downstream Recipients.
|
||||||
|
|
||||||
|
Each time you convey a covered work, the recipient automatically
|
||||||
|
receives a license from the original licensors, to run, modify and
|
||||||
|
propagate that work, subject to this License. You are not responsible
|
||||||
|
for enforcing compliance by third parties with this License.
|
||||||
|
|
||||||
|
An "entity transaction" is a transaction transferring control of an
|
||||||
|
organization, or substantially all assets of one, or subdividing an
|
||||||
|
organization, or merging organizations. If propagation of a covered
|
||||||
|
work results from an entity transaction, each party to that
|
||||||
|
transaction who receives a copy of the work also receives whatever
|
||||||
|
licenses to the work the party's predecessor in interest had or could
|
||||||
|
give under the previous paragraph, plus a right to possession of the
|
||||||
|
Corresponding Source of the work from the predecessor in interest, if
|
||||||
|
the predecessor has it or can get it with reasonable efforts.
|
||||||
|
|
||||||
|
You may not impose any further restrictions on the exercise of the
|
||||||
|
rights granted or affirmed under this License. For example, you may
|
||||||
|
not impose a license fee, royalty, or other charge for exercise of
|
||||||
|
rights granted under this License, and you may not initiate litigation
|
||||||
|
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||||
|
any patent claim is infringed by making, using, selling, offering for
|
||||||
|
sale, or importing the Program or any portion of it.
|
||||||
|
|
||||||
|
11. Patents.
|
||||||
|
|
||||||
|
A "contributor" is a copyright holder who authorizes use under this
|
||||||
|
License of the Program or a work on which the Program is based. The
|
||||||
|
work thus licensed is called the contributor's "contributor version".
|
||||||
|
|
||||||
|
A contributor's "essential patent claims" are all patent claims
|
||||||
|
owned or controlled by the contributor, whether already acquired or
|
||||||
|
hereafter acquired, that would be infringed by some manner, permitted
|
||||||
|
by this License, of making, using, or selling its contributor version,
|
||||||
|
but do not include claims that would be infringed only as a
|
||||||
|
consequence of further modification of the contributor version. For
|
||||||
|
purposes of this definition, "control" includes the right to grant
|
||||||
|
patent sublicenses in a manner consistent with the requirements of
|
||||||
|
this License.
|
||||||
|
|
||||||
|
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||||
|
patent license under the contributor's essential patent claims, to
|
||||||
|
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||||
|
propagate the contents of its contributor version.
|
||||||
|
|
||||||
|
In the following three paragraphs, a "patent license" is any express
|
||||||
|
agreement or commitment, however denominated, not to enforce a patent
|
||||||
|
(such as an express permission to practice a patent or covenant not to
|
||||||
|
sue for patent infringement). To "grant" such a patent license to a
|
||||||
|
party means to make such an agreement or commitment not to enforce a
|
||||||
|
patent against the party.
|
||||||
|
|
||||||
|
If you convey a covered work, knowingly relying on a patent license,
|
||||||
|
and the Corresponding Source of the work is not available for anyone
|
||||||
|
to copy, free of charge and under the terms of this License, through a
|
||||||
|
publicly available network server or other readily accessible means,
|
||||||
|
then you must either (1) cause the Corresponding Source to be so
|
||||||
|
available, or (2) arrange to deprive yourself of the benefit of the
|
||||||
|
patent license for this particular work, or (3) arrange, in a manner
|
||||||
|
consistent with the requirements of this License, to extend the patent
|
||||||
|
license to downstream recipients. "Knowingly relying" means you have
|
||||||
|
actual knowledge that, but for the patent license, your conveying the
|
||||||
|
covered work in a country, or your recipient's use of the covered work
|
||||||
|
in a country, would infringe one or more identifiable patents in that
|
||||||
|
country that you have reason to believe are valid.
|
||||||
|
|
||||||
|
If, pursuant to or in connection with a single transaction or
|
||||||
|
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||||
|
covered work, and grant a patent license to some of the parties
|
||||||
|
receiving the covered work authorizing them to use, propagate, modify
|
||||||
|
or convey a specific copy of the covered work, then the patent license
|
||||||
|
you grant is automatically extended to all recipients of the covered
|
||||||
|
work and works based on it.
|
||||||
|
|
||||||
|
A patent license is "discriminatory" if it does not include within
|
||||||
|
the scope of its coverage, prohibits the exercise of, or is
|
||||||
|
conditioned on the non-exercise of one or more of the rights that are
|
||||||
|
specifically granted under this License. You may not convey a covered
|
||||||
|
work if you are a party to an arrangement with a third party that is
|
||||||
|
in the business of distributing software, under which you make payment
|
||||||
|
to the third party based on the extent of your activity of conveying
|
||||||
|
the work, and under which the third party grants, to any of the
|
||||||
|
parties who would receive the covered work from you, a discriminatory
|
||||||
|
patent license (a) in connection with copies of the covered work
|
||||||
|
conveyed by you (or copies made from those copies), or (b) primarily
|
||||||
|
for and in connection with specific products or compilations that
|
||||||
|
contain the covered work, unless you entered into that arrangement,
|
||||||
|
or that patent license was granted, prior to 28 March 2007.
|
||||||
|
|
||||||
|
Nothing in this License shall be construed as excluding or limiting
|
||||||
|
any implied license or other defenses to infringement that may
|
||||||
|
otherwise be available to you under applicable patent law.
|
||||||
|
|
||||||
|
12. No Surrender of Others' Freedom.
|
||||||
|
|
||||||
|
If conditions are imposed on you (whether by court order, agreement or
|
||||||
|
otherwise) that contradict the conditions of this License, they do not
|
||||||
|
excuse you from the conditions of this License. If you cannot convey a
|
||||||
|
covered work so as to satisfy simultaneously your obligations under this
|
||||||
|
License and any other pertinent obligations, then as a consequence you may
|
||||||
|
not convey it at all. For example, if you agree to terms that obligate you
|
||||||
|
to collect a royalty for further conveying from those to whom you convey
|
||||||
|
the Program, the only way you could satisfy both those terms and this
|
||||||
|
License would be to refrain entirely from conveying the Program.
|
||||||
|
|
||||||
|
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, if you modify the
|
||||||
|
Program, your modified version must prominently offer all users
|
||||||
|
interacting with it remotely through a computer network (if your version
|
||||||
|
supports such interaction) an opportunity to receive the Corresponding
|
||||||
|
Source of your version by providing access to the Corresponding Source
|
||||||
|
from a network server at no charge, through some standard or customary
|
||||||
|
means of facilitating copying of software. This Corresponding Source
|
||||||
|
shall include the Corresponding Source for any work covered by version 3
|
||||||
|
of the GNU General Public License that is incorporated pursuant to the
|
||||||
|
following paragraph.
|
||||||
|
|
||||||
|
Notwithstanding any other provision of this License, you have
|
||||||
|
permission to link or combine any covered work with a work licensed
|
||||||
|
under version 3 of the GNU General Public License into a single
|
||||||
|
combined work, and to convey the resulting work. The terms of this
|
||||||
|
License will continue to apply to the part which is the covered work,
|
||||||
|
but the work with which it is combined will remain governed by version
|
||||||
|
3 of the GNU General Public License.
|
||||||
|
|
||||||
|
14. Revised Versions of this License.
|
||||||
|
|
||||||
|
The Free Software Foundation may publish revised and/or new versions of
|
||||||
|
the GNU Affero General Public License from time to time. Such new versions
|
||||||
|
will be similar in spirit to the present version, but may differ in detail to
|
||||||
|
address new problems or concerns.
|
||||||
|
|
||||||
|
Each version is given a distinguishing version number. If the
|
||||||
|
Program specifies that a certain numbered version of the GNU Affero General
|
||||||
|
Public License "or any later version" applies to it, you have the
|
||||||
|
option of following the terms and conditions either of that numbered
|
||||||
|
version or of any later version published by the Free Software
|
||||||
|
Foundation. If the Program does not specify a version number of the
|
||||||
|
GNU Affero General Public License, you may choose any version ever published
|
||||||
|
by the Free Software Foundation.
|
||||||
|
|
||||||
|
If the Program specifies that a proxy can decide which future
|
||||||
|
versions of the GNU Affero General Public License can be used, that proxy's
|
||||||
|
public statement of acceptance of a version permanently authorizes you
|
||||||
|
to choose that version for the Program.
|
||||||
|
|
||||||
|
Later license versions may give you additional or different
|
||||||
|
permissions. However, no additional obligations are imposed on any
|
||||||
|
author or copyright holder as a result of your choosing to follow a
|
||||||
|
later version.
|
||||||
|
|
||||||
|
15. Disclaimer of Warranty.
|
||||||
|
|
||||||
|
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||||
|
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||||
|
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||||
|
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||||
|
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||||
|
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||||
|
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||||
|
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||||
|
|
||||||
|
16. Limitation of Liability.
|
||||||
|
|
||||||
|
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||||
|
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||||
|
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||||
|
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||||
|
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||||
|
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||||
|
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||||
|
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||||
|
SUCH DAMAGES.
|
||||||
|
|
||||||
|
17. Interpretation of Sections 15 and 16.
|
||||||
|
|
||||||
|
If the disclaimer of warranty and limitation of liability provided
|
||||||
|
above cannot be given local legal effect according to their terms,
|
||||||
|
reviewing courts shall apply local law that most closely approximates
|
||||||
|
an absolute waiver of all civil liability in connection with the
|
||||||
|
Program, unless a warranty or assumption of liability accompanies a
|
||||||
|
copy of the Program in return for a fee.
|
||||||
|
|
||||||
|
END OF TERMS AND CONDITIONS
|
||||||
|
|
||||||
|
How to Apply These Terms to Your New Programs
|
||||||
|
|
||||||
|
If you develop a new program, and you want it to be of the greatest
|
||||||
|
possible use to the public, the best way to achieve this is to make it
|
||||||
|
free software which everyone can redistribute and change under these terms.
|
||||||
|
|
||||||
|
To do so, attach the following notices to the program. It is safest
|
||||||
|
to attach them to the start of each source file to most effectively
|
||||||
|
state the exclusion of warranty; and each file should have at least
|
||||||
|
the "copyright" line and a pointer to where the full notice is found.
|
||||||
|
|
||||||
|
<one line to give the program's name and a brief idea of what it does.>
|
||||||
|
Copyright (C) <year> <name of author>
|
||||||
|
|
||||||
|
This program is free software: you can redistribute it and/or modify
|
||||||
|
it under the terms of the GNU Affero General Public License as published by
|
||||||
|
the Free Software Foundation, either version 3 of the License, or
|
||||||
|
(at your option) any later version.
|
||||||
|
|
||||||
|
This program is distributed in the hope that it will be useful,
|
||||||
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
GNU Affero General Public License for more details.
|
||||||
|
|
||||||
|
You should have received a copy of the GNU Affero General Public License
|
||||||
|
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
Also add information on how to contact you by electronic and paper mail.
|
||||||
|
|
||||||
|
If your software can interact with users remotely through a computer
|
||||||
|
network, you should also make sure that it provides a way for users to
|
||||||
|
get its source. For example, if your program is a web application, its
|
||||||
|
interface could display a "Source" link that leads users to an archive
|
||||||
|
of the code. There are many ways you could offer source, and different
|
||||||
|
solutions will be better for different programs; see section 13 for the
|
||||||
|
specific requirements.
|
||||||
|
|
||||||
|
You should also get your employer (if you work as a programmer) or school,
|
||||||
|
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||||
|
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||||
|
<https://www.gnu.org/licenses/>.
|
||||||
@@ -1,14 +1,233 @@
|
|||||||
|
<img src="frontend/public/logo.svg" alt="" width="132" align="right" />
|
||||||
|
|
||||||
# FabledCurator
|
# FabledCurator
|
||||||
|
|
||||||
Self-hosted media curation — gallery, ML tagging, and subscription-driven downloading in one app. Part of the FabledSword family.
|
<!-- overview:start -->
|
||||||
|
Self-hosted media curation — a gallery, ML auto-tagging, and subscription-driven
|
||||||
|
downloading in one application. Part of the FabledSword family.
|
||||||
|
|
||||||
Combines what was [ImageRepo](https://git.fabledsword.com/bvandeusen/ImageRepo) (gallery, ML, importer) and [GallerySubscriber](https://git.fabledsword.com/bvandeusen/GallerySubscriber) (gallery-dl wrapper, subscriptions, credential capture) into a single product.
|
## What it does
|
||||||
|
|
||||||
## Status
|
You point it at creators you follow. It downloads what they post, files it,
|
||||||
|
tags it, and gives you something better than a folder full of images to look
|
||||||
|
through afterwards.
|
||||||
|
|
||||||
In production. `main` is continuously deployed — every merge to `main` builds
|
- **Gallery and browsing.** Images, videos and multi-page works, organised by
|
||||||
and publishes `:latest` images, so whatever is on `main` is what is running.
|
artist, tag, post and series. A newest-first feed of what just arrived as the
|
||||||
Day-to-day work happens on `dev`, which publishes `:dev` images.
|
front page, a random Showcase, a filterable gallery, a similarity-driven
|
||||||
|
Explore view, and a page-turning reader for series.
|
||||||
|
- **Subscriptions.** Follows creators on Patreon, SubscribeStar, Pixiv and
|
||||||
|
anything `gallery-dl` supports, on a schedule. Handles paywalled posts using
|
||||||
|
your own logged-in session.
|
||||||
|
- **ML tagging.** Runs image models in-container to suggest tags, group
|
||||||
|
characters, find near-duplicates and power similarity search. Suggestions are
|
||||||
|
reviewable — it proposes, you confirm, and it learns which proposals you keep
|
||||||
|
rejecting.
|
||||||
|
- **Deduplication and provenance.** Everything that arrives is hashed and
|
||||||
|
deduplicated by content, metadata sidecars are read wherever the source
|
||||||
|
writes them, and every file keeps a record of where it came from.
|
||||||
|
- **Maintenance.** Backups, library audits, thumbnail and embedding backfills,
|
||||||
|
orphan cleanup — all from the UI, all as background jobs you can watch.
|
||||||
|
|
||||||
|
Everything is configured from the Settings UI and stored in the database. There
|
||||||
|
is no config file to edit beyond a handful of bootstrap environment variables.
|
||||||
|
<!-- overview:end -->
|
||||||
|
|
||||||
|
## Before you expose it
|
||||||
|
|
||||||
|
**FabledCurator has no login.** There are no user accounts, no passwords and no
|
||||||
|
permission model. Anything that can reach the port is an administrator.
|
||||||
|
|
||||||
|
That matters more here than it would in most self-hosted apps, because of what
|
||||||
|
this one stores: **live platform session cookies for Patreon, SubscribeStar and
|
||||||
|
Pixiv** — accounts that usually have a payment method attached. Whoever reaches
|
||||||
|
the port can read them, alongside your entire library.
|
||||||
|
|
||||||
|
So:
|
||||||
|
|
||||||
|
- Bind it to a LAN, a VPN, or a tunnel you control.
|
||||||
|
- Do not port-forward it. Do not put it on a public hostname.
|
||||||
|
- A reverse proxy that adds TLS but no authentication **does not help**. If you
|
||||||
|
want it reachable from outside, put an authenticating proxy in front of it —
|
||||||
|
a forward-auth provider, HTTP basic auth, an identity-aware tunnel — and treat
|
||||||
|
that layer as the only thing standing between the internet and your accounts.
|
||||||
|
|
||||||
|
This is a deliberate design decision for a single-operator tool on a trusted
|
||||||
|
network, not a bug and not an oversight. It is stated here because it decides
|
||||||
|
how you are allowed to deploy it. [SECURITY.md](SECURITY.md) covers the rest of
|
||||||
|
the threat model.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
- **Docker** with Compose v2.
|
||||||
|
- **~4 GB RAM** for the app, plus whatever Postgres needs for your library size.
|
||||||
|
- **Disk** for your media, plus several GB for ML model weights.
|
||||||
|
- **No GPU required.** The ML worker runs on CPU — tagging and embedding are
|
||||||
|
slower, and that is the whole difference. A GPU is only involved if you
|
||||||
|
separately run the optional agent (below), which is a different machine's job.
|
||||||
|
|
||||||
|
## Install
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://git.fabledsword.com/bvandeusen/FabledCurator.git
|
||||||
|
cd FabledCurator
|
||||||
|
|
||||||
|
cp .env.example .env
|
||||||
|
$EDITOR .env # set DB_PASSWORD and SECRET_KEY
|
||||||
|
|
||||||
|
docker compose -f docker-compose.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Then open <http://localhost:8080>.
|
||||||
|
|
||||||
|
**The `-f docker-compose.yml` is required, not decoration.** Compose
|
||||||
|
auto-merges `docker-compose.override.yml` when you leave it off, and that
|
||||||
|
override builds the images locally from source — the contributor path, not
|
||||||
|
yours. Naming the file explicitly skips the override and pulls the published
|
||||||
|
`:latest` images, which is the stable channel built from `main`.
|
||||||
|
|
||||||
|
If you forget it, the symptom is a long build instead of a quick pull.
|
||||||
|
|
||||||
|
## First run
|
||||||
|
|
||||||
|
The database schema is created automatically on first start — the web container
|
||||||
|
runs its migrations before serving. Nothing to initialise by hand.
|
||||||
|
|
||||||
|
**One thing does need a deliberate act, and the app will not start without it.**
|
||||||
|
FabledCurator encrypts your stored platform credentials with a key it keeps at
|
||||||
|
`./images/secrets/credential_key.b64`. On a brand-new install that file does not
|
||||||
|
exist, and rather than quietly creating one the app stops:
|
||||||
|
|
||||||
|
```
|
||||||
|
MissingCredentialKey: Fernet key file not found at /images/secrets/credential_key.b64
|
||||||
|
```
|
||||||
|
|
||||||
|
Set `CURATOR_BOOTSTRAP_NEW_KEY=1` in your `.env` for the first `up`, then delete
|
||||||
|
the line once the container is running. `.env.example` ships it with that
|
||||||
|
instruction attached.
|
||||||
|
|
||||||
|
The refusal is deliberate, and worth understanding rather than working around:
|
||||||
|
auto-creating a key is indistinguishable from the disaster case — a restore that
|
||||||
|
brought the database back but lost `./images/secrets` — where it would mint a key
|
||||||
|
that cannot decrypt anything, leaving an instance that looks healthy while every
|
||||||
|
paywalled download fails. Making you say so once, on an empty install, is the
|
||||||
|
price of that not happening silently later.
|
||||||
|
|
||||||
|
**Which means: back up `./images/secrets/` alongside your database.** It is the
|
||||||
|
only thing that can read your stored credentials. A database restored without it
|
||||||
|
needs every credential entered again by hand.
|
||||||
|
|
||||||
|
A few other things are worth knowing about the first few minutes:
|
||||||
|
|
||||||
|
- **The ML worker downloads its model weights on first boot**, several GB from
|
||||||
|
HuggingFace into `./models`. Until that finishes, tagging is queued rather
|
||||||
|
than broken. It is idempotent — a restart resumes rather than refetches.
|
||||||
|
- **The gallery starts empty**, and that is the expected state. Add a creator
|
||||||
|
under **Subscriptions** and it fills as posts come down.
|
||||||
|
- **If you already have a library on disk**, there is no screen that imports
|
||||||
|
it, and there is not going to be one. Folder ingestion had a UI until July
|
||||||
|
2026; it was retired once posts began arriving entirely through
|
||||||
|
subscriptions and the browser extension, and the decision to leave it
|
||||||
|
retired is deliberate — the folder path carries complexity the product does
|
||||||
|
not need in order to do its job. The supported way to fill a new install is
|
||||||
|
to add the creators you follow under **Subscriptions** and let it pull.
|
||||||
|
|
||||||
|
The `/api/import/trigger` endpoint is still wired up for anyone who wants to
|
||||||
|
script a one-off against a folder mounted at `./import`, and its progress
|
||||||
|
shows under **Settings → Activity**. Treat it as an unsupported escape
|
||||||
|
hatch rather than a feature: nothing in the UI drives it and nothing else
|
||||||
|
in this README depends on it.
|
||||||
|
- **To download from a paywalled account**, FabledCurator needs that account's
|
||||||
|
session — see the browser extension below. Without one it can still fetch
|
||||||
|
public posts.
|
||||||
|
- **Check Settings → Overview** to confirm the workers are alive. Every long
|
||||||
|
operation in FabledCurator is a background job, so if the queues are not
|
||||||
|
running, the UI will look like it is ignoring you rather than like it is
|
||||||
|
broken.
|
||||||
|
|
||||||
|
## The browser extension
|
||||||
|
|
||||||
|
A Firefox extension does two jobs: it hands your logged-in platform sessions to
|
||||||
|
FabledCurator so it can download on your behalf, and it adds a creator as a
|
||||||
|
subscription in one click from their page.
|
||||||
|
|
||||||
|
It ships **inside the web image** — there is no add-on store listing to find.
|
||||||
|
Go to **Subscriptions → Settings**, find the *Browser extension* card, and click
|
||||||
|
**Install Firefox extension**. The XPI is Mozilla-signed, so Firefox installs it
|
||||||
|
like any other add-on; the button serves it directly rather than making you
|
||||||
|
download and side-load a file.
|
||||||
|
|
||||||
|
It pairs with your instance using an API key generated automatically on first
|
||||||
|
use. The bar directly under that card shows the key and can rotate it.
|
||||||
|
|
||||||
|
See [extension/README.md](extension/README.md) for what it does in detail.
|
||||||
|
|
||||||
|
## The GPU agent
|
||||||
|
|
||||||
|
Optional, and separate. If you have a desktop with a graphics card, you can run
|
||||||
|
an agent on it that leases ML jobs from FabledCurator over HTTP, does them on
|
||||||
|
the GPU, and hands the results back. It never touches the database or Redis, so
|
||||||
|
it is safe to run somewhere the rest of the stack is not.
|
||||||
|
|
||||||
|
Run it for a burst of tagging, stop it to get your card back. It deploys from
|
||||||
|
`agent/docker-compose.yml`, not the main stack — see
|
||||||
|
[agent/README.md](agent/README.md).
|
||||||
|
|
||||||
|
## Upgrading
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose -f docker-compose.yml pull
|
||||||
|
docker compose -f docker-compose.yml up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
Migrations run automatically on start. Take a database backup first — Settings →
|
||||||
|
Maintenance has one — because the schema moves forward and does not move back.
|
||||||
|
|
||||||
|
## Deployment posture
|
||||||
|
|
||||||
|
FabledCurator is built to run inside a homelab over plain HTTP. It does not
|
||||||
|
generate certificates, redirect to HTTPS, or set HSTS. If you want TLS,
|
||||||
|
terminate it at your reverse proxy. See [Before you expose it](#before-you-expose-it)
|
||||||
|
for why TLS alone is not enough.
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
**The UI loads but nothing ever finishes.** The web container is up and the
|
||||||
|
workers are not. `docker compose -f docker-compose.yml ps` — check `worker`,
|
||||||
|
`scheduler` and `ml-worker` are healthy, not restarting.
|
||||||
|
|
||||||
|
**`docker compose up` started building instead of pulling.** You left off
|
||||||
|
`-f docker-compose.yml`, so the dev override took over. See [Install](#install).
|
||||||
|
|
||||||
|
**Downloads fail with an auth error.** The stored session for that platform has
|
||||||
|
expired. Re-capture it with the extension; sessions do not last forever.
|
||||||
|
|
||||||
|
**Which build am I running?** The foot of Settings shows a version and a
|
||||||
|
channel, and `/api/health` returns the same two fields. There are no version
|
||||||
|
tags on the images, so this is the authoritative answer.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
# Developing FabledCurator
|
||||||
|
|
||||||
|
Everything below is about working on FabledCurator rather than running it. If
|
||||||
|
you are installing it, you are done — see [CONTRIBUTING.md](CONTRIBUTING.md) if
|
||||||
|
you want to send a patch.
|
||||||
|
|
||||||
|
## Status and channels
|
||||||
|
|
||||||
|
In production. `main` is continuously deployed — every merge builds and
|
||||||
|
publishes `:latest`, so whatever is on `main` is what is running. Day-to-day
|
||||||
|
work happens on `dev`, which publishes `:dev`.
|
||||||
|
|
||||||
|
For local development, the dev override handles everything:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d # note: no -f, so the override applies
|
||||||
|
```
|
||||||
|
|
||||||
|
That builds the images from source, turns on DEBUG logging, and exposes
|
||||||
|
Postgres and Redis on the host. No `.env` required.
|
||||||
|
|
||||||
## Versions and tags
|
## Versions and tags
|
||||||
|
|
||||||
@@ -47,38 +266,10 @@ Five deployable pieces, built by `.forgejo/workflows/build.yml`:
|
|||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| **Web / workers** | `Dockerfile` | `fabledcurator` | Quart API + the built Vue SPA in one image. `entrypoint.sh` picks the role: `web`, `worker`, `scheduler`. The `maintenance-long` service is a second `worker` pinned to the long-running maintenance queue. |
|
| **Web / workers** | `Dockerfile` | `fabledcurator` | Quart API + the built Vue SPA in one image. `entrypoint.sh` picks the role: `web`, `worker`, `scheduler`. The `maintenance-long` service is a second `worker` pinned to the long-running maintenance queue. |
|
||||||
| **ML worker** | `Dockerfile.ml` | `fabledcurator-ml` | Same app, plus `requirements-ml.txt` — tagging and embedding models that run in-container. |
|
| **ML worker** | `Dockerfile.ml` | `fabledcurator-ml` | Same app, plus `requirements-ml.txt` — tagging and embedding models that run in-container. |
|
||||||
| **GPU agent** | `agent/Dockerfile` | `fabledcurator-agent` | Optional desktop-GPU worker (`agent/`). Leases jobs over **HTTP only** — never touches the database or Redis. Run it for a burst, stop it to reclaim the card. See `agent/README.md`. |
|
| **GPU agent** | `agent/Dockerfile` | `fabledcurator-agent` | Optional desktop-GPU worker (`agent/`). Leases jobs over **HTTP only** — never touches the database or Redis. See `agent/README.md`. |
|
||||||
| **Firefox extension** | `extension/` | signed XPI | MV3 extension: pushes platform session cookies into FC and adds a creator as a Source in one click. AMO-signed on both `dev` and `main` (one signature per extension change, shared by the two channels), bundled into that channel's web image and served from Settings → Maintenance. See `extension/README.md`. |
|
| **Firefox extension** | `extension/` | signed XPI | MV3 extension: pushes platform session cookies into FC and adds a creator as a Source in one click. AMO-signed on both `dev` and `main` (one signature per extension change, shared by the two channels), bundled into that channel's web image and served from Settings → Maintenance. See `extension/README.md`. |
|
||||||
| **Data** | — | `pgvector/pgvector:pg16`, `redis:7-alpine` | Postgres with pgvector for embeddings; Redis as the Celery broker. |
|
| **Data** | — | `pgvector/pgvector:pg16`, `redis:7-alpine` | Postgres with pgvector for embeddings; Redis as the Celery broker. |
|
||||||
|
|
||||||
## Quick start
|
|
||||||
|
|
||||||
For local development and testing, just:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose up -d
|
|
||||||
# UI: http://localhost:8080
|
|
||||||
```
|
|
||||||
|
|
||||||
That uses sane dev defaults baked into `docker-compose.yml` and the dev
|
|
||||||
override (`docker-compose.override.yml`, auto-merged) — local builds, DEBUG
|
|
||||||
logging, exposed Postgres + Redis ports on the host. No `.env` required.
|
|
||||||
|
|
||||||
For a production-like deployment, override the dev defaults via shell env
|
|
||||||
or a `.env` file (see `.env.example` for the variable names) and use:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
docker compose -f docker-compose.yml up -d
|
|
||||||
# (skips the override so containers pull registry images)
|
|
||||||
```
|
|
||||||
|
|
||||||
The GPU agent is deployed separately, on the machine with the card —
|
|
||||||
`agent/docker-compose.yml`, not this stack.
|
|
||||||
|
|
||||||
## Deployment posture
|
|
||||||
|
|
||||||
FabledCurator is designed to run inside a self-hosted homelab environment over plain HTTP. If you want TLS, terminate it at your reverse proxy. The app does not generate certificates, redirect to HTTPS, or set HSTS.
|
|
||||||
|
|
||||||
## CI / Forgejo setup
|
## CI / Forgejo setup
|
||||||
|
|
||||||
Four workflows: `ci.yml` (lint, extension-version check, backend unit tests,
|
Four workflows: `ci.yml` (lint, extension-version check, backend unit tests,
|
||||||
@@ -108,6 +299,29 @@ source, so `main` finds `dev`'s signature already cached and makes no second AMO
|
|||||||
call. That cache is why signing must be one-shot — AMO rejects a re-signed
|
call. That cache is why signing must be one-shot — AMO rejects a re-signed
|
||||||
version.
|
version.
|
||||||
|
|
||||||
|
## History
|
||||||
|
|
||||||
|
FabledCurator combines what was
|
||||||
|
[ImageRepo](https://git.fabledsword.com/bvandeusen/ImageRepo) (gallery, ML,
|
||||||
|
importer) and
|
||||||
|
[GallerySubscriber](https://git.fabledsword.com/bvandeusen/GallerySubscriber)
|
||||||
|
(gallery-dl wrapper, subscriptions, credential capture) into a single product.
|
||||||
|
Both are superseded; neither is maintained.
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
Personal project; use at your own discretion.
|
**GNU Affero General Public License v3.0** — see [LICENSE](LICENSE).
|
||||||
|
|
||||||
|
You may run, study, modify and redistribute this software. The condition is
|
||||||
|
reciprocity: if you distribute a modified version, or **run one as a network
|
||||||
|
service that other people use**, you must offer those users the corresponding
|
||||||
|
source under the same licence. That second clause (AGPL §13) is the reason this
|
||||||
|
licence rather than the GPL — for a self-hosted web application, "distribution"
|
||||||
|
otherwise never happens, and the obligation would never bite.
|
||||||
|
|
||||||
|
Running an unmodified copy for yourself, your household or your organisation
|
||||||
|
carries no obligation at all. Neither does modifying it privately. The licence
|
||||||
|
asks something of you only when you hand your modified version to others.
|
||||||
|
|
||||||
|
Contributions ship under the same licence — see [CONTRIBUTING](CONTRIBUTING.md).
|
||||||
|
Security reports: [SECURITY.md](SECURITY.md).
|
||||||
|
|||||||
+77
@@ -0,0 +1,77 @@
|
|||||||
|
# Security Policy
|
||||||
|
|
||||||
|
## Reporting a vulnerability
|
||||||
|
|
||||||
|
**Please do not put vulnerability details in a public issue.**
|
||||||
|
|
||||||
|
This project has no private disclosure channel yet. Until it does, open an
|
||||||
|
issue on the repository that says only that you have a security report — no
|
||||||
|
reproduction steps, no affected endpoint, no payload — and a maintainer will
|
||||||
|
reply with a private contact to send the details to.
|
||||||
|
|
||||||
|
That is a deliberately awkward first step, and it exists because the
|
||||||
|
alternative is worse: an issue tracker is public the moment it is written to,
|
||||||
|
and every self-hosted instance stays vulnerable until its operator has had a
|
||||||
|
chance to update.
|
||||||
|
|
||||||
|
Please include, once you have a private channel:
|
||||||
|
|
||||||
|
- what an attacker can do, and what access they need to start
|
||||||
|
- the version or commit you tested
|
||||||
|
- reproduction steps
|
||||||
|
|
||||||
|
## Scope — what this software actually handles
|
||||||
|
|
||||||
|
FabledCurator is self-hosted and holds things worth stating plainly, because
|
||||||
|
they shape what counts as a serious bug here:
|
||||||
|
|
||||||
|
- **Platform credentials.** The app captures and stores session cookies for
|
||||||
|
third-party subscription sites (Patreon, SubscribeStar, Pixiv) so it can
|
||||||
|
download on the operator's behalf. These are live credentials for accounts
|
||||||
|
that usually carry a payment method. Anything that discloses them, decrypts
|
||||||
|
them, or lets one user of a shared instance read another's is high severity.
|
||||||
|
- **An extension API key.** The Firefox extension authenticates to the backend
|
||||||
|
with a shared key. Anything that leaks it or lets it be bypassed is a way in.
|
||||||
|
- **No authentication of its own.** This is the most important thing on this
|
||||||
|
page. FabledCurator has no login, no user accounts and no permission model —
|
||||||
|
there is no `User` table and no session auth anywhere in the backend. Every
|
||||||
|
HTTP client that can reach the port is the administrator, with full read and
|
||||||
|
write access to everything above, including the stored platform credentials.
|
||||||
|
Access control is entirely the operator's job, done at the network layer.
|
||||||
|
Reports that an unauthenticated caller can reach an endpoint are therefore
|
||||||
|
describing the design; reports that something *crosses the network boundary
|
||||||
|
the operator drew* — an SSRF, a request forgery that rides a browser the
|
||||||
|
operator already has open, a path that leaks state to an origin the operator
|
||||||
|
did not authorise — are in scope and are serious.
|
||||||
|
- **Arbitrary media from the internet.** Downloaded files are decoded, hashed,
|
||||||
|
thumbnailed and fed to ML models. Anything that turns a hostile file into
|
||||||
|
code execution is in scope.
|
||||||
|
|
||||||
|
## Deployment posture — read this before reporting
|
||||||
|
|
||||||
|
FabledCurator is designed to run **inside a private network, over plain HTTP,
|
||||||
|
reachable only by its operator**. It does not terminate TLS, redirect to
|
||||||
|
HTTPS, or set HSTS; if you want transport security, terminate it at your
|
||||||
|
reverse proxy. It also does not authenticate anyone — see above. These are
|
||||||
|
documented design decisions, not oversights.
|
||||||
|
|
||||||
|
Putting this on the public internet, with or without TLS, hands whoever finds
|
||||||
|
it your Patreon, SubscribeStar and Pixiv sessions. A reverse proxy that adds
|
||||||
|
TLS but not an authentication layer does not change that.
|
||||||
|
|
||||||
|
Reports that reduce to "the application is served over HTTP", "there is no
|
||||||
|
HSTS header", or "the API needs no credentials" describe those decisions
|
||||||
|
rather than vulnerabilities. Reports that the operator can cause the software
|
||||||
|
to do something destructive are usually also by design — the operator is the
|
||||||
|
administrator of their own instance.
|
||||||
|
|
||||||
|
What remains in scope is everything that crosses a boundary the software is
|
||||||
|
actually supposed to hold: between untrusted downloaded content and the host,
|
||||||
|
between a third-party origin and an operator's open browser session, and
|
||||||
|
between the credentials at rest and anything that is not the operator.
|
||||||
|
|
||||||
|
## Supported versions
|
||||||
|
|
||||||
|
Fixes land on the `main` branch and reach the `:latest` image. There are no
|
||||||
|
maintained release branches — the supported version is the current one, and
|
||||||
|
the remedy for a security issue is to update.
|
||||||
@@ -1,78 +1,107 @@
|
|||||||
"""Collapsed baseline — the whole schema in one revision.
|
"""The whole schema, in one migration.
|
||||||
|
|
||||||
Replaces revisions 0001..0087, which narrated the build-out of this project
|
This replaces alembic revisions 0001..0089 — the entire build-out of the
|
||||||
and were deleted in milestone 328 step 1. A new install creates the schema in
|
project, 89 files and ~6,000 lines that a new installation used to replay in
|
||||||
one step instead of replaying that history.
|
order to arrive at a schema this file creates in one pass. Nothing about the
|
||||||
|
resulting database changes; what goes away is the requirement that a stranger
|
||||||
|
re-run our development history to get it.
|
||||||
|
|
||||||
WHY THE REVISION ID IS "0087" AND NOT "0001"
|
## Why the revision id is 0089
|
||||||
--------------------------------------------
|
|
||||||
It is deliberately the id of the LAST revision this baseline collapses, so an
|
|
||||||
existing database needs no intervention at all:
|
|
||||||
|
|
||||||
* a fresh install finds current=none, head=0087, runs this file once, and
|
`revision = "0089"` and `down_revision = None` are both deliberate, and the
|
||||||
ends stamped at 0087.
|
combination is the entire migration strategy for existing installations.
|
||||||
* an existing install is ALREADY at 0087, so `alembic upgrade head` finds
|
|
||||||
current == head and does nothing.
|
|
||||||
|
|
||||||
The alternative — numbering this 0001 and stamping every existing database —
|
An already-deployed database has `alembic_version = '0089'`, because it ran the
|
||||||
means running `alembic stamp` against live data, and stamp VALIDATES NOTHING.
|
real 0089. This file claims that same id, so alembic reads the version table,
|
||||||
It writes a version string whether or not the schema actually matches, so a
|
sees head already reached, and does nothing at all. No stamp is needed — which
|
||||||
wrong baseline would be discovered later, by the next real migration, with no
|
matters because `alembic stamp` writes a version string without validating
|
||||||
clean way back. Keeping the id removes that operation instead of making it
|
anything about the schema it is writing it against, and a stamp that is wrong
|
||||||
safe. Future revisions continue at 0088.
|
is indistinguishable from one that is right until the next migration fails.
|
||||||
|
|
||||||
The one case this makes worse, and it fails LOUDLY rather than silently: a
|
An empty database has no version row, so alembic runs this file and then
|
||||||
database still sitting between 0001 and 0086 (i.e. never upgraded to head)
|
records `0089`. Both paths converge on the same schema and the same version,
|
||||||
cannot be located in this chain and errors out. Upgrade to 0087 on a
|
and neither requires anyone to assert anything by hand.
|
||||||
pre-squash build first, then take this one.
|
|
||||||
|
|
||||||
WHAT IS HAND-WRITTEN HERE
|
The next migration written after this one is `0090`, exactly as it would have
|
||||||
-------------------------
|
been. The numbering is continuous across the collapse on purpose.
|
||||||
Most of this file is `alembic revision --autogenerate` output, but four
|
|
||||||
things are NOT in SQLAlchemy metadata and the generator cannot produce them.
|
|
||||||
Each fails differently, and none of them fail at generation time:
|
|
||||||
|
|
||||||
1. CREATE EXTENSION vector (was 0001) — without it the VECTOR
|
## What was added to the generated output, and why
|
||||||
columns below cannot be created at all.
|
|
||||||
2. CREATE EXTENSION tsm_system_rows (was 0004) — used by the random-sample
|
|
||||||
query path; its absence surfaces only when that query runs.
|
|
||||||
3. The HNSW index on image_record.siglip_embedding (was 0036). Raw SQL
|
|
||||||
because alembic's create_index cannot express `USING hnsw (...
|
|
||||||
vector_cosine_ops)`. Its absence is the quietest failure of the four:
|
|
||||||
everything works, similarity search just stops using an index.
|
|
||||||
4. `import pgvector.sqlalchemy.vector`. Autogenerate EMITS references to
|
|
||||||
pgvector.sqlalchemy.vector.VECTOR but does not add the import, so the
|
|
||||||
generated file dies with NameError on first run.
|
|
||||||
|
|
||||||
The acceptance test for this file is not that it reads correctly — it is
|
`alembic revision --autogenerate` produced almost all of this from the models,
|
||||||
`.forgejo/workflows/baseline.yml`, which builds a database from the old
|
which is only true because #3275 first made the models actually describe the
|
||||||
0001..0087 chain (read out of git) and one from this file, and diffs
|
schema. Before that reconciliation the generator silently omitted eleven
|
||||||
pg_dump --schema-only output. That is what proves nothing was missed.
|
indexes and three uniqueness guarantees, and an earlier attempt at this squash
|
||||||
|
had to be reverted for exactly that reason.
|
||||||
|
|
||||||
Revision ID: 0087
|
Four things still had to be added by hand, because they are not in the models:
|
||||||
|
|
||||||
|
1. **`CREATE EXTENSION vector`** (from 0001) and **`tsm_system_rows`** (0004).
|
||||||
|
Extensions are database objects, not table metadata, so no model can carry
|
||||||
|
them. `IF NOT EXISTS` because a re-run must not fail.
|
||||||
|
|
||||||
|
2. **Three seed inserts** — the two settings singletons (0002, 0003) and the
|
||||||
|
three hygiene system tags (0075). Some migrations did not only build schema;
|
||||||
|
they inserted rows the product needs in order to function, and nothing in
|
||||||
|
the application ever creates them. Every consumer reads them with
|
||||||
|
`scalar_one()`, which RAISES `NoResultFound` on an empty result rather than
|
||||||
|
returning None, so their absence is a crash and not a degradation.
|
||||||
|
|
||||||
|
Distinguishing these from the other data statements in the chain is the
|
||||||
|
whole trick, and the rule turns out to be mechanical:
|
||||||
|
|
||||||
|
* `INSERT ... VALUES (...)` with literal values is a SEED. It creates
|
||||||
|
something the product ships. It must be carried.
|
||||||
|
* `INSERT ... SELECT ... FROM <table>` is a BACKFILL. It derives rows
|
||||||
|
from rows that already exist, so on an empty database it inserts
|
||||||
|
nothing and carrying it would be pointless. 0034 (artist_visit), 0040
|
||||||
|
and 0047 (series_chapter) are all of this shape and are correctly
|
||||||
|
absent here.
|
||||||
|
|
||||||
|
This category is invisible to every automated check this project has:
|
||||||
|
`baseline.yml` compares SCHEMA, and a baseline missing all three seeds still
|
||||||
|
produces a byte-identical schema and a perfectly green diff. What caught the
|
||||||
|
system tags was the integration suite — 36 tests failing on
|
||||||
|
`NoResultFound` — after a first version of this file shipped with only the
|
||||||
|
two settings rows. A first-run check against the real application is the
|
||||||
|
only thing that finds this class of defect.
|
||||||
|
|
||||||
|
3. **The `pgvector` import.** Autogenerate emits qualified
|
||||||
|
`pgvector.sqlalchemy.vector.VECTOR(...)` references without importing the
|
||||||
|
package, so the file it writes cannot execute — `NameError: name 'pgvector'
|
||||||
|
is not defined`, observed on run 4988.
|
||||||
|
|
||||||
|
The other data statements in the old chain were deliberately NOT carried over.
|
||||||
|
0023's `DELETE FROM tag WHERE kind IN (...)`, and 0047's `series_page` /
|
||||||
|
`series_chapter` deletes, are historical cleanups that operate on rows an empty
|
||||||
|
database does not have.
|
||||||
|
|
||||||
|
## Downgrade
|
||||||
|
|
||||||
|
There is none. A baseline's downgrade would be "drop the entire schema", which
|
||||||
|
is not a migration but a data-loss event wearing one as a disguise. Restore
|
||||||
|
from a backup instead — that is what backup_run exists for.
|
||||||
|
|
||||||
|
Revision ID: 0089
|
||||||
Revises:
|
Revises:
|
||||||
Create Date: 2026-08-30
|
Create Date: 2026-09-01
|
||||||
|
|
||||||
"""
|
"""
|
||||||
from typing import Sequence, Union
|
from typing import Sequence, Union
|
||||||
|
|
||||||
from alembic import op
|
from alembic import op
|
||||||
import sqlalchemy as sa
|
import sqlalchemy as sa
|
||||||
|
import pgvector.sqlalchemy.vector
|
||||||
from sqlalchemy.dialects import postgresql
|
from sqlalchemy.dialects import postgresql
|
||||||
|
|
||||||
# Autogenerate references pgvector.sqlalchemy.vector.VECTOR without importing
|
revision: str = "0089"
|
||||||
# it. Item 4 above.
|
|
||||||
import pgvector.sqlalchemy.vector
|
|
||||||
|
|
||||||
revision: str = "0087"
|
|
||||||
down_revision: Union[str, None] = None
|
down_revision: Union[str, None] = None
|
||||||
branch_labels: Union[str, Sequence[str], None] = None
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
depends_on: Union[str, Sequence[str], None] = None
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
# Extensions FIRST: the VECTOR columns below cannot be created without
|
# Extensions first: image_record.siglip_embedding is a vector column and
|
||||||
# `vector`, so ordering here is load-bearing, not tidiness.
|
# cannot be created before the type exists. From 0001 and 0004.
|
||||||
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
||||||
op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows")
|
op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows")
|
||||||
|
|
||||||
@@ -87,8 +116,8 @@ def upgrade() -> None:
|
|||||||
sa.Column('name', sa.String(length=255), nullable=False),
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
sa.Column('slug', sa.String(length=255), nullable=False),
|
sa.Column('slug', sa.String(length=255), nullable=False),
|
||||||
sa.Column('notes', sa.Text(), nullable=True),
|
sa.Column('notes', sa.Text(), nullable=True),
|
||||||
sa.Column('is_subscription', sa.Boolean(), nullable=False),
|
sa.Column('is_subscription', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('auto_check', sa.Boolean(), nullable=False),
|
sa.Column('auto_check', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('check_interval_seconds', sa.Integer(), nullable=True),
|
sa.Column('check_interval_seconds', sa.Integer(), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_artist')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_artist')),
|
||||||
@@ -97,7 +126,7 @@ def upgrade() -> None:
|
|||||||
op.create_table('backup_run',
|
op.create_table('backup_run',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('kind', sa.String(length=16), nullable=False),
|
sa.Column('kind', sa.String(length=16), nullable=False),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||||
sa.Column('tag', sa.String(length=64), nullable=True),
|
sa.Column('tag', sa.String(length=64), nullable=True),
|
||||||
sa.Column('triggered_by', sa.String(length=32), nullable=False),
|
sa.Column('triggered_by', sa.String(length=32), nullable=False),
|
||||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
@@ -112,10 +141,12 @@ def upgrade() -> None:
|
|||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_backup_run'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_backup_run'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_backup_run_finished_at'), 'backup_run', ['finished_at'], unique=False)
|
op.create_index(op.f('ix_backup_run_finished_at'), 'backup_run', ['finished_at'], unique=False)
|
||||||
op.create_index(op.f('ix_backup_run_kind'), 'backup_run', ['kind'], unique=False)
|
op.create_index('ix_backup_run_kind_started', 'backup_run', ['kind', sa.literal_column('started_at DESC')], unique=False)
|
||||||
|
op.create_index(op.f('ix_backup_run_restored_from_id'), 'backup_run', ['restored_from_id'], unique=False)
|
||||||
op.create_index(op.f('ix_backup_run_started_at'), 'backup_run', ['started_at'], unique=False)
|
op.create_index(op.f('ix_backup_run_started_at'), 'backup_run', ['started_at'], unique=False)
|
||||||
op.create_index(op.f('ix_backup_run_status'), 'backup_run', ['status'], unique=False)
|
op.create_index('ix_backup_run_status_finished', 'backup_run', ['status', sa.literal_column('finished_at DESC')], unique=False)
|
||||||
op.create_index(op.f('ix_backup_run_tag'), 'backup_run', ['tag'], unique=False)
|
op.create_index(op.f('ix_backup_run_tag'), 'backup_run', ['tag'], unique=False)
|
||||||
|
op.create_index('ix_backup_run_tag_partial', 'backup_run', ['tag'], unique=False, postgresql_where=sa.text('tag IS NOT NULL'))
|
||||||
op.create_table('credential',
|
op.create_table('credential',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('platform', sa.String(length=64), nullable=False),
|
sa.Column('platform', sa.String(length=64), nullable=False),
|
||||||
@@ -129,9 +160,9 @@ def upgrade() -> None:
|
|||||||
)
|
)
|
||||||
op.create_table('head_auto_apply_run',
|
op.create_table('head_auto_apply_run',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('dry_run', sa.Boolean(), nullable=False),
|
sa.Column('dry_run', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('n_applied', sa.Integer(), nullable=True),
|
sa.Column('n_applied', sa.Integer(), nullable=True),
|
||||||
@@ -144,7 +175,7 @@ def upgrade() -> None:
|
|||||||
op.create_table('head_training_run',
|
op.create_table('head_training_run',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('n_trained', sa.Integer(), nullable=True),
|
sa.Column('n_trained', sa.Integer(), nullable=True),
|
||||||
@@ -161,38 +192,38 @@ def upgrade() -> None:
|
|||||||
sa.Column('scan_mode', sa.String(length=16), nullable=False),
|
sa.Column('scan_mode', sa.String(length=16), nullable=False),
|
||||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('total_files', sa.Integer(), nullable=False),
|
sa.Column('total_files', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('imported', sa.Integer(), nullable=False),
|
sa.Column('imported', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('skipped', sa.Integer(), nullable=False),
|
sa.Column('skipped', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('failed', sa.Integer(), nullable=False),
|
sa.Column('failed', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('attachments', sa.Integer(), nullable=False),
|
sa.Column('attachments', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('refreshed', sa.Integer(), nullable=False),
|
sa.Column('refreshed', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_import_batch'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_import_batch'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_import_batch_status'), 'import_batch', ['status'], unique=False)
|
op.create_index(op.f('ix_import_batch_status'), 'import_batch', ['status'], unique=False)
|
||||||
op.create_table('import_settings',
|
op.create_table('import_settings',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('import_scan_path', sa.Text(), nullable=False),
|
sa.Column('import_scan_path', sa.Text(), server_default='/import', nullable=False),
|
||||||
sa.Column('min_width', sa.Integer(), nullable=False),
|
sa.Column('min_width', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('min_height', sa.Integer(), nullable=False),
|
sa.Column('min_height', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('skip_transparent', sa.Boolean(), nullable=False),
|
sa.Column('skip_transparent', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('transparency_threshold', sa.Float(), nullable=False),
|
sa.Column('transparency_threshold', sa.Float(), server_default='0.9', nullable=False),
|
||||||
sa.Column('skip_single_color', sa.Boolean(), nullable=False),
|
sa.Column('skip_single_color', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('single_color_threshold', sa.Float(), nullable=False),
|
sa.Column('single_color_threshold', sa.Float(), server_default='0.95', nullable=False),
|
||||||
sa.Column('single_color_tolerance', sa.Integer(), nullable=False),
|
sa.Column('single_color_tolerance', sa.Integer(), server_default='30', nullable=False),
|
||||||
sa.Column('phash_threshold', sa.Integer(), nullable=False),
|
sa.Column('phash_threshold', sa.Integer(), server_default='10', nullable=False),
|
||||||
sa.Column('download_rate_limit_seconds', sa.Float(), nullable=False),
|
sa.Column('download_rate_limit_seconds', sa.Float(), server_default='3', nullable=False),
|
||||||
sa.Column('download_validate_files', sa.Boolean(), nullable=False),
|
sa.Column('download_validate_files', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('download_schedule_default_seconds', sa.Integer(), nullable=False),
|
sa.Column('download_schedule_default_seconds', sa.Integer(), server_default='28800', nullable=False),
|
||||||
sa.Column('download_event_retention_days', sa.Integer(), nullable=False),
|
sa.Column('download_event_retention_days', sa.Integer(), server_default='90', nullable=False),
|
||||||
sa.Column('download_failure_warning_threshold', sa.Integer(), nullable=False),
|
sa.Column('download_failure_warning_threshold', sa.Integer(), server_default='5', nullable=False),
|
||||||
sa.Column('backup_db_nightly_enabled', sa.Boolean(), nullable=False),
|
sa.Column('backup_db_nightly_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('backup_db_nightly_hour_utc', sa.Integer(), nullable=False),
|
sa.Column('backup_db_nightly_hour_utc', sa.Integer(), server_default='3', nullable=False),
|
||||||
sa.Column('backup_db_keep_last_n', sa.Integer(), nullable=False),
|
sa.Column('backup_db_keep_last_n', sa.Integer(), server_default='14', nullable=False),
|
||||||
sa.Column('backup_images_keep_last_n', sa.Integer(), nullable=False),
|
sa.Column('backup_images_keep_last_n', sa.Integer(), server_default='3', nullable=False),
|
||||||
sa.Column('series_suggest_enabled', sa.Boolean(), nullable=False),
|
sa.Column('series_suggest_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('series_suggest_threshold', sa.Float(), nullable=False),
|
sa.Column('series_suggest_threshold', sa.Float(), server_default='0.5', nullable=False),
|
||||||
sa.Column('extdl_mega_enabled', sa.Boolean(), server_default='true', nullable=False),
|
sa.Column('extdl_mega_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('extdl_gdrive_enabled', sa.Boolean(), server_default='true', nullable=False),
|
sa.Column('extdl_gdrive_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('extdl_mediafire_enabled', sa.Boolean(), server_default='true', nullable=False),
|
sa.Column('extdl_mediafire_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
@@ -201,7 +232,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('translation_enabled', sa.Boolean(), server_default='false', nullable=False),
|
sa.Column('translation_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('interpreter_base_url', sa.Text(), server_default='', nullable=False),
|
sa.Column('interpreter_base_url', sa.Text(), server_default='', nullable=False),
|
||||||
sa.Column('translation_target_lang', sa.Text(), server_default='en', nullable=False),
|
sa.Column('translation_target_lang', sa.Text(), server_default='en', nullable=False),
|
||||||
sa.Column('translation_min_confidence', sa.Float(), server_default='0.9', nullable=False),
|
sa.Column('translation_min_confidence', sa.Float(), server_default=sa.text('0.9'), nullable=False),
|
||||||
sa.Column('wip_title_tagging_enabled', sa.Boolean(), server_default='true', nullable=False),
|
sa.Column('wip_title_tagging_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('wip_soft_title_tagging_enabled', sa.Boolean(), server_default='false', nullable=False),
|
sa.Column('wip_soft_title_tagging_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.CheckConstraint('id = 1', name=op.f('ck_import_settings_singleton')),
|
sa.CheckConstraint('id = 1', name=op.f('ck_import_settings_singleton')),
|
||||||
@@ -211,14 +242,14 @@ def upgrade() -> None:
|
|||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('rule', sa.String(length=32), nullable=False),
|
sa.Column('rule', sa.String(length=32), nullable=False),
|
||||||
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('scanned_count', sa.Integer(), nullable=False),
|
sa.Column('scanned_count', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('matched_count', sa.Integer(), nullable=False),
|
sa.Column('matched_count', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('matched_ids', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
sa.Column('matched_ids', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'[]'::jsonb"), nullable=False),
|
||||||
sa.Column('error', sa.Text(), nullable=True),
|
sa.Column('error', sa.Text(), nullable=True),
|
||||||
sa.Column('resume_after_id', sa.Integer(), nullable=False),
|
sa.Column('resume_after_id', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_library_audit_run'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_library_audit_run'))
|
||||||
)
|
)
|
||||||
@@ -226,40 +257,40 @@ def upgrade() -> None:
|
|||||||
op.create_index(op.f('ix_library_audit_run_status'), 'library_audit_run', ['status'], unique=False)
|
op.create_index(op.f('ix_library_audit_run_status'), 'library_audit_run', ['status'], unique=False)
|
||||||
op.create_table('ml_settings',
|
op.create_table('ml_settings',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('cpu_embed_enabled', sa.Boolean(), nullable=False),
|
sa.Column('cpu_embed_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('video_frame_interval_seconds', sa.Float(), nullable=False),
|
sa.Column('video_frame_interval_seconds', sa.Float(), server_default='4', nullable=False),
|
||||||
sa.Column('video_max_frames', sa.Integer(), nullable=False),
|
sa.Column('video_max_frames', sa.Integer(), server_default='64', nullable=False),
|
||||||
sa.Column('head_min_positives', sa.Integer(), nullable=False),
|
sa.Column('head_min_positives', sa.Integer(), server_default='8', nullable=False),
|
||||||
sa.Column('head_auto_apply_precision', sa.Float(), nullable=False),
|
sa.Column('head_auto_apply_precision', sa.Float(), server_default='0.97', nullable=False),
|
||||||
sa.Column('head_auto_apply_enabled', sa.Boolean(), nullable=False),
|
sa.Column('head_auto_apply_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('head_auto_apply_min_positives', sa.Integer(), nullable=False),
|
sa.Column('head_auto_apply_min_positives', sa.Integer(), server_default='30', nullable=False),
|
||||||
sa.Column('ccip_match_threshold', sa.Float(), nullable=False),
|
sa.Column('ccip_match_threshold', sa.Float(), server_default='0.85', nullable=False),
|
||||||
sa.Column('ccip_auto_apply_enabled', sa.Boolean(), nullable=False),
|
sa.Column('ccip_auto_apply_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('ccip_auto_apply_threshold', sa.Float(), nullable=False),
|
sa.Column('ccip_auto_apply_threshold', sa.Float(), server_default='0.92', nullable=False),
|
||||||
sa.Column('presentation_auto_apply_enabled', sa.Boolean(), nullable=False),
|
sa.Column('presentation_auto_apply_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('presentation_auto_apply_threshold', sa.Float(), nullable=False),
|
sa.Column('presentation_auto_apply_threshold', sa.Float(), server_default=sa.text('0.90'), nullable=False),
|
||||||
sa.Column('presentation_conflict_threshold', sa.Float(), nullable=False),
|
sa.Column('presentation_conflict_threshold', sa.Float(), server_default=sa.text('0.50'), nullable=False),
|
||||||
sa.Column('process_auto_apply_enabled', sa.Boolean(), nullable=False),
|
sa.Column('process_auto_apply_enabled', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('process_auto_apply_threshold', sa.Float(), nullable=False),
|
sa.Column('process_auto_apply_threshold', sa.Float(), server_default='0.90', nullable=False),
|
||||||
sa.Column('process_conflict_threshold', sa.Float(), nullable=False),
|
sa.Column('process_conflict_threshold', sa.Float(), server_default='0.50', nullable=False),
|
||||||
sa.Column('embedder_model_version', sa.String(length=128), nullable=False),
|
sa.Column('embedder_model_version', sa.String(length=128), server_default='siglip2-so400m-patch16-512', nullable=False),
|
||||||
sa.Column('embedder_model_name', sa.String(length=128), nullable=False),
|
sa.Column('embedder_model_name', sa.String(length=128), server_default='google/siglip2-so400m-patch16-512', nullable=False),
|
||||||
sa.Column('detector_person_enabled', sa.Boolean(), nullable=False),
|
sa.Column('detector_person_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('detector_person_weights', sa.String(length=512), nullable=False),
|
sa.Column('detector_person_weights', sa.String(length=512), server_default='yolo11n.pt', nullable=False),
|
||||||
sa.Column('detector_person_conf', sa.Float(), nullable=False),
|
sa.Column('detector_person_conf', sa.Float(), server_default=sa.text('0.35'), nullable=False),
|
||||||
sa.Column('detector_anatomy_enabled', sa.Boolean(), nullable=False),
|
sa.Column('detector_anatomy_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('detector_anatomy_weights', sa.String(length=512), nullable=False),
|
sa.Column('detector_anatomy_weights', sa.String(length=512), server_default='https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt', nullable=False),
|
||||||
sa.Column('detector_anatomy_conf', sa.Float(), nullable=False),
|
sa.Column('detector_anatomy_conf', sa.Float(), server_default=sa.text('0.30'), nullable=False),
|
||||||
sa.Column('detector_panel_enabled', sa.Boolean(), nullable=False),
|
sa.Column('detector_panel_enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('detector_panel_weights', sa.String(length=512), nullable=False),
|
sa.Column('detector_panel_weights', sa.String(length=512), server_default='mosesb/best-comic-panel-detection::best.pt', nullable=False),
|
||||||
sa.Column('detector_panel_conf', sa.Float(), nullable=False),
|
sa.Column('detector_panel_conf', sa.Float(), server_default=sa.text('0.30'), nullable=False),
|
||||||
sa.Column('detector_max_figures', sa.Integer(), nullable=False),
|
sa.Column('detector_max_figures', sa.Integer(), server_default='8', nullable=False),
|
||||||
sa.Column('detector_max_components', sa.Integer(), nullable=False),
|
sa.Column('detector_max_components', sa.Integer(), server_default='8', nullable=False),
|
||||||
sa.Column('detector_max_panels', sa.Integer(), nullable=False),
|
sa.Column('detector_max_panels', sa.Integer(), server_default='8', nullable=False),
|
||||||
sa.Column('detector_max_regions', sa.Integer(), nullable=False),
|
sa.Column('detector_max_regions', sa.Integer(), server_default='128', nullable=False),
|
||||||
sa.Column('detector_dedupe_iou', sa.Float(), nullable=False),
|
sa.Column('detector_dedupe_iou', sa.Float(), server_default=sa.text('0.85'), nullable=False),
|
||||||
sa.Column('ccip_ref_signature', sa.String(length=128), nullable=True),
|
sa.Column('ccip_ref_signature', sa.String(length=128), nullable=True),
|
||||||
sa.Column('ccip_prototype_cap', sa.Integer(), nullable=False),
|
sa.Column('ccip_prototype_cap', sa.Integer(), server_default='64', nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.CheckConstraint('id = 1', name=op.f('ck_ml_settings_singleton')),
|
sa.CheckConstraint('id = 1', name=op.f('ck_ml_settings_singleton')),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_ml_settings'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_ml_settings'))
|
||||||
@@ -267,15 +298,16 @@ def upgrade() -> None:
|
|||||||
op.create_table('tag',
|
op.create_table('tag',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('name', sa.String(length=255), nullable=False),
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
sa.Column('kind', sa.Enum('artist', 'character', 'fandom', 'general', 'series', 'archive', 'post', name='tag_kind'), nullable=False),
|
sa.Column('kind', sa.Enum('artist', 'character', 'fandom', 'general', 'series', 'archive', 'post', name='tag_kind'), server_default='general', nullable=False),
|
||||||
sa.Column('fandom_id', sa.Integer(), nullable=True),
|
sa.Column('fandom_id', sa.Integer(), nullable=True),
|
||||||
sa.Column('is_system', sa.Boolean(), server_default=sa.text('false'), nullable=False),
|
sa.Column('is_system', sa.Boolean(), server_default=sa.text('false'), nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.CheckConstraint("(fandom_id IS NULL) OR (kind = 'character')", name=op.f('ck_tag_ck_tag_fandom_requires_character')),
|
sa.CheckConstraint("(fandom_id IS NULL) OR (kind = 'character')", name=op.f('ck_tag_fandom_requires_character')),
|
||||||
sa.ForeignKeyConstraint(['fandom_id'], ['tag.id'], name=op.f('fk_tag_fandom_id_tag'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['fandom_id'], ['tag.id'], name=op.f('fk_tag_fandom_id_tag'), ondelete='SET NULL'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_tag'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_tag_fandom_id'), 'tag', ['fandom_id'], unique=False)
|
op.create_index(op.f('ix_tag_fandom_id'), 'tag', ['fandom_id'], unique=False)
|
||||||
|
op.create_index('uq_tag_name_kind_fandom', 'tag', ['name', 'kind', sa.literal_column('COALESCE(fandom_id, 0)')], unique=True)
|
||||||
op.create_table('task_run',
|
op.create_table('task_run',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('celery_task_id', sa.String(length=64), nullable=False),
|
sa.Column('celery_task_id', sa.String(length=64), nullable=False),
|
||||||
@@ -285,7 +317,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
sa.Column('started_at', sa.DateTime(timezone=True), nullable=False),
|
||||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('duration_ms', sa.Integer(), nullable=True),
|
sa.Column('duration_ms', sa.Integer(), nullable=True),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='running', nullable=False),
|
||||||
sa.Column('error_type', sa.String(length=128), nullable=True),
|
sa.Column('error_type', sa.String(length=128), nullable=True),
|
||||||
sa.Column('error_message', sa.Text(), nullable=True),
|
sa.Column('error_message', sa.Text(), nullable=True),
|
||||||
sa.Column('retry_count', sa.Integer(), nullable=True),
|
sa.Column('retry_count', sa.Integer(), nullable=True),
|
||||||
@@ -295,10 +327,10 @@ def upgrade() -> None:
|
|||||||
)
|
)
|
||||||
op.create_index(op.f('ix_task_run_celery_task_id'), 'task_run', ['celery_task_id'], unique=False)
|
op.create_index(op.f('ix_task_run_celery_task_id'), 'task_run', ['celery_task_id'], unique=False)
|
||||||
op.create_index(op.f('ix_task_run_finished_at'), 'task_run', ['finished_at'], unique=False)
|
op.create_index(op.f('ix_task_run_finished_at'), 'task_run', ['finished_at'], unique=False)
|
||||||
op.create_index(op.f('ix_task_run_queue'), 'task_run', ['queue'], unique=False)
|
op.create_index('ix_task_run_name_started', 'task_run', ['task_name', sa.literal_column('started_at DESC')], unique=False)
|
||||||
|
op.create_index('ix_task_run_queue_started', 'task_run', ['queue', sa.literal_column('started_at DESC')], unique=False)
|
||||||
op.create_index(op.f('ix_task_run_started_at'), 'task_run', ['started_at'], unique=False)
|
op.create_index(op.f('ix_task_run_started_at'), 'task_run', ['started_at'], unique=False)
|
||||||
op.create_index(op.f('ix_task_run_status'), 'task_run', ['status'], unique=False)
|
op.create_index('ix_task_run_status_started', 'task_run', ['status', sa.literal_column('started_at DESC')], unique=False)
|
||||||
op.create_index(op.f('ix_task_run_task_name'), 'task_run', ['task_name'], unique=False)
|
|
||||||
op.create_table('artist_visit',
|
op.create_table('artist_visit',
|
||||||
sa.Column('artist_id', sa.Integer(), nullable=False),
|
sa.Column('artist_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('last_viewed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('last_viewed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
@@ -314,20 +346,20 @@ def upgrade() -> None:
|
|||||||
)
|
)
|
||||||
op.create_table('head_metric',
|
op.create_table('head_metric',
|
||||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('n_misfires', sa.Integer(), nullable=False),
|
sa.Column('n_misfires', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('n_underfires', sa.Integer(), nullable=False),
|
sa.Column('n_underfires', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metric_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metric_tag_id_tag'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_head_metric'))
|
sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_head_metric'))
|
||||||
)
|
)
|
||||||
op.create_table('head_metrics_snapshot',
|
op.create_table('head_metrics_snapshot',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
sa.Column('tag_id', sa.Integer(), nullable=True),
|
||||||
sa.Column('name', sa.String(length=255), nullable=False),
|
sa.Column('name', sa.String(length=255), nullable=False),
|
||||||
sa.Column('snapshot_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('snapshot_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('n_auto_applied', sa.Integer(), nullable=False),
|
sa.Column('n_auto_applied', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('n_misfires', sa.Integer(), nullable=False),
|
sa.Column('n_misfires', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('n_underfires', sa.Integer(), nullable=False),
|
sa.Column('n_underfires', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('ap', sa.Float(), nullable=True),
|
sa.Column('ap', sa.Float(), nullable=True),
|
||||||
sa.Column('precision_cv', sa.Float(), nullable=True),
|
sa.Column('precision_cv', sa.Float(), nullable=True),
|
||||||
sa.Column('recall', sa.Float(), nullable=True),
|
sa.Column('recall', sa.Float(), nullable=True),
|
||||||
@@ -342,16 +374,17 @@ def upgrade() -> None:
|
|||||||
sa.Column('artist_id', sa.Integer(), nullable=False),
|
sa.Column('artist_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('platform', sa.String(length=64), nullable=False),
|
sa.Column('platform', sa.String(length=64), nullable=False),
|
||||||
sa.Column('url', sa.Text(), nullable=False),
|
sa.Column('url', sa.Text(), nullable=False),
|
||||||
sa.Column('enabled', sa.Boolean(), nullable=False),
|
sa.Column('enabled', sa.Boolean(), server_default='true', nullable=False),
|
||||||
sa.Column('config_overrides', sa.JSON(), nullable=True),
|
sa.Column('config_overrides', sa.JSON(), nullable=True),
|
||||||
sa.Column('last_checked_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('last_checked_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('last_error', sa.Text(), nullable=True),
|
sa.Column('last_error', sa.Text(), nullable=True),
|
||||||
sa.Column('error_type', sa.String(length=32), nullable=True),
|
sa.Column('error_type', sa.String(length=32), nullable=True),
|
||||||
sa.Column('check_interval_override', sa.Integer(), nullable=True),
|
sa.Column('check_interval_override', sa.Integer(), nullable=True),
|
||||||
sa.Column('consecutive_failures', sa.Integer(), nullable=False),
|
sa.Column('consecutive_failures', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('backfill_runs_remaining', sa.Integer(), server_default='0', nullable=False),
|
sa.Column('backfill_runs_remaining', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_source_artist_id_artist'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_source_artist_id_artist'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_source'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_source')),
|
||||||
|
sa.UniqueConstraint('artist_id', 'platform', 'url', name='uq_source_artist_platform_url')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_source_artist_id'), 'source', ['artist_id'], unique=False)
|
op.create_index(op.f('ix_source_artist_id'), 'source', ['artist_id'], unique=False)
|
||||||
op.create_index(op.f('ix_source_error_type'), 'source', ['error_type'], unique=False)
|
op.create_index(op.f('ix_source_error_type'), 'source', ['error_type'], unique=False)
|
||||||
@@ -363,7 +396,7 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(['canonical_tag_id'], ['tag.id'], name=op.f('fk_tag_alias_canonical_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['canonical_tag_id'], ['tag.id'], name=op.f('fk_tag_alias_canonical_tag_id_tag'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('alias_string', 'alias_category', name=op.f('pk_tag_alias'))
|
sa.PrimaryKeyConstraint('alias_string', 'alias_category', name=op.f('pk_tag_alias'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_tag_alias_canonical_tag_id'), 'tag_alias', ['canonical_tag_id'], unique=False)
|
op.create_index('ix_tag_alias_canonical', 'tag_alias', ['canonical_tag_id'], unique=False)
|
||||||
op.create_table('tag_head',
|
op.create_table('tag_head',
|
||||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('embedding_version', sa.String(length=128), nullable=False),
|
sa.Column('embedding_version', sa.String(length=128), nullable=False),
|
||||||
@@ -386,7 +419,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||||
sa.Column('attempts', sa.Integer(), nullable=False),
|
sa.Column('attempts', sa.Integer(), server_default='1', nullable=False),
|
||||||
sa.Column('last_error', sa.Text(), nullable=True),
|
sa.Column('last_error', sa.Text(), nullable=True),
|
||||||
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
@@ -410,7 +443,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||||
sa.Column('attempts', sa.Integer(), nullable=False),
|
sa.Column('attempts', sa.Integer(), server_default='1', nullable=False),
|
||||||
sa.Column('last_error', sa.Text(), nullable=True),
|
sa.Column('last_error', sa.Text(), nullable=True),
|
||||||
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
@@ -448,7 +481,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('translated_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('translated_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('translation_override', sa.String(length=16), server_default='auto', nullable=False),
|
sa.Column('translation_override', sa.String(length=16), server_default='auto', nullable=False),
|
||||||
sa.Column('downloaded_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('downloaded_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.CheckConstraint("translation_override IN ('auto', 'force', 'original')", name=op.f('ck_post_ck_post_translation_override')),
|
sa.CheckConstraint("translation_override IN ('auto', 'force', 'original')", name=op.f('ck_post_translation_override')),
|
||||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_artist_id_artist'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_artist_id_artist'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_post_source_id_source'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_post_source_id_source'), ondelete='SET NULL'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_post')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_post')),
|
||||||
@@ -456,11 +489,12 @@ def upgrade() -> None:
|
|||||||
)
|
)
|
||||||
op.create_index(op.f('ix_post_artist_id'), 'post', ['artist_id'], unique=False)
|
op.create_index(op.f('ix_post_artist_id'), 'post', ['artist_id'], unique=False)
|
||||||
op.create_index(op.f('ix_post_source_id'), 'post', ['source_id'], unique=False)
|
op.create_index(op.f('ix_post_source_id'), 'post', ['source_id'], unique=False)
|
||||||
|
op.create_index('uq_post_artist_external_id_null_source', 'post', ['artist_id', 'external_post_id'], unique=True, postgresql_where=sa.text('source_id IS NULL'))
|
||||||
op.create_table('subscribestar_failed_media',
|
op.create_table('subscribestar_failed_media',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('source_id', sa.Integer(), nullable=False),
|
sa.Column('source_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('filehash', sa.String(length=128), nullable=False),
|
sa.Column('filehash', sa.String(length=128), nullable=False),
|
||||||
sa.Column('attempts', sa.Integer(), nullable=False),
|
sa.Column('attempts', sa.Integer(), server_default='1', nullable=False),
|
||||||
sa.Column('last_error', sa.Text(), nullable=True),
|
sa.Column('last_error', sa.Text(), nullable=True),
|
||||||
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
@@ -487,8 +521,8 @@ def upgrade() -> None:
|
|||||||
sa.Column('status', sa.String(length=32), nullable=False),
|
sa.Column('status', sa.String(length=32), nullable=False),
|
||||||
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('bytes_downloaded', sa.BigInteger(), nullable=False),
|
sa.Column('bytes_downloaded', sa.BigInteger(), server_default='0', nullable=False),
|
||||||
sa.Column('files_count', sa.Integer(), nullable=False),
|
sa.Column('files_count', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('error', sa.Text(), nullable=True),
|
sa.Column('error', sa.Text(), nullable=True),
|
||||||
sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
|
sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_download_event_post_id_post'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_download_event_post_id_post'), ondelete='SET NULL'),
|
||||||
@@ -507,7 +541,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('width', sa.Integer(), nullable=True),
|
sa.Column('width', sa.Integer(), nullable=True),
|
||||||
sa.Column('height', sa.Integer(), nullable=True),
|
sa.Column('height', sa.Integer(), nullable=True),
|
||||||
sa.Column('duration_seconds', sa.Float(), nullable=True),
|
sa.Column('duration_seconds', sa.Float(), nullable=True),
|
||||||
sa.Column('integrity_status', sa.String(length=24), nullable=False),
|
sa.Column('integrity_status', sa.String(length=24), server_default='unknown', nullable=False),
|
||||||
sa.Column('thumbnail_path', sa.Text(), nullable=True),
|
sa.Column('thumbnail_path', sa.Text(), nullable=True),
|
||||||
sa.Column('source_url', sa.Text(), nullable=True),
|
sa.Column('source_url', sa.Text(), nullable=True),
|
||||||
sa.Column('source_filehash', sa.String(length=32), nullable=True),
|
sa.Column('source_filehash', sa.String(length=32), nullable=True),
|
||||||
@@ -520,16 +554,19 @@ def upgrade() -> None:
|
|||||||
sa.Column('effective_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('effective_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('earliest_post_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('earliest_post_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_image_record_artist_id_artist'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name='fk_image_record_artist_id', ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['primary_post_id'], ['post.id'], name=op.f('fk_image_record_primary_post_id_post'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['primary_post_id'], ['post.id'], name=op.f('fk_image_record_primary_post_id_post'), ondelete='SET NULL'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_image_record')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_image_record')),
|
||||||
sa.UniqueConstraint('path', name=op.f('uq_image_record_path'))
|
sa.UniqueConstraint('path', name=op.f('uq_image_record_path')),
|
||||||
|
sa.UniqueConstraint('sha256', name='uq_image_record_sha256')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_image_record_artist_id'), 'image_record', ['artist_id'], unique=False)
|
op.create_index(op.f('ix_image_record_artist_id'), 'image_record', ['artist_id'], unique=False)
|
||||||
|
op.create_index('ix_image_record_earliest_post_date', 'image_record', [sa.literal_column('earliest_post_date DESC'), sa.literal_column('id DESC')], unique=False)
|
||||||
|
op.create_index('ix_image_record_effective_date', 'image_record', [sa.literal_column('effective_date DESC'), sa.literal_column('id DESC')], unique=False)
|
||||||
op.create_index(op.f('ix_image_record_integrity_status'), 'image_record', ['integrity_status'], unique=False)
|
op.create_index(op.f('ix_image_record_integrity_status'), 'image_record', ['integrity_status'], unique=False)
|
||||||
op.create_index(op.f('ix_image_record_phash'), 'image_record', ['phash'], unique=False)
|
op.create_index(op.f('ix_image_record_phash'), 'image_record', ['phash'], unique=False)
|
||||||
op.create_index(op.f('ix_image_record_primary_post_id'), 'image_record', ['primary_post_id'], unique=False)
|
op.create_index(op.f('ix_image_record_primary_post_id'), 'image_record', ['primary_post_id'], unique=False)
|
||||||
op.create_index(op.f('ix_image_record_sha256'), 'image_record', ['sha256'], unique=True)
|
op.create_index('ix_image_record_siglip_hnsw', 'image_record', ['siglip_embedding'], unique=False, postgresql_using='hnsw', postgresql_ops={'siglip_embedding': 'vector_cosine_ops'})
|
||||||
op.create_index(op.f('ix_image_record_source_filehash'), 'image_record', ['source_filehash'], unique=False)
|
op.create_index(op.f('ix_image_record_source_filehash'), 'image_record', ['source_filehash'], unique=False)
|
||||||
op.create_table('post_attachment',
|
op.create_table('post_attachment',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
@@ -582,24 +619,26 @@ def upgrade() -> None:
|
|||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('duration_seconds', sa.Float(), nullable=True),
|
sa.Column('duration_seconds', sa.Float(), nullable=True),
|
||||||
|
sa.CheckConstraint("host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')", name=op.f('ck_external_link_host')),
|
||||||
|
sa.CheckConstraint("status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')", name=op.f('ck_external_link_status')),
|
||||||
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_external_link_artist_id_artist'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_external_link_artist_id_artist'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['attachment_id'], ['post_attachment.id'], name=op.f('fk_external_link_attachment_id_post_attachment'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['attachment_id'], ['post_attachment.id'], name=op.f('fk_external_link_attachment_id_post_attachment'), ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_external_link_post_id_post'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_external_link_post_id_post'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_external_link'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_external_link'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_external_link_artist_id'), 'external_link', ['artist_id'], unique=False)
|
op.create_index(op.f('ix_external_link_artist_id'), 'external_link', ['artist_id'], unique=False)
|
||||||
op.create_index(op.f('ix_external_link_post_id'), 'external_link', ['post_id'], unique=False)
|
op.create_index('ix_external_link_attachment_id', 'external_link', ['attachment_id'], unique=False)
|
||||||
op.create_index('ix_external_link_status', 'external_link', ['status'], unique=False)
|
op.create_index('ix_external_link_status', 'external_link', ['status'], unique=False)
|
||||||
op.create_index('uq_external_link_post_url', 'external_link', ['post_id', 'url'], unique=True)
|
op.create_index('uq_external_link_post_url', 'external_link', ['post_id', 'url'], unique=True)
|
||||||
op.create_table('gpu_job',
|
op.create_table('gpu_job',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('task', sa.String(length=32), nullable=False),
|
sa.Column('task', sa.String(length=32), nullable=False),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||||
sa.Column('lease_token', sa.String(length=64), nullable=True),
|
sa.Column('lease_token', sa.String(length=64), nullable=True),
|
||||||
sa.Column('leased_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('leased_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('lease_expires_at', sa.DateTime(timezone=True), nullable=True),
|
sa.Column('lease_expires_at', sa.DateTime(timezone=True), nullable=True),
|
||||||
sa.Column('attempts', sa.Integer(), nullable=False),
|
sa.Column('attempts', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('error', sa.Text(), nullable=True),
|
sa.Column('error', sa.Text(), nullable=True),
|
||||||
sa.Column('triage_status', sa.String(length=16), nullable=True),
|
sa.Column('triage_status', sa.String(length=16), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
@@ -619,7 +658,7 @@ def upgrade() -> None:
|
|||||||
sa.Column('from_attachment_id', sa.Integer(), nullable=True),
|
sa.Column('from_attachment_id', sa.Integer(), nullable=True),
|
||||||
sa.Column('captured_metadata', sa.JSON(), nullable=True),
|
sa.Column('captured_metadata', sa.JSON(), nullable=True),
|
||||||
sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['from_attachment_id'], ['post_attachment.id'], name=op.f('fk_image_provenance_from_attachment_id_post_attachment'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['from_attachment_id'], ['post_attachment.id'], name='fk_image_provenance_from_attachment', ondelete='SET NULL'),
|
||||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_provenance_image_record_id_image_record'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_provenance_image_record_id_image_record'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_image_provenance_post_id_post'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_image_provenance_post_id_post'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_image_provenance_source_id_source'), ondelete='SET NULL'),
|
sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_image_provenance_source_id_source'), ondelete='SET NULL'),
|
||||||
@@ -653,20 +692,21 @@ def upgrade() -> None:
|
|||||||
op.create_table('image_tag',
|
op.create_table('image_tag',
|
||||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('source', sa.String(length=32), nullable=False),
|
sa.Column('source', sa.String(length=32), server_default='manual', nullable=False),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_tag_image_record_id_image_record'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_tag_image_record_id_image_record'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_image_tag_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_image_tag_tag_id_tag'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_image_tag'))
|
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_image_tag'))
|
||||||
)
|
)
|
||||||
|
op.create_index('ix_image_tag_tag_id', 'image_tag', ['tag_id'], unique=False)
|
||||||
op.create_table('import_task',
|
op.create_table('import_task',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('batch_id', sa.Integer(), nullable=False),
|
sa.Column('batch_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('source_path', sa.Text(), nullable=False),
|
sa.Column('source_path', sa.Text(), nullable=False),
|
||||||
sa.Column('task_type', sa.String(length=16), nullable=False),
|
sa.Column('task_type', sa.String(length=16), nullable=False),
|
||||||
sa.Column('status', sa.String(length=16), nullable=False),
|
sa.Column('status', sa.String(length=16), server_default='pending', nullable=False),
|
||||||
sa.Column('recovery_count', sa.Integer(), nullable=False),
|
sa.Column('recovery_count', sa.Integer(), server_default='0', nullable=False),
|
||||||
sa.Column('refetched', sa.Boolean(), nullable=False),
|
sa.Column('refetched', sa.Boolean(), server_default='false', nullable=False),
|
||||||
sa.Column('result_image_id', sa.Integer(), nullable=True),
|
sa.Column('result_image_id', sa.Integer(), nullable=True),
|
||||||
sa.Column('error', sa.Text(), nullable=True),
|
sa.Column('error', sa.Text(), nullable=True),
|
||||||
sa.Column('size_bytes', sa.BigInteger(), nullable=True),
|
sa.Column('size_bytes', sa.BigInteger(), nullable=True),
|
||||||
@@ -678,6 +718,8 @@ def upgrade() -> None:
|
|||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_import_task'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_import_task'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_import_task_batch_id'), 'import_task', ['batch_id'], unique=False)
|
op.create_index(op.f('ix_import_task_batch_id'), 'import_task', ['batch_id'], unique=False)
|
||||||
|
op.create_index('ix_import_task_created_at_desc', 'import_task', [sa.literal_column('created_at DESC')], unique=False)
|
||||||
|
op.create_index('ix_import_task_result_image_id', 'import_task', ['result_image_id'], unique=False)
|
||||||
op.create_index(op.f('ix_import_task_status'), 'import_task', ['status'], unique=False)
|
op.create_index(op.f('ix_import_task_status'), 'import_task', ['status'], unique=False)
|
||||||
op.create_table('presentation_review',
|
op.create_table('presentation_review',
|
||||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||||
@@ -692,6 +734,9 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_presentation_review_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_presentation_review_tag_id_tag'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_presentation_review'))
|
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_presentation_review'))
|
||||||
)
|
)
|
||||||
|
op.create_index('ix_presentation_review_conflict_tag_id', 'presentation_review', ['conflict_tag_id'], unique=False)
|
||||||
|
op.create_index('ix_presentation_review_resolved_at', 'presentation_review', ['resolved_at'], unique=False)
|
||||||
|
op.create_index('ix_presentation_review_tag_id', 'presentation_review', ['tag_id'], unique=False)
|
||||||
op.create_table('series_page',
|
op.create_table('series_page',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('series_tag_id', sa.Integer(), nullable=False),
|
sa.Column('series_tag_id', sa.Integer(), nullable=False),
|
||||||
@@ -704,7 +749,7 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], name=op.f('fk_series_page_image_id_image_record'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], name=op.f('fk_series_page_image_id_image_record'), ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_page_series_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_page_series_tag_id_tag'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_series_page')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_series_page')),
|
||||||
sa.UniqueConstraint('image_id', name=op.f('uq_series_page_image_id'))
|
sa.UniqueConstraint('image_id', name='uq_series_page_image')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_series_page_series_tag_id'), 'series_page', ['series_tag_id'], unique=False)
|
op.create_index(op.f('ix_series_page_series_tag_id'), 'series_page', ['series_tag_id'], unique=False)
|
||||||
op.create_table('tag_positive_confirmation',
|
op.create_table('tag_positive_confirmation',
|
||||||
@@ -720,11 +765,11 @@ def upgrade() -> None:
|
|||||||
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
sa.Column('image_record_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||||
sa.Column('rejected_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('rejected_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_tag_suggestion_rejection_image_record_id_image_record'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name='fk_tsr_image_record_id_image_record', ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_suggestion_rejection_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name='fk_tsr_tag_id_tag', ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_suggestion_rejection'))
|
sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_suggestion_rejection'))
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_tag_suggestion_rejection_tag_id'), 'tag_suggestion_rejection', ['tag_id'], unique=False)
|
op.create_index('ix_tag_suggestion_rejection_tag', 'tag_suggestion_rejection', ['tag_id'], unique=False)
|
||||||
op.create_table('character_prototype',
|
op.create_table('character_prototype',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
sa.Column('tag_id', sa.Integer(), nullable=False),
|
sa.Column('tag_id', sa.Integer(), nullable=False),
|
||||||
@@ -734,6 +779,7 @@ def upgrade() -> None:
|
|||||||
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_character_prototype_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_character_prototype_tag_id_tag'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_character_prototype'))
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_character_prototype'))
|
||||||
)
|
)
|
||||||
|
op.create_index(op.f('ix_character_prototype_region_id'), 'character_prototype', ['region_id'], unique=False)
|
||||||
op.create_index(op.f('ix_character_prototype_tag_id'), 'character_prototype', ['tag_id'], unique=False)
|
op.create_index(op.f('ix_character_prototype_tag_id'), 'character_prototype', ['tag_id'], unique=False)
|
||||||
op.create_table('series_chapter',
|
op.create_table('series_chapter',
|
||||||
sa.Column('id', sa.Integer(), nullable=False),
|
sa.Column('id', sa.Integer(), nullable=False),
|
||||||
@@ -743,130 +789,48 @@ def upgrade() -> None:
|
|||||||
sa.Column('stated_part', sa.Integer(), nullable=True),
|
sa.Column('stated_part', sa.Integer(), nullable=True),
|
||||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
||||||
sa.ForeignKeyConstraint(['anchor_page_id'], ['series_page.id'], name=op.f('fk_series_chapter_anchor_page_id_series_page'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['anchor_page_id'], ['series_page.id'], name='fk_series_chapter_anchor_page', ondelete='CASCADE'),
|
||||||
sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_chapter_series_tag_id_tag'), ondelete='CASCADE'),
|
sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_chapter_series_tag_id_tag'), ondelete='CASCADE'),
|
||||||
sa.PrimaryKeyConstraint('id', name=op.f('pk_series_chapter')),
|
sa.PrimaryKeyConstraint('id', name=op.f('pk_series_chapter')),
|
||||||
sa.UniqueConstraint('anchor_page_id', name=op.f('uq_series_chapter_anchor_page_id'))
|
sa.UniqueConstraint('anchor_page_id', name='uq_series_chapter_anchor_page')
|
||||||
)
|
)
|
||||||
op.create_index(op.f('ix_series_chapter_series_tag_id'), 'series_chapter', ['series_tag_id'], unique=False)
|
op.create_index(op.f('ix_series_chapter_series_tag_id'), 'series_chapter', ['series_tag_id'], unique=False)
|
||||||
|
|
||||||
# The HNSW index, item 3 above. Must match the query's cosine-distance
|
# The singleton settings rows. NOT schema — see the note above; the app
|
||||||
# operator class or the planner will not use it.
|
# reads these with scalar_one() and never creates them, so a fresh
|
||||||
|
# install without these two rows raises NoResultFound on first use.
|
||||||
|
# From 0002 and 0003.
|
||||||
|
op.execute("INSERT INTO import_settings (id) VALUES (1)")
|
||||||
|
op.execute("INSERT INTO ml_settings (id) VALUES (1)")
|
||||||
|
|
||||||
|
# The three hygiene system tags, from 0075. These are PRODUCT data, not
|
||||||
|
# operator configuration — 0075's own docstring says so: "the fix keys on
|
||||||
|
# SYSTEM tags the product ships". The presentation and process auto-apply
|
||||||
|
# sweeps look them up with scalar_one(), so without these rows those
|
||||||
|
# features raise NoResultFound rather than degrading.
|
||||||
|
#
|
||||||
|
# 0075 adopted an existing same-name general tag before inserting, because
|
||||||
|
# an operator might already have tagged `wip` by hand. That cannot happen
|
||||||
|
# on the empty database this file runs against, but the guard is kept: it
|
||||||
|
# costs nothing and makes the statement safe to re-run.
|
||||||
|
for _name in ("wip", "banner", "editor screenshot"):
|
||||||
op.execute(
|
op.execute(
|
||||||
"CREATE INDEX ix_image_record_siglip_hnsw "
|
sa.text(
|
||||||
"ON image_record USING hnsw (siglip_embedding vector_cosine_ops)"
|
"INSERT INTO tag (name, kind, is_system) "
|
||||||
|
"SELECT :name, 'general', true WHERE NOT EXISTS ("
|
||||||
|
" SELECT 1 FROM tag WHERE lower(name) = lower(:name)"
|
||||||
|
")"
|
||||||
|
).bindparams(name=_name)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
# Dropping image_record takes its indexes with it, so the HNSW index needs
|
"""Deliberately not implemented.
|
||||||
# no separate drop. The extensions are deliberately left in place: they are
|
|
||||||
# database-scoped and something else may be using them.
|
Downgrading a baseline means dropping every table in the database. That is
|
||||||
op.drop_index(op.f('ix_series_chapter_series_tag_id'), table_name='series_chapter')
|
not a migration, and offering it as one invites someone to run it. Restore
|
||||||
op.drop_table('series_chapter')
|
from a backup instead.
|
||||||
op.drop_index(op.f('ix_character_prototype_tag_id'), table_name='character_prototype')
|
"""
|
||||||
op.drop_table('character_prototype')
|
raise NotImplementedError(
|
||||||
op.drop_index(op.f('ix_tag_suggestion_rejection_tag_id'), table_name='tag_suggestion_rejection')
|
"0089 is the baseline; there is nothing below it. Restore from a backup."
|
||||||
op.drop_table('tag_suggestion_rejection')
|
)
|
||||||
op.drop_index(op.f('ix_tag_positive_confirmation_tag_id'), table_name='tag_positive_confirmation')
|
|
||||||
op.drop_table('tag_positive_confirmation')
|
|
||||||
op.drop_index(op.f('ix_series_page_series_tag_id'), table_name='series_page')
|
|
||||||
op.drop_table('series_page')
|
|
||||||
op.drop_table('presentation_review')
|
|
||||||
op.drop_index(op.f('ix_import_task_status'), table_name='import_task')
|
|
||||||
op.drop_index(op.f('ix_import_task_batch_id'), table_name='import_task')
|
|
||||||
op.drop_table('import_task')
|
|
||||||
op.drop_table('image_tag')
|
|
||||||
op.drop_index(op.f('ix_image_region_image_record_id'), table_name='image_region')
|
|
||||||
op.drop_table('image_region')
|
|
||||||
op.drop_index(op.f('ix_image_provenance_source_id'), table_name='image_provenance')
|
|
||||||
op.drop_index(op.f('ix_image_provenance_post_id'), table_name='image_provenance')
|
|
||||||
op.drop_index(op.f('ix_image_provenance_image_record_id'), table_name='image_provenance')
|
|
||||||
op.drop_index(op.f('ix_image_provenance_from_attachment_id'), table_name='image_provenance')
|
|
||||||
op.drop_table('image_provenance')
|
|
||||||
op.drop_index(op.f('ix_gpu_job_status'), table_name='gpu_job')
|
|
||||||
op.drop_index('ix_gpu_job_pending', table_name='gpu_job', postgresql_where=sa.text("status = 'pending'"))
|
|
||||||
op.drop_index('ix_gpu_job_leased_expires', table_name='gpu_job', postgresql_where=sa.text("status = 'leased'"))
|
|
||||||
op.drop_index(op.f('ix_gpu_job_image_record_id'), table_name='gpu_job')
|
|
||||||
op.drop_table('gpu_job')
|
|
||||||
op.drop_index('uq_external_link_post_url', table_name='external_link')
|
|
||||||
op.drop_index('ix_external_link_status', table_name='external_link')
|
|
||||||
op.drop_index(op.f('ix_external_link_post_id'), table_name='external_link')
|
|
||||||
op.drop_index(op.f('ix_external_link_artist_id'), table_name='external_link')
|
|
||||||
op.drop_table('external_link')
|
|
||||||
op.drop_index(op.f('ix_series_suggestion_status'), table_name='series_suggestion')
|
|
||||||
op.drop_index(op.f('ix_series_suggestion_series_tag_id'), table_name='series_suggestion')
|
|
||||||
op.drop_index(op.f('ix_series_suggestion_post_id'), table_name='series_suggestion')
|
|
||||||
op.drop_table('series_suggestion')
|
|
||||||
op.drop_index('uq_post_attachment_post_sha', table_name='post_attachment', postgresql_where=sa.text('post_id IS NOT NULL'))
|
|
||||||
op.drop_index('uq_post_attachment_null_post_sha', table_name='post_attachment', postgresql_where=sa.text('post_id IS NULL'))
|
|
||||||
op.drop_index(op.f('ix_post_attachment_sha256'), table_name='post_attachment')
|
|
||||||
op.drop_index(op.f('ix_post_attachment_post_id'), table_name='post_attachment')
|
|
||||||
op.drop_index(op.f('ix_post_attachment_artist_id'), table_name='post_attachment')
|
|
||||||
op.drop_table('post_attachment')
|
|
||||||
op.drop_index(op.f('ix_image_record_source_filehash'), table_name='image_record')
|
|
||||||
op.drop_index(op.f('ix_image_record_sha256'), table_name='image_record')
|
|
||||||
op.drop_index(op.f('ix_image_record_primary_post_id'), table_name='image_record')
|
|
||||||
op.drop_index(op.f('ix_image_record_phash'), table_name='image_record')
|
|
||||||
op.drop_index(op.f('ix_image_record_integrity_status'), table_name='image_record')
|
|
||||||
op.drop_index(op.f('ix_image_record_artist_id'), table_name='image_record')
|
|
||||||
op.drop_table('image_record')
|
|
||||||
op.drop_index(op.f('ix_download_event_source_id'), table_name='download_event')
|
|
||||||
op.drop_index(op.f('ix_download_event_post_id'), table_name='download_event')
|
|
||||||
op.drop_table('download_event')
|
|
||||||
op.drop_index(op.f('ix_subscribestar_seen_media_source_id'), table_name='subscribestar_seen_media')
|
|
||||||
op.drop_table('subscribestar_seen_media')
|
|
||||||
op.drop_index(op.f('ix_subscribestar_failed_media_source_id'), table_name='subscribestar_failed_media')
|
|
||||||
op.drop_table('subscribestar_failed_media')
|
|
||||||
op.drop_index(op.f('ix_post_source_id'), table_name='post')
|
|
||||||
op.drop_index(op.f('ix_post_artist_id'), table_name='post')
|
|
||||||
op.drop_table('post')
|
|
||||||
op.drop_index(op.f('ix_pixiv_seen_media_source_id'), table_name='pixiv_seen_media')
|
|
||||||
op.drop_table('pixiv_seen_media')
|
|
||||||
op.drop_index(op.f('ix_pixiv_failed_media_source_id'), table_name='pixiv_failed_media')
|
|
||||||
op.drop_table('pixiv_failed_media')
|
|
||||||
op.drop_index(op.f('ix_patreon_seen_media_source_id'), table_name='patreon_seen_media')
|
|
||||||
op.drop_table('patreon_seen_media')
|
|
||||||
op.drop_index(op.f('ix_patreon_failed_media_source_id'), table_name='patreon_failed_media')
|
|
||||||
op.drop_table('patreon_failed_media')
|
|
||||||
op.drop_table('tag_head')
|
|
||||||
op.drop_index(op.f('ix_tag_alias_canonical_tag_id'), table_name='tag_alias')
|
|
||||||
op.drop_table('tag_alias')
|
|
||||||
op.drop_index(op.f('ix_source_error_type'), table_name='source')
|
|
||||||
op.drop_index(op.f('ix_source_artist_id'), table_name='source')
|
|
||||||
op.drop_table('source')
|
|
||||||
op.drop_index(op.f('ix_head_metrics_snapshot_tag_id'), table_name='head_metrics_snapshot')
|
|
||||||
op.drop_index(op.f('ix_head_metrics_snapshot_snapshot_at'), table_name='head_metrics_snapshot')
|
|
||||||
op.drop_table('head_metrics_snapshot')
|
|
||||||
op.drop_table('head_metric')
|
|
||||||
op.drop_table('ccip_prototype_state')
|
|
||||||
op.drop_table('artist_visit')
|
|
||||||
op.drop_index(op.f('ix_task_run_task_name'), table_name='task_run')
|
|
||||||
op.drop_index(op.f('ix_task_run_status'), table_name='task_run')
|
|
||||||
op.drop_index(op.f('ix_task_run_started_at'), table_name='task_run')
|
|
||||||
op.drop_index(op.f('ix_task_run_queue'), table_name='task_run')
|
|
||||||
op.drop_index(op.f('ix_task_run_finished_at'), table_name='task_run')
|
|
||||||
op.drop_index(op.f('ix_task_run_celery_task_id'), table_name='task_run')
|
|
||||||
op.drop_table('task_run')
|
|
||||||
op.drop_index(op.f('ix_tag_fandom_id'), table_name='tag')
|
|
||||||
op.drop_table('tag')
|
|
||||||
op.drop_table('ml_settings')
|
|
||||||
op.drop_index(op.f('ix_library_audit_run_status'), table_name='library_audit_run')
|
|
||||||
op.drop_index(op.f('ix_library_audit_run_rule'), table_name='library_audit_run')
|
|
||||||
op.drop_table('library_audit_run')
|
|
||||||
op.drop_table('import_settings')
|
|
||||||
op.drop_index(op.f('ix_import_batch_status'), table_name='import_batch')
|
|
||||||
op.drop_table('import_batch')
|
|
||||||
op.drop_index(op.f('ix_head_training_run_status'), table_name='head_training_run')
|
|
||||||
op.drop_table('head_training_run')
|
|
||||||
op.drop_index(op.f('ix_head_auto_apply_run_status'), table_name='head_auto_apply_run')
|
|
||||||
op.drop_table('head_auto_apply_run')
|
|
||||||
op.drop_table('credential')
|
|
||||||
op.drop_index(op.f('ix_backup_run_tag'), table_name='backup_run')
|
|
||||||
op.drop_index(op.f('ix_backup_run_status'), table_name='backup_run')
|
|
||||||
op.drop_index(op.f('ix_backup_run_started_at'), table_name='backup_run')
|
|
||||||
op.drop_index(op.f('ix_backup_run_kind'), table_name='backup_run')
|
|
||||||
op.drop_index(op.f('ix_backup_run_finished_at'), table_name='backup_run')
|
|
||||||
op.drop_table('backup_run')
|
|
||||||
op.drop_table('artist')
|
|
||||||
op.drop_table('app_setting')
|
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
"""service_seen — the learned roster that makes a stopped part observable.
|
||||||
|
|
||||||
|
Milestone 365. Nothing in FabledCurator knew what was SUPPOSED to be running:
|
||||||
|
`celery inspect` reports the workers that answer, so a dead worker was a
|
||||||
|
shorter list rather than a red light, and the only surface that could tell an
|
||||||
|
operator otherwise was Portainer. This table is the memory that turns an
|
||||||
|
absence into something the app can see.
|
||||||
|
|
||||||
|
Keyed on the queue set for a celery role and on agent_id for the GPU agent —
|
||||||
|
NOT on the celery worker name, which here is `celery@<container id>` and is
|
||||||
|
minted fresh on every deploy. See the model docstring for why that choice is
|
||||||
|
the whole design.
|
||||||
|
|
||||||
|
## First migration on the collapsed baseline
|
||||||
|
|
||||||
|
0089 is the single generated baseline that replaced revisions 0001..0089
|
||||||
|
(milestone 328). This is the first revision written on top of it, so it is
|
||||||
|
also the first evidence that the chain steps forward from the collapse rather
|
||||||
|
than merely reproducing the schema — which nothing had demonstrated yet.
|
||||||
|
|
||||||
|
An existing install is at 0089 because it ran the real 0089; a fresh one is at
|
||||||
|
0089 because it ran the baseline. Both arrive here identically, which was the
|
||||||
|
property the collapse was designed around.
|
||||||
|
|
||||||
|
Revision ID: 0090
|
||||||
|
Revises: 0089
|
||||||
|
Create Date: 2026-09-02
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0090"
|
||||||
|
down_revision: Union[str, None] = "0089"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"service_seen",
|
||||||
|
sa.Column("key", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||||
|
sa.Column("display_name", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column(
|
||||||
|
"first_seen_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"last_seen_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("details", sa.JSON(), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("key", name=op.f("pk_service_seen")),
|
||||||
|
)
|
||||||
|
# No secondary indexes, deliberately: one row per moving part means every
|
||||||
|
# read is a handful of rows and an index would be write cost buying
|
||||||
|
# nothing (#3301 removed seven of exactly that shape).
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("service_seen")
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""platform_membership — the learned roster of what the account actually pays for.
|
||||||
|
|
||||||
|
Milestone 387, phase C. FC knows which creators it was told to follow and
|
||||||
|
nothing about which ones the operator is subscribed to; this table is the
|
||||||
|
memory that makes the drift in both directions observable. See the model
|
||||||
|
docstring for why the roster is learned rather than looked up live, and why
|
||||||
|
`status` holds the platform's own word rather than a normalised FC value.
|
||||||
|
|
||||||
|
## Nothing populates this yet, on purpose
|
||||||
|
|
||||||
|
The sweep that fills it (C3) depends on a client seam (C2) that depends on
|
||||||
|
characterising Patreon's real membership response from a captured sample (C0),
|
||||||
|
which needs the operator's authenticated browser session. The table's SHAPE
|
||||||
|
does not wait on that: it is deliberately free-form where C0's findings would
|
||||||
|
otherwise dictate a column — `status` is an unconstrained String and `details`
|
||||||
|
keeps the raw payload — so no capture can invalidate what is created here.
|
||||||
|
|
||||||
|
An empty table is the correct intermediate state. It is not dead code: C5 reads
|
||||||
|
it to explain a tier-limited source, and C4 reads it to reconcile.
|
||||||
|
|
||||||
|
Revision ID: 0091
|
||||||
|
Revises: 0090
|
||||||
|
Create Date: 2026-09-10
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0091"
|
||||||
|
down_revision: Union[str, None] = "0090"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"platform_membership",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("platform", sa.String(length=64), nullable=False),
|
||||||
|
# Text, not a bounded String: an opaque upstream identifier we do not
|
||||||
|
# mint, and guessing a ceiling for one is how a walk dies on a silent
|
||||||
|
# truncation.
|
||||||
|
sa.Column("external_campaign_id", sa.Text(), nullable=False),
|
||||||
|
sa.Column("display_name", sa.Text(), nullable=True),
|
||||||
|
sa.Column("url", sa.Text(), nullable=True),
|
||||||
|
# No CHECK, deliberately (rule 36 considered and declined): the
|
||||||
|
# vocabulary is each platform's own and is not ours to fix before C0
|
||||||
|
# has characterised even one of them. The service owns the whitelist.
|
||||||
|
sa.Column("status", sa.String(length=32), nullable=True),
|
||||||
|
sa.Column("tier_names", sa.JSON(), nullable=True),
|
||||||
|
sa.Column("amount_cents", sa.Integer(), nullable=True),
|
||||||
|
sa.Column("currency", sa.String(length=8), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"first_seen_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"last_seen_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column("details", sa.JSON(), nullable=False),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_platform_membership")),
|
||||||
|
# The upsert's conflict target. Named explicitly because
|
||||||
|
# touch_membership references it by name in ON CONFLICT — an
|
||||||
|
# autogenerated name would make that call break on a rename nobody
|
||||||
|
# connected to it.
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"platform", "external_campaign_id",
|
||||||
|
name="uq_platform_membership_platform_campaign",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# No secondary indexes. This table holds one row per subscription — tens,
|
||||||
|
# not millions — so every query against it is a short scan and an index
|
||||||
|
# would be write cost buying nothing (#3301 removed seven of that shape).
|
||||||
|
# The unique constraint above already backs the only lookup that matters.
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("platform_membership")
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
"""Synthetic posts — FC authors a post for content that arrived as chat.
|
||||||
|
|
||||||
|
Milestone 388, step E2. Discord is a delivery channel, not a publisher: one
|
||||||
|
message is not one post, and today every message becomes its own `post` row
|
||||||
|
competing with authored work for the same surface. This adds the three columns
|
||||||
|
that let FC group a creator's variant drop into a post it wrote itself, while
|
||||||
|
keeping that fact visible and the grouping reversible.
|
||||||
|
|
||||||
|
## Why a flag and a back-pointer rather than a separate table
|
||||||
|
|
||||||
|
A synthetic post has to BE a post — same row, same columns — or every existing
|
||||||
|
surface (feed, provenance, translation, attachments, series) would need a
|
||||||
|
second code path for it. `synthesized_by` marks the ones FC authored;
|
||||||
|
`absorbed_by_post_id` points a member message-post at the post that replaced
|
||||||
|
it in the feed. The members are not deleted: they remain the images' true
|
||||||
|
origin, and destroying them would make the grouping un-auditable at exactly
|
||||||
|
the moment somebody wants to check it.
|
||||||
|
|
||||||
|
Reversal is one DELETE. `absorbed_by_post_id` is ON DELETE SET NULL, so
|
||||||
|
removing a synthetic post releases its members and they return to the feed
|
||||||
|
unaided.
|
||||||
|
|
||||||
|
Revision ID: 0092
|
||||||
|
Revises: 0091
|
||||||
|
Create Date: 2026-09-10
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0092"
|
||||||
|
down_revision: Union[str, None] = "0091"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# No CHECK on synthesized_by (rule 36 considered and declined): there is one
|
||||||
|
# grouper today and a second would be a new VALUE, not a new invariant —
|
||||||
|
# matching source.error_type and service_seen.kind.
|
||||||
|
op.add_column("post", sa.Column("synthesized_by", sa.String(length=32), nullable=True))
|
||||||
|
op.add_column("post", sa.Column("synthesis_details", sa.JSON(), nullable=True))
|
||||||
|
op.add_column(
|
||||||
|
"post", sa.Column("absorbed_by_post_id", sa.Integer(), nullable=True),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_post_absorbed_by_post_id"), "post", ["absorbed_by_post_id"],
|
||||||
|
)
|
||||||
|
# SET NULL, not CASCADE: deleting the synthetic post must RELEASE its
|
||||||
|
# members, never take them with it. The members are the real capture.
|
||||||
|
op.create_foreign_key(
|
||||||
|
"fk_post_absorbed_by_post_id_post", "post", "post",
|
||||||
|
["absorbed_by_post_id"], ["id"], ondelete="SET NULL",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Grouping tunables. Every one of these is operator-facing (project rule
|
||||||
|
# 25) because the quality bar here is a judgement call no test can settle:
|
||||||
|
# too greedy merges distinct pieces, too shy leaves a drop scattered.
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_grouping_enabled", sa.Boolean(),
|
||||||
|
server_default="true", nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_group_max_distance", sa.Float(),
|
||||||
|
server_default=sa.text("0.10"), nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_group_window_minutes", sa.Float(),
|
||||||
|
server_default=sa.text("60"), nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("ml_settings", "discord_group_window_minutes")
|
||||||
|
op.drop_column("ml_settings", "discord_group_max_distance")
|
||||||
|
op.drop_column("ml_settings", "discord_grouping_enabled")
|
||||||
|
op.drop_constraint("fk_post_absorbed_by_post_id_post", "post", type_="foreignkey")
|
||||||
|
op.drop_index(op.f("ix_post_absorbed_by_post_id"), table_name="post")
|
||||||
|
op.drop_column("post", "absorbed_by_post_id")
|
||||||
|
op.drop_column("post", "synthesis_details")
|
||||||
|
op.drop_column("post", "synthesized_by")
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""An open grouping — a synthetic post that a later drop can still join.
|
||||||
|
|
||||||
|
Milestone 388, step E3. E2's synthetic post was sealed at creation: a creator
|
||||||
|
who added two more variants the next day started a second post. These two
|
||||||
|
columns let the group stay open and absorb the follow-up, without the post
|
||||||
|
either freezing or thrashing the feed.
|
||||||
|
|
||||||
|
## Why openness is derived rather than stored
|
||||||
|
|
||||||
|
There is no `closed_at` here on purpose. A group is open if it grew (or
|
||||||
|
started) within `ml_settings.discord_group_close_after_hours`, so openness is a
|
||||||
|
comparison rather than a state — which means lowering the setting closes old
|
||||||
|
groups and raising it reopens them, with nothing to repair either way. A stored
|
||||||
|
flag would need its own sweep to set it and its own repair path to ever change
|
||||||
|
the policy, for no gain.
|
||||||
|
|
||||||
|
## Why `resurfaced_at` is separate from `last_grew_at`
|
||||||
|
|
||||||
|
They answer different questions. `last_grew_at` is when the group last
|
||||||
|
absorbed something — it decides how long the group stays joinable and is what
|
||||||
|
the card shows. `resurfaced_at` is the FEED POSITION, advanced only when the
|
||||||
|
anti-thrash rule fires, so a group that gains one image a day updates in place
|
||||||
|
while a genuine second wave moves once. Folding them together would make every
|
||||||
|
addition a bump, which is the annoyance this step exists to avoid.
|
||||||
|
|
||||||
|
Both are NULL on every ordinary post, so the feed's sort key can COALESCE
|
||||||
|
through `resurfaced_at` without moving anything that is not a grouping.
|
||||||
|
|
||||||
|
Revision ID: 0093
|
||||||
|
Revises: 0092
|
||||||
|
Create Date: 2026-09-10
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0093"
|
||||||
|
down_revision: Union[str, None] = "0092"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column(
|
||||||
|
"post", sa.Column("last_grew_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"post", sa.Column("resurfaced_at", sa.DateTime(timezone=True), nullable=True),
|
||||||
|
)
|
||||||
|
# No index on either. The feed already sorts on an unindexed
|
||||||
|
# COALESCE(post_date, downloaded_at) expression, so adding resurfaced_at to
|
||||||
|
# that COALESCE changes nothing about how the query plans — and inventing a
|
||||||
|
# functional index here would be guessing at the fix for a cost nobody has
|
||||||
|
# measured. Measuring it is step B2's job.
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_group_close_after_hours", sa.Float(),
|
||||||
|
server_default=sa.text("168"), nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_group_resurface_min_images", sa.Integer(),
|
||||||
|
server_default="2", nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"ml_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_group_resurface_cooldown_hours", sa.Float(),
|
||||||
|
server_default=sa.text("24"), nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("ml_settings", "discord_group_resurface_cooldown_hours")
|
||||||
|
op.drop_column("ml_settings", "discord_group_resurface_min_images")
|
||||||
|
op.drop_column("ml_settings", "discord_group_close_after_hours")
|
||||||
|
op.drop_column("post", "resurfaced_at")
|
||||||
|
op.drop_column("post", "last_grew_at")
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""post_association — "this Patreon post announced that Discord drop".
|
||||||
|
|
||||||
|
Milestone 388, step E5.
|
||||||
|
|
||||||
|
Two of the operator's artists post a deliberately cropped fragment on Patreon
|
||||||
|
to signal that the real thing has landed in their Discord. This table holds the
|
||||||
|
proposed and accepted links between the announcement and the drop.
|
||||||
|
|
||||||
|
Directional and confirm-only. The pair is asymmetric (the teaser announces the
|
||||||
|
drop, not the reverse), the two posts are never merged (the creator published
|
||||||
|
twice, deliberately — flattening that hides the behaviour being modelled), and
|
||||||
|
nothing is linked until the operator accepts, following the FC-6.3 series
|
||||||
|
matcher. A wrongly-asserted association tells them two different pieces are
|
||||||
|
one, which is worse than no link at all.
|
||||||
|
|
||||||
|
Dismissed rows are KEPT. The row is what remembers the rejection, and
|
||||||
|
re-proposing a rejected pair on every scan is what makes a review queue get
|
||||||
|
ignored.
|
||||||
|
|
||||||
|
Revision ID: 0094
|
||||||
|
Revises: 0093
|
||||||
|
Create Date: 2026-09-10
|
||||||
|
|
||||||
|
"""
|
||||||
|
from typing import Sequence, Union
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision: str = "0094"
|
||||||
|
down_revision: Union[str, None] = "0093"
|
||||||
|
branch_labels: Union[str, Sequence[str], None] = None
|
||||||
|
depends_on: Union[str, Sequence[str], None] = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"post_association",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("announcement_post_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("payload_post_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("score", sa.Float(), nullable=False),
|
||||||
|
sa.Column("signals", sa.JSON(), nullable=True),
|
||||||
|
# No CHECK on status (rule 36 considered and declined), matching
|
||||||
|
# series_suggestion.status — the same review-queue vocabulary, and the
|
||||||
|
# same check-existing-enums lesson.
|
||||||
|
sa.Column(
|
||||||
|
"status", sa.String(length=16), server_default="pending", nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"created_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"updated_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_post_association")),
|
||||||
|
# CASCADE on both sides: an association to a post that no longer exists
|
||||||
|
# is not a fact worth keeping, and E3's reversal path (delete the
|
||||||
|
# grouping) must not leave a dangling proposal behind.
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["announcement_post_id"], ["post.id"], ondelete="CASCADE",
|
||||||
|
name=op.f("fk_post_association_announcement_post_id_post"),
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["payload_post_id"], ["post.id"], ondelete="CASCADE",
|
||||||
|
name=op.f("fk_post_association_payload_post_id_post"),
|
||||||
|
),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"announcement_post_id", "payload_post_id",
|
||||||
|
name="uq_post_association_pair",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_post_association_announcement_post_id"),
|
||||||
|
"post_association", ["announcement_post_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_post_association_payload_post_id"),
|
||||||
|
"post_association", ["payload_post_id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_post_association_status"), "post_association", ["status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_link_enabled", sa.Boolean(),
|
||||||
|
server_default="true", nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# 0.60 sits ABOVE the largest single signal weight on purpose — see
|
||||||
|
# post_association_service.WEIGHTS. That is what makes "time proximity
|
||||||
|
# alone is never sufficient" arithmetic rather than aspirational.
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_link_threshold", sa.Float(),
|
||||||
|
server_default="0.60", nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"discord_link_window_hours", sa.Float(),
|
||||||
|
server_default="24", nullable=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("import_settings", "discord_link_window_hours")
|
||||||
|
op.drop_column("import_settings", "discord_link_threshold")
|
||||||
|
op.drop_column("import_settings", "discord_link_enabled")
|
||||||
|
op.drop_index(op.f("ix_post_association_status"), table_name="post_association")
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_post_association_payload_post_id"), table_name="post_association",
|
||||||
|
)
|
||||||
|
op.drop_index(
|
||||||
|
op.f("ix_post_association_announcement_post_id"), table_name="post_association",
|
||||||
|
)
|
||||||
|
op.drop_table("post_association")
|
||||||
@@ -38,6 +38,7 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
from .suggestions import suggestions_bp
|
from .suggestions import suggestions_bp
|
||||||
from .system_activity import system_activity_bp
|
from .system_activity import system_activity_bp
|
||||||
from .system_backup import system_backup_bp
|
from .system_backup import system_backup_bp
|
||||||
|
from .system_health import system_health_bp
|
||||||
from .tags import tags_bp
|
from .tags import tags_bp
|
||||||
from .thumbnails import thumbnails_bp
|
from .thumbnails import thumbnails_bp
|
||||||
return [
|
return [
|
||||||
@@ -51,6 +52,7 @@ def all_blueprints() -> list[Blueprint]:
|
|||||||
showcase_bp,
|
showcase_bp,
|
||||||
settings_bp,
|
settings_bp,
|
||||||
system_activity_bp,
|
system_activity_bp,
|
||||||
|
system_health_bp,
|
||||||
system_backup_bp,
|
system_backup_bp,
|
||||||
admin_bp,
|
admin_bp,
|
||||||
cleanup_bp,
|
cleanup_bp,
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from ..services.gallery_service import image_url
|
|||||||
from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
|
from ..services.ml.gpu_jobs import GpuJobService, error_dedupe_statements
|
||||||
from ..services.ml.gpu_triage import classify_reason, recover_defective_image
|
from ..services.ml.gpu_triage import classify_reason, recover_defective_image
|
||||||
from ..services.ml.regions import RegionService
|
from ..services.ml.regions import RegionService
|
||||||
|
from ..services.service_roster import touch_service
|
||||||
|
|
||||||
gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
|
gpu_bp = Blueprint("gpu", __name__, url_prefix="/api/gpu")
|
||||||
|
|
||||||
@@ -256,6 +257,18 @@ async def lease():
|
|||||||
if not await _agent_authed(session):
|
if not await _agent_authed(session):
|
||||||
return jsonify({"error": "unauthorized"}), 401
|
return jsonify({"error": "unauthorized"}), 401
|
||||||
jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
|
jobs = await GpuJobService(session).lease(agent_id, batch_size=batch)
|
||||||
|
# The agent cannot be polled — it is HTTP-only and pulls from here, so
|
||||||
|
# web never dials it. A lease IS the check-in, and until milestone 365
|
||||||
|
# it was thrown away: an agent sitting idle with nothing to lease left
|
||||||
|
# no trace at all and was indistinguishable from one switched off a
|
||||||
|
# week ago. Recorded on the call that was already happening.
|
||||||
|
await touch_service(
|
||||||
|
session,
|
||||||
|
key=f"agent:{agent_id}",
|
||||||
|
kind="agent",
|
||||||
|
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||||
|
details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)},
|
||||||
|
)
|
||||||
ml = await MLSettings.load(session)
|
ml = await MLSettings.load(session)
|
||||||
# image rows for url/mime in one shot
|
# image rows for url/mime in one shot
|
||||||
ids = [j.image_record_id for j in jobs]
|
ids = [j.image_record_id for j in jobs]
|
||||||
@@ -329,6 +342,13 @@ async def heartbeat():
|
|||||||
if not await _agent_authed(session):
|
if not await _agent_authed(session):
|
||||||
return jsonify({"error": "unauthorized"}), 401
|
return jsonify({"error": "unauthorized"}), 401
|
||||||
n = await GpuJobService(session).heartbeat(agent_id, job_ids)
|
n = await GpuJobService(session).heartbeat(agent_id, job_ids)
|
||||||
|
await touch_service(
|
||||||
|
session,
|
||||||
|
key=f"agent:{agent_id}",
|
||||||
|
kind="agent",
|
||||||
|
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||||
|
details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n},
|
||||||
|
)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
return jsonify({"extended": n})
|
return jsonify({"extended": n})
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,17 @@ _EDITABLE = (
|
|||||||
"process_conflict_threshold",
|
"process_conflict_threshold",
|
||||||
"embedder_model_name",
|
"embedder_model_name",
|
||||||
"embedder_model_version",
|
"embedder_model_version",
|
||||||
|
# Discord drop grouping (#388 E2). Operator-facing because the quality bar
|
||||||
|
# is a judgement no test can settle: too greedy merges distinct pieces, too
|
||||||
|
# shy leaves a drop scattered.
|
||||||
|
"discord_grouping_enabled",
|
||||||
|
"discord_group_max_distance",
|
||||||
|
"discord_group_window_minutes",
|
||||||
|
# E3: how long a grouping stays open, and the anti-thrash rule that keeps
|
||||||
|
# a growing one from monopolising the feed.
|
||||||
|
"discord_group_close_after_hours",
|
||||||
|
"discord_group_resurface_min_images",
|
||||||
|
"discord_group_resurface_cooldown_hours",
|
||||||
*_DETECTOR_FIELDS,
|
*_DETECTOR_FIELDS,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -148,6 +159,24 @@ def _validate(p: dict) -> str | None:
|
|||||||
return f"process_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
return f"process_auto_apply_threshold must be between {AUTO_APPLY_THRESHOLD_MIN} and {AUTO_APPLY_THRESHOLD_MAX}"
|
||||||
if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
|
if not (0.0 <= float(p["process_conflict_threshold"]) <= 1.0):
|
||||||
return "process_conflict_threshold must be between 0 and 1"
|
return "process_conflict_threshold must be between 0 and 1"
|
||||||
|
# Discord drop grouping (#388 E2). max_distance is a cosine DISTANCE, so
|
||||||
|
# unlike the *_threshold family above it is not on the auto-apply scale:
|
||||||
|
# 0 is identical and 1 is unrelated, and both ends are legal. The upper
|
||||||
|
# bound is 1.0 rather than AUTO_APPLY_THRESHOLD_MAX for that reason.
|
||||||
|
if not (0.0 <= float(p["discord_group_max_distance"]) <= 1.0):
|
||||||
|
return "discord_group_max_distance must be between 0 and 1"
|
||||||
|
if float(p["discord_group_window_minutes"]) <= 0:
|
||||||
|
return "discord_group_window_minutes must be > 0"
|
||||||
|
# A group must stay open at least as long as the drop window it was cut
|
||||||
|
# with, or the joiner could never reach a message the grouper deferred —
|
||||||
|
# the two would fight, and the symptom (drops that never grow) would look
|
||||||
|
# like the predicate failing rather than a settings contradiction.
|
||||||
|
if float(p["discord_group_close_after_hours"]) * 60 < float(p["discord_group_window_minutes"]):
|
||||||
|
return "discord_group_close_after_hours must be at least the drop window"
|
||||||
|
if int(p["discord_group_resurface_min_images"]) < 1:
|
||||||
|
return "discord_group_resurface_min_images must be >= 1"
|
||||||
|
if float(p["discord_group_resurface_cooldown_hours"]) < 0:
|
||||||
|
return "discord_group_resurface_cooldown_hours must be >= 0"
|
||||||
# Embedder model swap (#1190): both must be non-empty. Changing them means a
|
# Embedder model swap (#1190): both must be non-empty. Changing them means a
|
||||||
# different embedding space — the operator must re-embed + retrain after.
|
# different embedding space — the operator must re-embed + retrain after.
|
||||||
for key in ("embedder_model_name", "embedder_model_version"):
|
for key in ("embedder_model_name", "embedder_model_version"):
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ from quart import Blueprint, jsonify, request
|
|||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import ImportSettings, Post
|
from ..models import ImportSettings, Post
|
||||||
from ..services import interpreter_client as ic
|
from ..services import interpreter_client as ic
|
||||||
|
from ..services.post_association_service import PostAssociationService
|
||||||
|
from ..services.post_association_service import rescan as association_rescan
|
||||||
from ..services.post_feed_service import PostFeedService
|
from ..services.post_feed_service import PostFeedService
|
||||||
from ..services.source_service import KNOWN_PLATFORMS
|
from ..services.source_service import KNOWN_PLATFORMS
|
||||||
from ..utils.text import html_to_plain
|
from ..utils.text import html_to_plain
|
||||||
@@ -165,3 +167,46 @@ async def set_translation_override(post_id: int):
|
|||||||
"translated_source_lang": post.translated_source_lang,
|
"translated_source_lang": post.translated_source_lang,
|
||||||
"applied": applied,
|
"applied": applied,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
# --- #388 E5: the announcement review queue -------------------------------
|
||||||
|
#
|
||||||
|
# Confirm-only, following the series-suggestion routes (api/tags.py). Nothing
|
||||||
|
# here links anything on its own: the matcher proposes, the operator decides.
|
||||||
|
|
||||||
|
|
||||||
|
@posts_bp.route("/associations", methods=["GET"])
|
||||||
|
async def list_associations():
|
||||||
|
async with get_session() as session:
|
||||||
|
return jsonify({"items": await PostAssociationService(session).list_pending()})
|
||||||
|
|
||||||
|
|
||||||
|
@posts_bp.route("/associations/<int:association_id>/accept", methods=["POST"])
|
||||||
|
async def accept_association(association_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await PostAssociationService(session).accept(association_id)
|
||||||
|
if result is None:
|
||||||
|
return _bad("association not found", 404)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@posts_bp.route("/associations/<int:association_id>/dismiss", methods=["POST"])
|
||||||
|
async def dismiss_association(association_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await PostAssociationService(session).dismiss(association_id)
|
||||||
|
if result is None:
|
||||||
|
return _bad("association not found", 404)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@posts_bp.route("/associations/rescan", methods=["POST"])
|
||||||
|
async def rescan_associations():
|
||||||
|
"""Manual re-scan. The beat sweep only looks at recent posts (a pair has to
|
||||||
|
be within the window to exist at all); this is the button for a first run
|
||||||
|
over a library that predates the feature."""
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await association_rescan(session)
|
||||||
|
await session.commit()
|
||||||
|
return jsonify(result)
|
||||||
|
|||||||
@@ -39,6 +39,10 @@ _EDITABLE_FIELDS = (
|
|||||||
"download_failure_warning_threshold",
|
"download_failure_warning_threshold",
|
||||||
"series_suggest_enabled",
|
"series_suggest_enabled",
|
||||||
"series_suggest_threshold",
|
"series_suggest_threshold",
|
||||||
|
# #388 E5 — the announcement matcher (Patreon teaser ↔ Discord drop).
|
||||||
|
"discord_link_enabled",
|
||||||
|
"discord_link_threshold",
|
||||||
|
"discord_link_window_hours",
|
||||||
"extdl_mega_enabled",
|
"extdl_mega_enabled",
|
||||||
"extdl_gdrive_enabled",
|
"extdl_gdrive_enabled",
|
||||||
"extdl_mediafire_enabled",
|
"extdl_mediafire_enabled",
|
||||||
@@ -150,6 +154,22 @@ async def update_import_settings():
|
|||||||
return jsonify(
|
return jsonify(
|
||||||
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
{"error": "series_suggest_threshold must be a number in [0, 1]"}
|
||||||
), 400
|
), 400
|
||||||
|
if "discord_link_enabled" in body and not isinstance(
|
||||||
|
body["discord_link_enabled"], bool
|
||||||
|
):
|
||||||
|
return jsonify({"error": "discord_link_enabled must be a boolean"}), 400
|
||||||
|
if "discord_link_threshold" in body:
|
||||||
|
v = body["discord_link_threshold"]
|
||||||
|
if not isinstance(v, (int, float)) or isinstance(v, bool) or v < 0 or v > 1:
|
||||||
|
return jsonify(
|
||||||
|
{"error": "discord_link_threshold must be a number in [0, 1]"}
|
||||||
|
), 400
|
||||||
|
if "discord_link_window_hours" in body:
|
||||||
|
v = body["discord_link_window_hours"]
|
||||||
|
if not isinstance(v, (int, float)) or isinstance(v, bool) or v <= 0:
|
||||||
|
return jsonify(
|
||||||
|
{"error": "discord_link_window_hours must be a positive number"}
|
||||||
|
), 400
|
||||||
if "wip_title_tagging_enabled" in body and not isinstance(
|
if "wip_title_tagging_enabled" in body and not isinstance(
|
||||||
body["wip_title_tagging_enabled"], bool
|
body["wip_title_tagging_enabled"], bool
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
"""Is every part of FabledCurator running? One verdict, one endpoint.
|
||||||
|
|
||||||
|
Milestone 365. The nav indicator and the System page both read this and
|
||||||
|
nothing else — composing a verdict is this module's job, not the UI's.
|
||||||
|
|
||||||
|
## Two kinds of part, answered two different ways
|
||||||
|
|
||||||
|
**Learned** — celery roles and the GPU agent, from `service_seen`. The
|
||||||
|
question is "how long since it checked in", and these are the parts that can
|
||||||
|
be ABSENT, which is the whole point: `celery inspect` alone reports presence,
|
||||||
|
so a dead worker is a shorter list rather than a red light.
|
||||||
|
|
||||||
|
**Probed live** — Postgres and Redis. Always expected, never learned, and a
|
||||||
|
last-seen for them would be actively misleading: that Redis answered thirty
|
||||||
|
seconds ago says nothing about now.
|
||||||
|
|
||||||
|
## This endpoint must never fail because something it checks has failed
|
||||||
|
|
||||||
|
The inversion is easy to write by accident and it destroys the feature exactly
|
||||||
|
when it is needed — a 500 when Redis is down, instead of `redis: down`. Every
|
||||||
|
probe is wrapped, every wait has a deadline (rule 156), and the roster refresh
|
||||||
|
swallows its own errors. The worst case is a part reported `unknown`, which is
|
||||||
|
a true statement.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from quart import Blueprint, jsonify
|
||||||
|
from sqlalchemy import select, text
|
||||||
|
|
||||||
|
from ..config import get_config
|
||||||
|
from ..extensions import get_session
|
||||||
|
from ..models import ServiceSeen
|
||||||
|
from ..services.service_roster import refresh_if_stale
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
system_health_bp = Blueprint("system_health", __name__, url_prefix="/api/system")
|
||||||
|
|
||||||
|
# How long a learned part may go quiet before it is doubted, then disbelieved.
|
||||||
|
#
|
||||||
|
# These are deliberately generous, and the reason is a deploy rather than a
|
||||||
|
# worker: `docker compose up -d` rolls start-first, so a role is briefly served
|
||||||
|
# by two containers and then by neither while the old one drains. Thresholds
|
||||||
|
# tight enough to catch a crash in seconds would paint the page red every time
|
||||||
|
# the stack is updated, and an alarm that cries wolf on every deploy is one
|
||||||
|
# nobody reads. Tune down only after watching a real deploy pass through.
|
||||||
|
STALE_AFTER_SECONDS = 90
|
||||||
|
DOWN_AFTER_SECONDS = 300
|
||||||
|
|
||||||
|
# Probes cross a process boundary, so they carry deadlines. A hung Postgres
|
||||||
|
# must make this endpoint say "postgres: down", not hang alongside it.
|
||||||
|
PROBE_TIMEOUT_SECONDS = 2.0
|
||||||
|
|
||||||
|
_OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
|
||||||
|
|
||||||
|
# Worst-first, so an overall verdict is just the max.
|
||||||
|
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3}
|
||||||
|
|
||||||
|
|
||||||
|
def _age_state(age_seconds: float) -> str:
|
||||||
|
if age_seconds >= DOWN_AFTER_SECONDS:
|
||||||
|
return _DOWN
|
||||||
|
if age_seconds >= STALE_AFTER_SECONDS:
|
||||||
|
return _STALE
|
||||||
|
return _OK
|
||||||
|
|
||||||
|
|
||||||
|
def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
|
||||||
|
"""Say what the state MEANS. A red chip tells an operator less than a
|
||||||
|
sentence does at the moment they are deciding whether to go and look."""
|
||||||
|
if state == _OK:
|
||||||
|
replicas = details.get("replicas")
|
||||||
|
if replicas and replicas > 1:
|
||||||
|
return f"{name} is running ({replicas} replicas)"
|
||||||
|
return f"{name} is running"
|
||||||
|
mins = int(age // 60)
|
||||||
|
ago = f"{mins} min" if mins else f"{int(age)}s"
|
||||||
|
if state == _STALE:
|
||||||
|
return f"{name} has not checked in for {ago}"
|
||||||
|
return f"{name} has not checked in for {ago} — treat it as stopped"
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_postgres(session) -> dict:
|
||||||
|
started = time.monotonic()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(
|
||||||
|
session.execute(text("SELECT 1")), timeout=PROBE_TIMEOUT_SECONDS
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001 — a probe reports, it never raises
|
||||||
|
return {
|
||||||
|
"key": "postgres", "kind": "datastore", "name": "PostgreSQL",
|
||||||
|
"state": _DOWN, "detail": f"not answering: {type(exc).__name__}",
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"key": "postgres", "kind": "datastore", "name": "PostgreSQL", "state": _OK,
|
||||||
|
"detail": "answering", "latency_ms": round((time.monotonic() - started) * 1000, 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _ping_redis_sync() -> None:
|
||||||
|
import redis # local import; mirrors system_activity's pattern
|
||||||
|
|
||||||
|
client = redis.Redis.from_url(
|
||||||
|
get_config().celery_broker_url,
|
||||||
|
socket_connect_timeout=PROBE_TIMEOUT_SECONDS,
|
||||||
|
socket_timeout=PROBE_TIMEOUT_SECONDS,
|
||||||
|
)
|
||||||
|
client.ping()
|
||||||
|
|
||||||
|
|
||||||
|
async def _probe_redis() -> dict:
|
||||||
|
started = time.monotonic()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(
|
||||||
|
asyncio.to_thread(_ping_redis_sync), timeout=PROBE_TIMEOUT_SECONDS * 2
|
||||||
|
)
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
return {
|
||||||
|
"key": "redis", "kind": "datastore", "name": "Redis",
|
||||||
|
"state": _DOWN,
|
||||||
|
"detail": f"not answering: {type(exc).__name__} — queues and workers "
|
||||||
|
f"cannot be reached either",
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"key": "redis", "kind": "datastore", "name": "Redis", "state": _OK,
|
||||||
|
"detail": "answering", "latency_ms": round((time.monotonic() - started) * 1000, 1),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@system_health_bp.route("/health", methods=["GET"])
|
||||||
|
async def system_health():
|
||||||
|
"""Every part, its state, and one overall verdict.
|
||||||
|
|
||||||
|
Response: {overall, parts: [{key, kind, name, state, detail, last_seen_at,
|
||||||
|
…}], checked_at}
|
||||||
|
"""
|
||||||
|
parts: list[dict] = []
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
# Postgres first, and if it is unreachable nothing else can be read —
|
||||||
|
# say so rather than failing, because "the database is down" is the
|
||||||
|
# single most useful thing this endpoint can ever report.
|
||||||
|
pg = await _probe_postgres(session)
|
||||||
|
parts.append(pg)
|
||||||
|
|
||||||
|
if pg["state"] == _OK:
|
||||||
|
# Rate-limited inside; see service_roster on why the web process
|
||||||
|
# is the right observer.
|
||||||
|
try:
|
||||||
|
await refresh_if_stale(session)
|
||||||
|
await session.commit()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
log.warning("system health: roster refresh failed", exc_info=True)
|
||||||
|
|
||||||
|
rows = (
|
||||||
|
await session.execute(select(ServiceSeen).order_by(ServiceSeen.display_name))
|
||||||
|
).scalars().all()
|
||||||
|
for row in rows:
|
||||||
|
age = (now - row.last_seen_at).total_seconds()
|
||||||
|
state = _age_state(age)
|
||||||
|
parts.append({
|
||||||
|
"key": row.key,
|
||||||
|
"kind": row.kind,
|
||||||
|
"name": row.display_name,
|
||||||
|
"state": state,
|
||||||
|
"detail": _describe_learned(row.display_name, state, age, row.details or {}),
|
||||||
|
"last_seen_at": row.last_seen_at.isoformat(),
|
||||||
|
"first_seen_at": row.first_seen_at.isoformat(),
|
||||||
|
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
||||||
|
})
|
||||||
|
|
||||||
|
parts.append(await _probe_redis())
|
||||||
|
|
||||||
|
overall = max((p["state"] for p in parts), key=lambda s: _SEVERITY[s], default=_UNKNOWN)
|
||||||
|
return jsonify({
|
||||||
|
"overall": overall,
|
||||||
|
"parts": sorted(parts, key=lambda p: (-_SEVERITY[p["state"]], p["name"])),
|
||||||
|
"checked_at": now.isoformat(),
|
||||||
|
# So the UI can explain a `stale` without hard-coding the same numbers
|
||||||
|
# in a second place.
|
||||||
|
"thresholds": {
|
||||||
|
"stale_after_seconds": STALE_AFTER_SECONDS,
|
||||||
|
"down_after_seconds": DOWN_AFTER_SECONDS,
|
||||||
|
},
|
||||||
|
})
|
||||||
@@ -200,6 +200,20 @@ def make_celery() -> Celery:
|
|||||||
"task": "backend.app.tasks.maintenance.snapshot_head_metrics",
|
"task": "backend.app.tasks.maintenance.snapshot_head_metrics",
|
||||||
"schedule": 86400.0,
|
"schedule": 86400.0,
|
||||||
},
|
},
|
||||||
|
"group-discord-drops-hourly": {
|
||||||
|
"task": "backend.app.tasks.maintenance.group_discord_drops",
|
||||||
|
"schedule": 3600.0, # hourly. Not daily: the grouping signal is
|
||||||
|
# the SigLIP embedding, which lands asynchronously AFTER import
|
||||||
|
# (#388 E2), so this sweep is what picks up a drop once its
|
||||||
|
# vectors have caught up. No-op unless discord_grouping_enabled.
|
||||||
|
},
|
||||||
|
"match-post-associations-hourly": {
|
||||||
|
"task": "backend.app.tasks.maintenance.match_post_associations",
|
||||||
|
"schedule": 3600.0, # hourly, and AFTER the grouper's own cadence
|
||||||
|
# by construction: a pair cannot be proposed until the drop it
|
||||||
|
# points at exists as a grouping (#388 E5). No-op unless
|
||||||
|
# discord_link_enabled.
|
||||||
|
},
|
||||||
"integrity-verify-weekly": {
|
"integrity-verify-weekly": {
|
||||||
"task": "backend.app.tasks.maintenance.verify_integrity",
|
"task": "backend.app.tasks.maintenance.verify_integrity",
|
||||||
"schedule": 604800.0, # weekly
|
"schedule": 604800.0, # weekly
|
||||||
|
|||||||
@@ -16,8 +16,11 @@ class Config:
|
|||||||
celery_broker_url: str
|
celery_broker_url: str
|
||||||
celery_result_backend: str
|
celery_result_backend: str
|
||||||
|
|
||||||
|
# Sets Quart's app.secret_key. Nothing signs a cookie today (FC has no
|
||||||
|
# login and no session use), so this currently protects nothing — it is
|
||||||
|
# required rather than defaulted so that the day something session-backed
|
||||||
|
# does land, no instance is already running on a value we published.
|
||||||
secret_key: str
|
secret_key: str
|
||||||
extension_api_key: str # used by the Firefox extension; lands in FC-3 but read here
|
|
||||||
log_level: str
|
log_level: str
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -47,6 +50,5 @@ def get_config() -> Config:
|
|||||||
celery_broker_url=os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"),
|
celery_broker_url=os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"),
|
||||||
celery_result_backend=os.environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"),
|
celery_result_backend=os.environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"),
|
||||||
secret_key=os.environ["SECRET_KEY"],
|
secret_key=os.environ["SECRET_KEY"],
|
||||||
extension_api_key=os.environ.get("EXTENSION_API_KEY", ""),
|
|
||||||
log_level=os.environ.get("LOG_LEVEL", "INFO"),
|
log_level=os.environ.get("LOG_LEVEL", "INFO"),
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -26,12 +26,15 @@ from .patreon_failed_media import PatreonFailedMedia
|
|||||||
from .patreon_seen_media import PatreonSeenMedia
|
from .patreon_seen_media import PatreonSeenMedia
|
||||||
from .pixiv_failed_media import PixivFailedMedia
|
from .pixiv_failed_media import PixivFailedMedia
|
||||||
from .pixiv_seen_media import PixivSeenMedia
|
from .pixiv_seen_media import PixivSeenMedia
|
||||||
|
from .platform_membership import PlatformMembership
|
||||||
from .post import Post
|
from .post import Post
|
||||||
|
from .post_association import PostAssociation
|
||||||
from .post_attachment import PostAttachment, attachment_download_url
|
from .post_attachment import PostAttachment, attachment_download_url
|
||||||
from .presentation_review import PresentationReview
|
from .presentation_review import PresentationReview
|
||||||
from .series_chapter import SeriesChapter
|
from .series_chapter import SeriesChapter
|
||||||
from .series_page import SeriesPage
|
from .series_page import SeriesPage
|
||||||
from .series_suggestion import SeriesSuggestion
|
from .series_suggestion import SeriesSuggestion
|
||||||
|
from .service_seen import ServiceSeen
|
||||||
from .source import Source
|
from .source import Source
|
||||||
from .subscribestar_failed_media import SubscribeStarFailedMedia
|
from .subscribestar_failed_media import SubscribeStarFailedMedia
|
||||||
from .subscribestar_seen_media import SubscribeStarSeenMedia
|
from .subscribestar_seen_media import SubscribeStarSeenMedia
|
||||||
@@ -57,12 +60,15 @@ __all__ = [
|
|||||||
"SubscribeStarFailedMedia",
|
"SubscribeStarFailedMedia",
|
||||||
"SubscribeStarSeenMedia",
|
"SubscribeStarSeenMedia",
|
||||||
"Post",
|
"Post",
|
||||||
|
"PostAssociation",
|
||||||
"PostAttachment",
|
"PostAttachment",
|
||||||
"attachment_download_url",
|
"attachment_download_url",
|
||||||
"PresentationReview",
|
"PresentationReview",
|
||||||
"SeriesChapter",
|
"SeriesChapter",
|
||||||
"SeriesPage",
|
"SeriesPage",
|
||||||
"SeriesSuggestion",
|
"SeriesSuggestion",
|
||||||
|
"PlatformMembership",
|
||||||
|
"ServiceSeen",
|
||||||
"ImageRecord",
|
"ImageRecord",
|
||||||
"ImageProvenance",
|
"ImageProvenance",
|
||||||
"ImageRegion",
|
"ImageRegion",
|
||||||
|
|||||||
@@ -27,10 +27,10 @@ class Artist(Base):
|
|||||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
||||||
# True once a Source is attached; flips false if all sources removed.
|
# True once a Source is attached; flips false if all sources removed.
|
||||||
is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||||
|
|
||||||
# Per-artist scheduling overrides; null means "use global default".
|
# Per-artist scheduling overrides; null means "use global default".
|
||||||
auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||||
check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ feedback_check_existing_enums):
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Integer, String, Text
|
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, Integer, String, Text, text
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -29,10 +29,21 @@ from .base import Base
|
|||||||
class BackupRun(Base):
|
class BackupRun(Base):
|
||||||
__tablename__ = "backup_run"
|
__tablename__ = "backup_run"
|
||||||
|
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# alembic 0017: reporting indexes, never declared on the model (#3275).
|
||||||
|
Index("ix_backup_run_kind_started", "kind", text("started_at DESC")),
|
||||||
|
Index("ix_backup_run_status_finished", "status", text("finished_at DESC")),
|
||||||
|
Index("ix_backup_run_tag_partial", "tag", postgresql_where=text("tag IS NOT NULL")),
|
||||||
|
)
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
|
# No index=True: ix_backup_run_kind_started (above) already leads with
|
||||||
|
# `kind`, so a single-column index on it was pure write cost (#3301).
|
||||||
|
kind: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(
|
||||||
String(16), nullable=False, default="pending", index=True,
|
# No index=True — ix_backup_run_status_finished leads with `status`.
|
||||||
|
String(16), nullable=False, default="pending",
|
||||||
|
server_default="pending",
|
||||||
)
|
)
|
||||||
tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||||
triggered_by: Mapped[str] = mapped_column(String(32), nullable=False)
|
triggered_by: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
@@ -49,7 +60,9 @@ class BackupRun(Base):
|
|||||||
manifest: Mapped[dict] = mapped_column(
|
manifest: Mapped[dict] = mapped_column(
|
||||||
JSON, nullable=False, default=dict, server_default="{}",
|
JSON, nullable=False, default=dict, server_default="{}",
|
||||||
)
|
)
|
||||||
|
# Self-referential FK, unindexed until 0089 (#3300): SET NULL has to find
|
||||||
|
# the rows pointing at a deleted run before it can null them.
|
||||||
restored_from_id: Mapped[int | None] = mapped_column(
|
restored_from_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("backup_run.id", ondelete="SET NULL"),
|
ForeignKey("backup_run.id", ondelete="SET NULL"),
|
||||||
nullable=True,
|
nullable=True, index=True,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -40,8 +40,10 @@ class CharacterPrototype(Base):
|
|||||||
)
|
)
|
||||||
# Provenance: the region this vector was copied from. SET NULL so pruning a
|
# Provenance: the region this vector was copied from. SET NULL so pruning a
|
||||||
# region doesn't delete the prototype mid-cycle (the next refresh reconciles).
|
# region doesn't delete the prototype mid-cycle (the next refresh reconciles).
|
||||||
|
# index=True added in 0089 — the FK was unindexed (#3300).
|
||||||
region_id: Mapped[int | None] = mapped_column(
|
region_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True
|
ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True,
|
||||||
|
index=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -25,8 +25,8 @@ class DownloadEvent(Base):
|
|||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default="0")
|
||||||
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
metadata_: Mapped[dict] = mapped_column(
|
metadata_: Mapped[dict] = mapped_column(
|
||||||
"metadata", JSONB, nullable=False, default=dict,
|
"metadata", JSONB, nullable=False, default=dict,
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ doesn't delete the link record).
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
|
CheckConstraint,
|
||||||
DateTime,
|
DateTime,
|
||||||
Float,
|
Float,
|
||||||
ForeignKey,
|
ForeignKey,
|
||||||
@@ -38,15 +39,33 @@ STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead")
|
|||||||
class ExternalLink(Base):
|
class ExternalLink(Base):
|
||||||
__tablename__ = "external_link"
|
__tablename__ = "external_link"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
# alembic 0028 enum CHECKs. Rule 36 territory: a new host or status value
|
||||||
|
# needs its constraint swapped in the same migration (#3275).
|
||||||
|
CheckConstraint(
|
||||||
|
"host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')",
|
||||||
|
# Bare name: Base.metadata's naming convention prepends
|
||||||
|
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
||||||
|
# alembic 0088, which renames the four constraints that shipped
|
||||||
|
# that way (#3275).
|
||||||
|
name="host",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')",
|
||||||
|
name="status",
|
||||||
|
),
|
||||||
# One row per (post, url). The full url (incl. #fragment) is the identity
|
# One row per (post, url). The full url (incl. #fragment) is the identity
|
||||||
# — the same file linked twice in a post collapses to one row.
|
# — the same file linked twice in a post collapses to one row.
|
||||||
Index("uq_external_link_post_url", "post_id", "url", unique=True),
|
Index("uq_external_link_post_url", "post_id", "url", unique=True),
|
||||||
Index("ix_external_link_status", "status"),
|
Index("ix_external_link_status", "status"),
|
||||||
|
# Unindexed FK (#3300).
|
||||||
|
Index("ix_external_link_attachment_id", "attachment_id"),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
# No index=True: uq_external_link_post_url (post_id, url) already leads
|
||||||
|
# with post_id (#3301).
|
||||||
post_id: Mapped[int] = mapped_column(
|
post_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False
|
||||||
)
|
)
|
||||||
artist_id: Mapped[int | None] = mapped_column(
|
artist_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
|||||||
@@ -50,7 +50,8 @@ class GpuJob(Base):
|
|||||||
# What to compute, e.g. 'ccip' (detect figures + CCIP-embed) or 'siglip_region'.
|
# What to compute, e.g. 'ccip' (detect figures + CCIP-embed) or 'siglip_region'.
|
||||||
task: Mapped[str] = mapped_column(String(32), nullable=False)
|
task: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(
|
||||||
String(16), nullable=False, default="pending", index=True
|
String(16), nullable=False, default="pending", index=True,
|
||||||
|
server_default="pending",
|
||||||
)
|
)
|
||||||
# pending | leased | done | error
|
# pending | leased | done | error
|
||||||
lease_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
lease_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
@@ -60,7 +61,7 @@ class GpuJob(Base):
|
|||||||
lease_expires_at: Mapped[datetime | None] = mapped_column(
|
lease_expires_at: Mapped[datetime | None] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=True
|
DateTime(timezone=True), nullable=True
|
||||||
)
|
)
|
||||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
# Triage verdict for an ERRORED job (#125): NULL = not yet probed;
|
# Triage verdict for an ERRORED job (#125): NULL = not yet probed;
|
||||||
# 'defect' = the integrity probe says the FILE itself is bad (surfaced for
|
# 'defect' = the integrity probe says the FILE itself is bad (surfaced for
|
||||||
|
|||||||
@@ -24,10 +24,11 @@ class HeadAutoApplyRun(Base):
|
|||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
# dry_run=True is a PREVIEW: scores + counts what WOULD apply, writes nothing
|
# dry_run=True is a PREVIEW: scores + counts what WOULD apply, writes nothing
|
||||||
# (preview/apply parity, rule 93).
|
# (preview/apply parity, rule 93).
|
||||||
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(
|
||||||
String(16), nullable=False, default="running", index=True
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
server_default="running",
|
||||||
)
|
)
|
||||||
# running | ready | error
|
# running | ready | error
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
|||||||
@@ -24,9 +24,9 @@ class HeadMetric(Base):
|
|||||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
||||||
)
|
)
|
||||||
# An auto-applied (source='head_auto') tag the operator later REMOVED.
|
# An auto-applied (source='head_auto') tag the operator later REMOVED.
|
||||||
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
# A tag with a head that the operator added by HAND (the head missed it).
|
# A tag with a head that the operator added by HAND (the head missed it).
|
||||||
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
updated_at: Mapped[datetime] = mapped_column(
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -19,8 +19,14 @@ class HeadMetricsSnapshot(Base):
|
|||||||
__tablename__ = "head_metrics_snapshot"
|
__tablename__ = "head_metrics_snapshot"
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
tag_id: Mapped[int] = mapped_column(
|
# Nullable, matching alembic 0060, which declared this column without
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), index=True
|
# `nullable=False`. The model had it as `Mapped[int]` — NOT NULL — which
|
||||||
|
# was simply never true of the database (#3275). Left nullable rather than
|
||||||
|
# tightened: a snapshot of a tag that is later hard-deleted is a row worth
|
||||||
|
# keeping, and the FK is ON DELETE CASCADE, so tightening it would only
|
||||||
|
# change behaviour, not correct a bug.
|
||||||
|
tag_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("tag.id", ondelete="CASCADE"), nullable=True, index=True
|
||||||
)
|
)
|
||||||
# Denormalized so a snapshot stays readable even if the tag is later renamed.
|
# Denormalized so a snapshot stays readable even if the tag is later renamed.
|
||||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
@@ -28,9 +34,9 @@ class HeadMetricsSnapshot(Base):
|
|||||||
DateTime(timezone=True), nullable=False, server_default=func.now(), index=True
|
DateTime(timezone=True), nullable=False, server_default=func.now(), index=True
|
||||||
)
|
)
|
||||||
# Current count of source='head_auto' applications still standing.
|
# Current count of source='head_auto' applications still standing.
|
||||||
n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
# The head's measured quality at snapshot time (null if no head exists).
|
# The head's measured quality at snapshot time (null if no head exists).
|
||||||
ap: Mapped[float | None] = mapped_column(Float, nullable=True)
|
ap: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
precision_cv: Mapped[float | None] = mapped_column(Float, nullable=True)
|
precision_cv: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ class HeadTrainingRun(Base):
|
|||||||
# Training parameters: {min_positives, neg_ratio, precision_target, ...}.
|
# Training parameters: {min_positives, neg_ratio, precision_target, ...}.
|
||||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(
|
||||||
String(16), nullable=False, default="running", index=True
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
server_default="running",
|
||||||
)
|
)
|
||||||
# running | ready | error
|
# running | ready | error
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
|
|||||||
@@ -47,8 +47,15 @@ class ImageProvenance(Base):
|
|||||||
# attachment on the post. NULL for loose downloads and pre-backfill rows.
|
# attachment on the post. NULL for loose downloads and pre-backfill rows.
|
||||||
# SET NULL so deleting the archive attachment never destroys the (image,
|
# SET NULL so deleting the archive attachment never destroys the (image,
|
||||||
# post) edge — it just forgets which archive it came from.
|
# post) edge — it just forgets which archive it came from.
|
||||||
|
# FK named explicitly: the convention renders this
|
||||||
|
# `fk_image_provenance_from_attachment_id_post_attachment`, but alembic
|
||||||
|
# 0055 created it as `fk_image_provenance_from_attachment` (#3275).
|
||||||
from_attachment_id: Mapped[int | None] = mapped_column(
|
from_attachment_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("post_attachment.id", ondelete="SET NULL"),
|
ForeignKey(
|
||||||
|
"post_attachment.id",
|
||||||
|
ondelete="SET NULL",
|
||||||
|
name="fk_image_provenance_from_attachment",
|
||||||
|
),
|
||||||
nullable=True, index=True,
|
nullable=True, index=True,
|
||||||
)
|
)
|
||||||
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
|||||||
@@ -14,10 +14,13 @@ from sqlalchemy import (
|
|||||||
Enum,
|
Enum,
|
||||||
Float,
|
Float,
|
||||||
ForeignKey,
|
ForeignKey,
|
||||||
|
Index,
|
||||||
Integer,
|
Integer,
|
||||||
String,
|
String,
|
||||||
Text,
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
func,
|
func,
|
||||||
|
text,
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -29,11 +32,38 @@ ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded")
|
|||||||
class ImageRecord(Base):
|
class ImageRecord(Base):
|
||||||
__tablename__ = "image_record"
|
__tablename__ = "image_record"
|
||||||
|
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# alembic 0001. The database enforces sha256 uniqueness with a
|
||||||
|
# CONSTRAINT and carries a SEPARATE non-unique btree index; the model
|
||||||
|
# said `unique=True, index=True`, which collapses both into a single
|
||||||
|
# UNIQUE index under a different name. Same guarantee either way, but
|
||||||
|
# not the same objects, so autogenerate saw a drop and an add (#3275).
|
||||||
|
UniqueConstraint("sha256", name="uq_image_record_sha256"),
|
||||||
|
# alembic 0036, and the last thing in this schema that lived only in a
|
||||||
|
# migration. SQLAlchemy CAN express an hnsw index with an operator
|
||||||
|
# class, so there is no reason for it to be invisible to the models —
|
||||||
|
# and its absence was the quietest failure of the lot: everything
|
||||||
|
# works, similarity search just silently stops using an index.
|
||||||
|
Index(
|
||||||
|
"ix_image_record_siglip_hnsw",
|
||||||
|
"siglip_embedding",
|
||||||
|
postgresql_using="hnsw",
|
||||||
|
postgresql_ops={"siglip_embedding": "vector_cosine_ops"},
|
||||||
|
),
|
||||||
|
# alembic 0035/0071: the date-ordered browse indexes (#3275).
|
||||||
|
Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")),
|
||||||
|
Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")),
|
||||||
|
)
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
|
||||||
# On-disk identity
|
# On-disk identity
|
||||||
path: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
path: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
|
# Neither unique= nor index=: uq_image_record_sha256 in __table_args__
|
||||||
|
# above creates its own index, and the separate ix_image_record_sha256
|
||||||
|
# that 0001 also built was an exact duplicate of it — dropped in 0089
|
||||||
|
# (#3301). Lookups by sha256 use the constraint's index.
|
||||||
|
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||||
mime: Mapped[str] = mapped_column(String(64), nullable=False)
|
mime: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
@@ -47,7 +77,8 @@ class ImageRecord(Base):
|
|||||||
# Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'.
|
# Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'.
|
||||||
# Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'.
|
# Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'.
|
||||||
integrity_status: Mapped[str] = mapped_column(
|
integrity_status: Mapped[str] = mapped_column(
|
||||||
String(24), nullable=False, default="unknown", index=True
|
String(24), nullable=False, default="unknown", index=True,
|
||||||
|
server_default="unknown",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Thumbnail (populated by FC-2)
|
# Thumbnail (populated by FC-2)
|
||||||
@@ -72,8 +103,15 @@ class ImageRecord(Base):
|
|||||||
)
|
)
|
||||||
# FC-2d-vii-c: canonical per-image artist (the single source of truth
|
# FC-2d-vii-c: canonical per-image artist (the single source of truth
|
||||||
# for attribution; provenance posts remain lineage detail).
|
# for attribution; provenance posts remain lineage detail).
|
||||||
|
# FK named explicitly: the naming convention renders this
|
||||||
|
# `fk_image_record_artist_id_artist`, but alembic 0008 created it as
|
||||||
|
# `fk_image_record_artist_id` (#3275).
|
||||||
artist_id: Mapped[int | None] = mapped_column(
|
artist_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
ForeignKey(
|
||||||
|
"artist.id", ondelete="SET NULL", name="fk_image_record_artist_id"
|
||||||
|
),
|
||||||
|
nullable=True,
|
||||||
|
index=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
# ML fields (populated by the ml-worker / GPU agent). 1152 = SigLIP-so400m
|
# ML fields (populated by the ml-worker / GPU agent). 1152 = SigLIP-so400m
|
||||||
|
|||||||
@@ -21,17 +21,17 @@ class ImportBatch(Base):
|
|||||||
)
|
)
|
||||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
|
||||||
total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
# Deep-scan only: count of already-imported files whose sidecar metadata
|
# Deep-scan only: count of already-imported files whose sidecar metadata
|
||||||
# got re-applied this run (post/source/provenance upsert). Stays 0 on
|
# got re-applied this run (post/source/provenance upsert). Stays 0 on
|
||||||
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
|
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
|
||||||
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
|
|
||||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True, server_default="running")
|
||||||
# running | complete | cancelled
|
# running | complete | cancelled
|
||||||
|
|
||||||
tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan")
|
tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan")
|
||||||
|
|||||||
@@ -4,7 +4,15 @@ Enforced as a single row via a CHECK (id = 1) constraint. The application
|
|||||||
always SELECTs id=1 and never inserts/deletes after the initial migration.
|
always SELECTs id=1 and never inserts/deletes after the initial migration.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text, select
|
from sqlalchemy import (
|
||||||
|
Boolean,
|
||||||
|
CheckConstraint,
|
||||||
|
Float,
|
||||||
|
Integer,
|
||||||
|
Text,
|
||||||
|
select,
|
||||||
|
text,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -14,63 +22,106 @@ class ImportSettings(Base):
|
|||||||
__tablename__ = "import_settings"
|
__tablename__ = "import_settings"
|
||||||
# Bare constraint name — Base.metadata's naming convention applies the
|
# Bare constraint name — Base.metadata's naming convention applies the
|
||||||
# ck_<table>_<name> prefix, producing the final ck_import_settings_singleton.
|
# ck_<table>_<name> prefix, producing the final ck_import_settings_singleton.
|
||||||
|
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
|
||||||
|
# producing ck_import_settings_singleton. The chain shipped the DOUBLED
|
||||||
|
# ck_import_settings_ck_import_settings_singleton, because the migration
|
||||||
|
# pre-prefixed the name and the convention prefixed it again; alembic
|
||||||
|
# 0088 renames it to what this line has always produced (#3275).
|
||||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import")
|
import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import", server_default="/import")
|
||||||
|
|
||||||
min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
|
|
||||||
skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9)
|
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
|
||||||
|
|
||||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95)
|
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
|
||||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
|
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
|
||||||
|
|
||||||
phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10)
|
phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10, server_default="10")
|
||||||
|
|
||||||
# FC-3c downloader knobs
|
# FC-3c downloader knobs
|
||||||
download_rate_limit_seconds: Mapped[float] = mapped_column(
|
download_rate_limit_seconds: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=3.0
|
Float, nullable=False, default=3.0,
|
||||||
|
server_default="3",
|
||||||
)
|
)
|
||||||
download_validate_files: Mapped[bool] = mapped_column(
|
download_validate_files: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
|
|
||||||
# FC-3d scheduling knobs
|
# FC-3d scheduling knobs
|
||||||
download_schedule_default_seconds: Mapped[int] = mapped_column(
|
download_schedule_default_seconds: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=28800
|
Integer, nullable=False, default=28800,
|
||||||
|
server_default="28800",
|
||||||
)
|
)
|
||||||
download_event_retention_days: Mapped[int] = mapped_column(
|
download_event_retention_days: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=90
|
Integer, nullable=False, default=90,
|
||||||
|
server_default="90",
|
||||||
)
|
)
|
||||||
download_failure_warning_threshold: Mapped[int] = mapped_column(
|
download_failure_warning_threshold: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=5
|
Integer, nullable=False, default=5,
|
||||||
|
server_default="5",
|
||||||
)
|
)
|
||||||
|
|
||||||
# FC-3h backup knobs.
|
# FC-3h backup knobs.
|
||||||
backup_db_nightly_enabled: Mapped[bool] = mapped_column(
|
backup_db_nightly_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=False,
|
Boolean, nullable=False, default=False,
|
||||||
|
server_default="false",
|
||||||
)
|
)
|
||||||
backup_db_nightly_hour_utc: Mapped[int] = mapped_column(
|
backup_db_nightly_hour_utc: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=3,
|
Integer, nullable=False, default=3,
|
||||||
|
server_default="3",
|
||||||
)
|
)
|
||||||
backup_db_keep_last_n: Mapped[int] = mapped_column(
|
backup_db_keep_last_n: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=14,
|
Integer, nullable=False, default=14,
|
||||||
|
server_default="14",
|
||||||
)
|
)
|
||||||
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=3,
|
Integer, nullable=False, default=3,
|
||||||
|
server_default="3",
|
||||||
)
|
)
|
||||||
|
|
||||||
# FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
|
# FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
|
||||||
# the weighted-score cut-off (0..1) above which a pending suggestion is made.
|
# the weighted-score cut-off (0..1) above which a pending suggestion is made.
|
||||||
series_suggest_enabled: Mapped[bool] = mapped_column(
|
series_suggest_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True,
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
series_suggest_threshold: Mapped[float] = mapped_column(
|
series_suggest_threshold: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.5,
|
Float, nullable=False, default=0.5,
|
||||||
|
server_default="0.5",
|
||||||
|
)
|
||||||
|
|
||||||
|
# Milestone 388 E5 — the announcement matcher: "this Patreon post announced
|
||||||
|
# that Discord drop". Lives here rather than in MLSettings, with the series
|
||||||
|
# matcher it is modelled on, because it runs no inference: the signals are
|
||||||
|
# time proximity and whether the post says so.
|
||||||
|
discord_link_enabled: Mapped[bool] = mapped_column(
|
||||||
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
|
)
|
||||||
|
# The weighted-score cut-off. 0.60 is not arbitrary: it is deliberately set
|
||||||
|
# ABOVE the largest single signal weight, which is what makes "time
|
||||||
|
# proximity alone must never be sufficient" an ARITHMETIC property rather
|
||||||
|
# than a hope. On a busy day an artist posts several times; if proximity
|
||||||
|
# could carry a pair by itself, every one of those days would produce false
|
||||||
|
# pairs and the review queue would be abandoned. See
|
||||||
|
# post_association_service.WEIGHTS — a guard test pins the relationship.
|
||||||
|
discord_link_threshold: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=0.60,
|
||||||
|
server_default="0.60",
|
||||||
|
)
|
||||||
|
# How far apart the announcement and the drop may be. The Patreon post
|
||||||
|
# exists IN ORDER TO announce the drop, so they are minutes-to-hours apart;
|
||||||
|
# a day is generous and still excludes "same week".
|
||||||
|
discord_link_window_hours: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=24.0,
|
||||||
|
server_default="24",
|
||||||
)
|
)
|
||||||
|
|
||||||
# #830 off-platform file-host downloads — per-host enable lever (default on,
|
# #830 off-platform file-host downloads — per-host enable lever (default on,
|
||||||
@@ -113,7 +164,9 @@ class ImportSettings(Base):
|
|||||||
# English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
|
# English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
|
||||||
# trusted regardless (script-detected). Per-post overrides handle the misses.
|
# trusted regardless (script-detected). Per-post overrides handle the misses.
|
||||||
translation_min_confidence: Mapped[float] = mapped_column(
|
translation_min_confidence: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.9, server_default="0.9",
|
# text() because alembic 0084 used sa.text(); see ml_settings for why
|
||||||
|
# the form matters and why it is per-column (#3275).
|
||||||
|
Float, nullable=False, default=0.9, server_default=text("0.9"),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Title-based WIP auto-tagging (task #1458). When a freshly-imported post's
|
# Title-based WIP auto-tagging (task #1458). When a freshly-imported post's
|
||||||
|
|||||||
@@ -13,10 +13,12 @@ from sqlalchemy import (
|
|||||||
Boolean,
|
Boolean,
|
||||||
DateTime,
|
DateTime,
|
||||||
ForeignKey,
|
ForeignKey,
|
||||||
|
Index,
|
||||||
Integer,
|
Integer,
|
||||||
String,
|
String,
|
||||||
Text,
|
Text,
|
||||||
func,
|
func,
|
||||||
|
text,
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
@@ -26,6 +28,12 @@ from .base import Base
|
|||||||
class ImportTask(Base):
|
class ImportTask(Base):
|
||||||
__tablename__ = "import_task"
|
__tablename__ = "import_task"
|
||||||
|
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_import_task_created_at_desc", text("created_at DESC")),
|
||||||
|
# Unindexed FK (#3300).
|
||||||
|
Index("ix_import_task_result_image_id", "result_image_id"),
|
||||||
|
)
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
batch_id: Mapped[int] = mapped_column(
|
batch_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
@@ -33,14 +41,14 @@ class ImportTask(Base):
|
|||||||
|
|
||||||
source_path: Mapped[str] = mapped_column(Text, nullable=False)
|
source_path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
||||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True, server_default="pending")
|
||||||
|
|
||||||
# Poison-pill circuit breaker (alembic 0026). recovery_count tracks
|
# Poison-pill circuit breaker (alembic 0026). recovery_count tracks
|
||||||
# how many times the stuck-task sweep has re-queued this row; after
|
# how many times the stuck-task sweep has re-queued this row; after
|
||||||
# the cap it's failed with a diagnostic instead of looping. refetched
|
# the cap it's failed with a diagnostic instead of looping. refetched
|
||||||
# bounds the one-shot re-download remediation to a single attempt.
|
# bounds the one-shot re-download remediation to a single attempt.
|
||||||
recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||||
|
|
||||||
result_image_id: Mapped[int | None] = mapped_column(
|
result_image_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ reads it and routes through cleanup_service.delete_images.
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from sqlalchemy import DateTime, Integer, String, Text, func
|
from sqlalchemy import DateTime, Integer, String, Text, func, text
|
||||||
from sqlalchemy.dialects.postgresql import JSONB
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -23,6 +23,7 @@ class LibraryAuditRun(Base):
|
|||||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(
|
||||||
String(16), nullable=False, default="running", index=True,
|
String(16), nullable=False, default="running", index=True,
|
||||||
|
server_default="running",
|
||||||
)
|
)
|
||||||
# running | ready | applied | cancelled | error
|
# running | ready | applied | cancelled | error
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
@@ -31,14 +32,16 @@ class LibraryAuditRun(Base):
|
|||||||
finished_at: Mapped[datetime | None] = mapped_column(
|
finished_at: Mapped[datetime | None] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=True,
|
DateTime(timezone=True), nullable=True,
|
||||||
)
|
)
|
||||||
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
matched_ids: Mapped[list[int]] = mapped_column(
|
||||||
|
JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb")
|
||||||
|
)
|
||||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
# Chunked-scan state (alembic 0039): keyset cursor the next chunk resumes
|
# Chunked-scan state (alembic 0039): keyset cursor the next chunk resumes
|
||||||
# from, and the last time a chunk made progress (so the recovery sweep can
|
# from, and the last time a chunk made progress (so the recovery sweep can
|
||||||
# tell a progressing multi-chunk audit from a stuck one).
|
# tell a progressing multi-chunk audit from a stuck one).
|
||||||
resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
last_progress_at: Mapped[datetime | None] = mapped_column(
|
last_progress_at: Mapped[datetime | None] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=True,
|
DateTime(timezone=True), nullable=True,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ from sqlalchemy import (
|
|||||||
String,
|
String,
|
||||||
func,
|
func,
|
||||||
select,
|
select,
|
||||||
|
text,
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -20,7 +21,10 @@ from .base import Base
|
|||||||
class MLSettings(Base):
|
class MLSettings(Base):
|
||||||
__tablename__ = "ml_settings"
|
__tablename__ = "ml_settings"
|
||||||
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
|
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
|
||||||
# producing the final ck_ml_settings_singleton (matches migration 0003).
|
# producing ck_ml_settings_singleton. The chain shipped the DOUBLED
|
||||||
|
# ck_ml_settings_ck_ml_settings_singleton, because the migration
|
||||||
|
# pre-prefixed the name and the convention prefixed it again; alembic
|
||||||
|
# 0088 renames it to what this line has always produced (#3275).
|
||||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
@@ -31,17 +35,20 @@ class MLSettings(Base):
|
|||||||
# queueing embed work nothing will consume (the daily GPU 'embed' backfill
|
# queueing embed work nothing will consume (the daily GPU 'embed' backfill
|
||||||
# covers those images instead).
|
# covers those images instead).
|
||||||
cpu_embed_enabled: Mapped[bool] = mapped_column(
|
cpu_embed_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
# Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not
|
# Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not
|
||||||
# a fixed count) so coverage reflects real screen time regardless of length;
|
# a fixed count) so coverage reflects real screen time regardless of length;
|
||||||
# cap the total so a long video can't explode into hundreds of embeds. The
|
# cap the total so a long video can't explode into hundreds of embeds. The
|
||||||
# per-frame SigLIP embeddings are mean-pooled. Operator-tunable.
|
# per-frame SigLIP embeddings are mean-pooled. Operator-tunable.
|
||||||
video_frame_interval_seconds: Mapped[float] = mapped_column(
|
video_frame_interval_seconds: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=4.0
|
Float, nullable=False, default=4.0,
|
||||||
|
server_default="4",
|
||||||
)
|
)
|
||||||
video_max_frames: Mapped[int] = mapped_column(
|
video_max_frames: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=64
|
Integer, nullable=False, default=64,
|
||||||
|
server_default="64",
|
||||||
)
|
)
|
||||||
# Tagging-v2 head training (#114). The head is the suggestion source that
|
# Tagging-v2 head training (#114). The head is the suggestion source that
|
||||||
# LEARNS from the operator's tags (replacing Camie + centroid). A concept
|
# LEARNS from the operator's tags (replacing Camie + centroid). A concept
|
||||||
@@ -49,10 +56,12 @@ class MLSettings(Base):
|
|||||||
# head_auto_apply_precision is the precision bar a head must clear (at some
|
# head_auto_apply_precision is the precision bar a head must clear (at some
|
||||||
# operating point) to "graduate" into earned auto-apply. Operator-tunable.
|
# operating point) to "graduate" into earned auto-apply. Operator-tunable.
|
||||||
head_min_positives: Mapped[int] = mapped_column(
|
head_min_positives: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=8
|
Integer, nullable=False, default=8,
|
||||||
|
server_default="8",
|
||||||
)
|
)
|
||||||
head_auto_apply_precision: Mapped[float] = mapped_column(
|
head_auto_apply_precision: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.97
|
Float, nullable=False, default=0.97,
|
||||||
|
server_default="0.97",
|
||||||
)
|
)
|
||||||
# Earned auto-apply (#114). A graduated head fires (tags images without a
|
# Earned auto-apply (#114). A graduated head fires (tags images without a
|
||||||
# human) when this master switch is on AND the head has at least
|
# human) when this master switch is on AND the head has at least
|
||||||
@@ -61,29 +70,34 @@ class MLSettings(Base):
|
|||||||
# default (operator-asked 2026-06-29: opt-OUT, not opt-in); the support +
|
# default (operator-asked 2026-06-29: opt-OUT, not opt-in); the support +
|
||||||
# measured-precision gates keep it safe, and every auto-tag is reversible.
|
# measured-precision gates keep it safe, and every auto-tag is reversible.
|
||||||
head_auto_apply_enabled: Mapped[bool] = mapped_column(
|
head_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
||||||
# Support floor raised 30→50 (operator-asked 2026-07-06): a head needs
|
# Support floor raised 30→50 (operator-asked 2026-07-06): a head needs
|
||||||
# more human labels before it may fire without a human.
|
# more human labels before it may fire without a human.
|
||||||
Integer, nullable=False, default=50
|
Integer, nullable=False, default=50,
|
||||||
|
server_default="30",
|
||||||
)
|
)
|
||||||
# CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
|
# CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
|
||||||
# over-fired (high-reference characters matched a scatter of images); 0.85
|
# over-fired (high-reference characters matched a scatter of images); 0.85
|
||||||
# keeps the confident single-character matches. Tunable from the agent card.
|
# keeps the confident single-character matches. Tunable from the agent card.
|
||||||
ccip_match_threshold: Mapped[float] = mapped_column(
|
ccip_match_threshold: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.85
|
Float, nullable=False, default=0.85,
|
||||||
|
server_default="0.85",
|
||||||
)
|
)
|
||||||
# CCIP auto-apply (#114). Confident matches (>= ccip_auto_apply_threshold,
|
# CCIP auto-apply (#114). Confident matches (>= ccip_auto_apply_threshold,
|
||||||
# above the suggest cut) auto-tag on a daily sweep. ON by default (opt-out);
|
# above the suggest cut) auto-tag on a daily sweep. ON by default (opt-out);
|
||||||
# single-character references + the high bar keep it safe, every tag reversible.
|
# single-character references + the high bar keep it safe, every tag reversible.
|
||||||
ccip_auto_apply_enabled: Mapped[bool] = mapped_column(
|
ccip_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
ccip_auto_apply_threshold: Mapped[float] = mapped_column(
|
ccip_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||||
# Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident
|
# Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident
|
||||||
# character matches auto-tag.
|
# character matches auto-tag.
|
||||||
Float, nullable=False, default=0.95
|
Float, nullable=False, default=0.95,
|
||||||
|
server_default="0.92",
|
||||||
)
|
)
|
||||||
# -- Presentation chrome auto-hide (#141) -------------------------------
|
# -- Presentation chrome auto-hide (#141) -------------------------------
|
||||||
# `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
|
# `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
|
||||||
@@ -95,13 +109,21 @@ class MLSettings(Base):
|
|||||||
# (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
|
# (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
|
||||||
# screenshot` are no longer chrome — they went to the PROCESS path below.
|
# screenshot` are no longer chrome — they went to the PROCESS path below.
|
||||||
presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
|
presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
presentation_auto_apply_threshold: Mapped[float] = mapped_column(
|
presentation_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.90
|
Float, nullable=False, default=0.90,
|
||||||
|
# text(), not a string, because alembic 0082 used sa.text(): a bare
|
||||||
|
# string renders DEFAULT '0.90'::double precision while text() renders
|
||||||
|
# DEFAULT 0.90, and the chain is MIXED — some migrations used one,
|
||||||
|
# some the other. Same value, different stored expression, so each
|
||||||
|
# column here mirrors whichever form its own migration used (#3275).
|
||||||
|
server_default=text("0.90"),
|
||||||
)
|
)
|
||||||
presentation_conflict_threshold: Mapped[float] = mapped_column(
|
presentation_conflict_threshold: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.50
|
Float, nullable=False, default=0.50,
|
||||||
|
server_default=text("0.50"),
|
||||||
)
|
)
|
||||||
# -- Process auto-apply (#1464) ----------------------------------------
|
# -- Process auto-apply (#1464) ----------------------------------------
|
||||||
# `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
|
# `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
|
||||||
@@ -115,24 +137,29 @@ class MLSettings(Base):
|
|||||||
# (PresentationReview, mode='process') rather than silently marked. OFF by
|
# (PresentationReview, mode='process') rather than silently marked. OFF by
|
||||||
# default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
|
# default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
|
||||||
process_auto_apply_enabled: Mapped[bool] = mapped_column(
|
process_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=False
|
Boolean, nullable=False, default=False,
|
||||||
|
server_default="false",
|
||||||
)
|
)
|
||||||
process_auto_apply_threshold: Mapped[float] = mapped_column(
|
process_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.90
|
Float, nullable=False, default=0.90,
|
||||||
|
server_default="0.90",
|
||||||
)
|
)
|
||||||
process_conflict_threshold: Mapped[float] = mapped_column(
|
process_conflict_threshold: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.50
|
Float, nullable=False, default=0.50,
|
||||||
|
server_default="0.50",
|
||||||
)
|
)
|
||||||
# Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
|
# Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
|
||||||
# existing libraries keep their stored value until the operator re-embeds.
|
# existing libraries keep their stored value until the operator re-embeds.
|
||||||
embedder_model_version: Mapped[str] = mapped_column(
|
embedder_model_version: Mapped[str] = mapped_column(
|
||||||
String(128), nullable=False, default="siglip2-so400m-patch16-512"
|
String(128), nullable=False, default="siglip2-so400m-patch16-512",
|
||||||
|
server_default="siglip2-so400m-patch16-512",
|
||||||
)
|
)
|
||||||
# The HF model NAME the embedder loads (server CPU embed + announced to the
|
# The HF model NAME the embedder loads (server CPU embed + announced to the
|
||||||
# GPU agent in the lease). Operator-settable so the embedder is a choice, not
|
# GPU agent in the lease). Operator-settable so the embedder is a choice, not
|
||||||
# a hardcode (#1190): set name + version together, then re-embed + retrain.
|
# a hardcode (#1190): set name + version together, then re-embed + retrain.
|
||||||
embedder_model_name: Mapped[str] = mapped_column(
|
embedder_model_name: Mapped[str] = mapped_column(
|
||||||
String(128), nullable=False, default="google/siglip2-so400m-patch16-512"
|
String(128), nullable=False, default="google/siglip2-so400m-patch16-512",
|
||||||
|
server_default="google/siglip2-so400m-patch16-512",
|
||||||
)
|
)
|
||||||
# -- Crop proposers / detectors (#1202, #134) --------------------------
|
# -- Crop proposers / detectors (#1202, #134) --------------------------
|
||||||
# WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config
|
# WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config
|
||||||
@@ -145,20 +172,24 @@ class MLSettings(Base):
|
|||||||
# person: general COCO figure detector for Western/realistic art the anime
|
# person: general COCO figure detector for Western/realistic art the anime
|
||||||
# person-detector misses → NMS-merged with imgutils → CCIP + concept.
|
# person-detector misses → NMS-merged with imgutils → CCIP + concept.
|
||||||
detector_person_enabled: Mapped[bool] = mapped_column(
|
detector_person_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
detector_person_weights: Mapped[str] = mapped_column(
|
detector_person_weights: Mapped[str] = mapped_column(
|
||||||
String(512), nullable=False, default="yolo11n.pt"
|
String(512), nullable=False, default="yolo11n.pt",
|
||||||
|
server_default="yolo11n.pt",
|
||||||
)
|
)
|
||||||
detector_person_conf: Mapped[float] = mapped_column(
|
detector_person_conf: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.35
|
Float, nullable=False, default=0.35,
|
||||||
|
server_default=text("0.35"),
|
||||||
)
|
)
|
||||||
# anatomy: booru_yolo anime/furry/NSFW torso components → concept crops.
|
# anatomy: booru_yolo anime/furry/NSFW torso components → concept crops.
|
||||||
# Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the
|
# Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the
|
||||||
# upstream repo so the URL resolves. License UNSTATED — fine for a private
|
# upstream repo so the URL resolves. License UNSTATED — fine for a private
|
||||||
# homelab (operator accepted #1202).
|
# homelab (operator accepted #1202).
|
||||||
detector_anatomy_enabled: Mapped[bool] = mapped_column(
|
detector_anatomy_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
detector_anatomy_weights: Mapped[str] = mapped_column(
|
detector_anatomy_weights: Mapped[str] = mapped_column(
|
||||||
String(512), nullable=False,
|
String(512), nullable=False,
|
||||||
@@ -166,37 +197,47 @@ class MLSettings(Base):
|
|||||||
"https://github.com/aperveyev/booru_yolo/raw/main/models/"
|
"https://github.com/aperveyev/booru_yolo/raw/main/models/"
|
||||||
"yolov11m_aa22.pt"
|
"yolov11m_aa22.pt"
|
||||||
),
|
),
|
||||||
|
server_default="https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt",
|
||||||
)
|
)
|
||||||
detector_anatomy_conf: Mapped[float] = mapped_column(
|
detector_anatomy_conf: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.30
|
Float, nullable=False, default=0.30,
|
||||||
|
server_default=text("0.30"),
|
||||||
)
|
)
|
||||||
# panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x).
|
# panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x).
|
||||||
detector_panel_enabled: Mapped[bool] = mapped_column(
|
detector_panel_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
)
|
)
|
||||||
detector_panel_weights: Mapped[str] = mapped_column(
|
detector_panel_weights: Mapped[str] = mapped_column(
|
||||||
String(512), nullable=False,
|
String(512), nullable=False,
|
||||||
default="mosesb/best-comic-panel-detection::best.pt",
|
default="mosesb/best-comic-panel-detection::best.pt",
|
||||||
|
server_default="mosesb/best-comic-panel-detection::best.pt",
|
||||||
)
|
)
|
||||||
detector_panel_conf: Mapped[float] = mapped_column(
|
detector_panel_conf: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.30
|
Float, nullable=False, default=0.30,
|
||||||
|
server_default=text("0.30"),
|
||||||
)
|
)
|
||||||
# Per-frame caps bound the crop→embed explosion; max_regions is the hard
|
# Per-frame caps bound the crop→embed explosion; max_regions is the hard
|
||||||
# per-job backstop; dedupe_iou drops near-duplicate crops before the embed.
|
# per-job backstop; dedupe_iou drops near-duplicate crops before the embed.
|
||||||
detector_max_figures: Mapped[int] = mapped_column(
|
detector_max_figures: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=8
|
Integer, nullable=False, default=8,
|
||||||
|
server_default="8",
|
||||||
)
|
)
|
||||||
detector_max_components: Mapped[int] = mapped_column(
|
detector_max_components: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=8
|
Integer, nullable=False, default=8,
|
||||||
|
server_default="8",
|
||||||
)
|
)
|
||||||
detector_max_panels: Mapped[int] = mapped_column(
|
detector_max_panels: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=8
|
Integer, nullable=False, default=8,
|
||||||
|
server_default="8",
|
||||||
)
|
)
|
||||||
detector_max_regions: Mapped[int] = mapped_column(
|
detector_max_regions: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=128
|
Integer, nullable=False, default=128,
|
||||||
|
server_default="128",
|
||||||
)
|
)
|
||||||
detector_dedupe_iou: Mapped[float] = mapped_column(
|
detector_dedupe_iou: Mapped[float] = mapped_column(
|
||||||
Float, nullable=False, default=0.85
|
Float, nullable=False, default=0.85,
|
||||||
|
server_default=text("0.85"),
|
||||||
)
|
)
|
||||||
# -- CCIP character prototypes (#1317) ---------------------------------
|
# -- CCIP character prototypes (#1317) ---------------------------------
|
||||||
# The per-character reference set is precomputed + refreshed INCREMENTALLY
|
# The per-character reference set is precomputed + refreshed INCREMENTALLY
|
||||||
@@ -208,7 +249,71 @@ class MLSettings(Base):
|
|||||||
String(128), nullable=True
|
String(128), nullable=True
|
||||||
)
|
)
|
||||||
ccip_prototype_cap: Mapped[int] = mapped_column(
|
ccip_prototype_cap: Mapped[int] = mapped_column(
|
||||||
Integer, nullable=False, default=64
|
Integer, nullable=False, default=64,
|
||||||
|
server_default="64",
|
||||||
|
)
|
||||||
|
# -- Discord drop grouping (milestone 388) -----------------------------
|
||||||
|
# FC authors a post out of a creator's variant drop. The predicate is three
|
||||||
|
# axes ANDed together, and the time one does the real work: SIMILARITY
|
||||||
|
# ALONE OVER-GROUPS. Any two pieces of the same character by the same
|
||||||
|
# artist sit close in SigLIP space, so a cosine-only rule collapses a month
|
||||||
|
# of one character into a single "post". What makes a variant set a set is
|
||||||
|
# that it was dropped TOGETHER.
|
||||||
|
discord_grouping_enabled: Mapped[bool] = mapped_column(
|
||||||
|
# ON by default, matching the operator's standing opt-OUT preference for
|
||||||
|
# automatic behaviour (2026-06-29, recorded on the head/ccip auto-apply
|
||||||
|
# switches). Safe to default on because the act is reversible by one
|
||||||
|
# DELETE: removing a synthetic post un-absorbs its members.
|
||||||
|
Boolean, nullable=False, default=True,
|
||||||
|
server_default="true",
|
||||||
|
)
|
||||||
|
# Cosine DISTANCE, not similarity — this is the units gallery_service's
|
||||||
|
# `cosine_distance` already speaks, and converting at the query site is a
|
||||||
|
# step to get backwards. Lower = stricter. 0.10 is deliberately TIGHT: the
|
||||||
|
# two failure modes are not symmetric. Grouping too shy leaves a drop
|
||||||
|
# scattered, which is visible and fixable by raising this; grouping too
|
||||||
|
# greedy merges distinct pieces into a post that claims they belong
|
||||||
|
# together, which is the failure that would discredit the feature.
|
||||||
|
discord_group_max_distance: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=0.10,
|
||||||
|
server_default=text("0.10"),
|
||||||
|
)
|
||||||
|
# The gap that ENDS a drop, measured between CONSECUTIVE messages rather
|
||||||
|
# than from the first — an artist trickling variants out over an evening is
|
||||||
|
# one drop, and a window anchored on the first message would cut it in half
|
||||||
|
# at an arbitrary point.
|
||||||
|
discord_group_window_minutes: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=60.0,
|
||||||
|
server_default=text("60"),
|
||||||
|
)
|
||||||
|
# How long a synthetic post keeps accepting new members (#388 E3). This is
|
||||||
|
# NOT the drop window above: the window cuts one sweep's messages into
|
||||||
|
# drops, this decides how long a finished drop can still be REJOINED when a
|
||||||
|
# creator adds variants days later. A week by default — long enough for the
|
||||||
|
# "and here is the alt outfit" follow-up that motivated the feature, short
|
||||||
|
# enough that a group does not still be open when the same character comes
|
||||||
|
# round again months later and gets absorbed by mistake.
|
||||||
|
#
|
||||||
|
# Openness is DERIVED from this, not stored: a group is open if it grew (or
|
||||||
|
# started) within this period. So lowering it closes old groups and raising
|
||||||
|
# it reopens them, which is comprehensible and reversible — the alternative,
|
||||||
|
# a stored closed_at, would need its own repair path to ever change.
|
||||||
|
discord_group_close_after_hours: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=168.0,
|
||||||
|
server_default=text("168"),
|
||||||
|
)
|
||||||
|
# Anti-thrash (#388 E3). An updated post SHOULD be visible — that is the
|
||||||
|
# point of keeping it open — but a group gaining one image a day must not
|
||||||
|
# monopolise the feed. Growth smaller than this never moves the post, and
|
||||||
|
# no group moves more than once per cooldown, so a drip-feed updates in
|
||||||
|
# place while a real second wave resurfaces exactly once.
|
||||||
|
discord_group_resurface_min_images: Mapped[int] = mapped_column(
|
||||||
|
Integer, nullable=False, default=2,
|
||||||
|
server_default="2",
|
||||||
|
)
|
||||||
|
discord_group_resurface_cooldown_hours: Mapped[float] = mapped_column(
|
||||||
|
Float, nullable=False, default=24.0,
|
||||||
|
server_default=text("24"),
|
||||||
)
|
)
|
||||||
updated_at: Mapped[datetime] = mapped_column(
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ class PatreonFailedMedia(Base):
|
|||||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
)
|
)
|
||||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
first_failed_at: Mapped[datetime] = mapped_column(
|
first_failed_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ class PixivFailedMedia(Base):
|
|||||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
)
|
)
|
||||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
first_failed_at: Mapped[datetime] = mapped_column(
|
first_failed_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
|||||||
@@ -0,0 +1,130 @@
|
|||||||
|
"""platform_membership — the learned roster of what the account actually pays for.
|
||||||
|
|
||||||
|
Milestone 387, phase C. FabledCurator knows which creators it has been TOLD to
|
||||||
|
follow (`source`), and nothing about which ones the operator is actually
|
||||||
|
subscribed to. Those two sets drift in both directions and the app cannot
|
||||||
|
currently see either drift:
|
||||||
|
|
||||||
|
* A subscription the operator pays for that FC does not track is content they
|
||||||
|
believe they are archiving and are not.
|
||||||
|
* A source FC keeps walking after the subscription lapsed is requests spent on
|
||||||
|
a wall, reported as a creator who has gone quiet.
|
||||||
|
|
||||||
|
This table is the memory that makes both visible — every membership the account
|
||||||
|
has been observed to hold, and when it was last seen.
|
||||||
|
|
||||||
|
## Why a learned roster rather than a live lookup
|
||||||
|
|
||||||
|
Same reasoning as `service_seen` (milestone 365), and the same shape: an
|
||||||
|
absence is only observable against a record of presence. A membership that
|
||||||
|
stops appearing in a sweep is the signal — "you were subscribed to this, now
|
||||||
|
you aren't" — and there is nowhere to read that from a live call, because a
|
||||||
|
live call returns what IS, never what stopped being.
|
||||||
|
|
||||||
|
It also means the reconciliation surface keeps working when Patreon is
|
||||||
|
unreachable, degraded to a stale roster with a visible age rather than an empty
|
||||||
|
page (rule 164).
|
||||||
|
|
||||||
|
## Roster truth, NOT per-post truth
|
||||||
|
|
||||||
|
The single most important thing about this table: `tier_names` says which tiers
|
||||||
|
the account holds. It does **not** say which posts those tiers unlock. A
|
||||||
|
creator can gate a post behind an access rule that maps onto no tier name at
|
||||||
|
all.
|
||||||
|
|
||||||
|
`current_user_can_view` — read per post by `patreon_client.post_is_gated` — is
|
||||||
|
the authoritative signal, and phase A already turned it into a durable
|
||||||
|
per-source state. This roster EXPLAINS that state ("you are no longer a patron"
|
||||||
|
vs "your tier doesn't cover these posts"). It must never be used to decide
|
||||||
|
whether to fetch something. Getting that backwards would make FC silently stop
|
||||||
|
fetching content the operator is paying for, which is the worst failure
|
||||||
|
available in this milestone.
|
||||||
|
|
||||||
|
## status is a plain String, and deliberately the platform's own word
|
||||||
|
|
||||||
|
Not a Postgres ENUM, not CHECK-gated — matching `service_seen.kind`,
|
||||||
|
`gpu_job.status` and `source.error_type`. Two reasons, and the first is the
|
||||||
|
real one:
|
||||||
|
|
||||||
|
1. **The vocabulary is not ours to invent.** Patreon says `active_patron` /
|
||||||
|
`former_patron` / `declined_patron`; SubscribeStar and FANBOX will say
|
||||||
|
something else. Storing each platform's own word verbatim and mapping to
|
||||||
|
FC's meaning at the READ site keeps this table a record of what was
|
||||||
|
observed rather than a lossy translation of it. A lowest-common-denominator
|
||||||
|
enum picked before any platform has been characterised (step C0) would be a
|
||||||
|
guess baked into the schema.
|
||||||
|
2. A constraint swap per new value (rule 36) would be cost with no invariant
|
||||||
|
behind it, exactly as `service_seen.kind` records.
|
||||||
|
|
||||||
|
The service layer owns the whitelist and the mapping; the column owns the
|
||||||
|
evidence.
|
||||||
|
|
||||||
|
## Retention: aged out, never deleted on disappearance
|
||||||
|
|
||||||
|
A membership that stops appearing in a sweep is NOT removed. Its disappearance
|
||||||
|
is the fact the reconciliation surface reads, and deleting the row would
|
||||||
|
destroy the signal at the moment it became interesting. `last_seen_at` is what
|
||||||
|
makes "gone" decidable, and a retention policy ages rows out on time rather
|
||||||
|
than on absence.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import JSON, DateTime, Integer, String, Text, UniqueConstraint, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class PlatformMembership(Base):
|
||||||
|
__tablename__ = "platform_membership"
|
||||||
|
__table_args__ = (
|
||||||
|
# The natural key the sweep's upsert conflicts on. Named explicitly
|
||||||
|
# because `touch_membership` references it by name in ON CONFLICT.
|
||||||
|
UniqueConstraint(
|
||||||
|
"platform", "external_campaign_id",
|
||||||
|
name="uq_platform_membership_platform_campaign",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
|
||||||
|
platform: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
# The platform's own id for the thing subscribed to — a Patreon campaign
|
||||||
|
# id, whatever SubscribeStar and FANBOX call theirs. Text rather than a
|
||||||
|
# bounded String: these are opaque upstream identifiers and guessing a
|
||||||
|
# ceiling for a value we do not mint is how a walk dies on a truncation.
|
||||||
|
external_campaign_id: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
|
||||||
|
# For the reconciliation UI, and for matching against Source.url — the
|
||||||
|
# vanity/URL is what the two sides actually have in common.
|
||||||
|
display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
||||||
|
# The platform's own word. See the module docstring — this is evidence,
|
||||||
|
# not a normalised FC status.
|
||||||
|
status: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||||
|
|
||||||
|
# Nullable throughout: a free follow has no tier and no money attached, and
|
||||||
|
# a platform may not expose an amount at all. Absent must stay
|
||||||
|
# distinguishable from zero — "free" and "we don't know" are different
|
||||||
|
# answers to "what is this costing".
|
||||||
|
tier_names: Mapped[list | None] = mapped_column(JSON, nullable=True)
|
||||||
|
amount_cents: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
currency: Mapped[str | None] = mapped_column(String(8), nullable=True)
|
||||||
|
|
||||||
|
# NEVER updated after insert. The one field that answers "has this ever
|
||||||
|
# been true", which is what makes a disappearance readable rather than
|
||||||
|
# indistinguishable from never having existed. `touch_membership`
|
||||||
|
# deliberately excludes it from the ON CONFLICT update set.
|
||||||
|
first_seen_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||||
|
)
|
||||||
|
last_seen_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# The raw membership as the platform returned it, so a later question can
|
||||||
|
# be answered without re-fetching — and so a field we did not think to
|
||||||
|
# model is not lost. Displayed and never queried, like service_seen.details.
|
||||||
|
details: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
@@ -13,11 +13,13 @@ from sqlalchemy import (
|
|||||||
CheckConstraint,
|
CheckConstraint,
|
||||||
DateTime,
|
DateTime,
|
||||||
ForeignKey,
|
ForeignKey,
|
||||||
|
Index,
|
||||||
Integer,
|
Integer,
|
||||||
String,
|
String,
|
||||||
Text,
|
Text,
|
||||||
UniqueConstraint,
|
UniqueConstraint,
|
||||||
func,
|
func,
|
||||||
|
text,
|
||||||
)
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
@@ -27,6 +29,10 @@ from .base import Base
|
|||||||
class Post(Base):
|
class Post(Base):
|
||||||
__tablename__ = "post"
|
__tablename__ = "post"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
# alembic 0030. The comment above described this index; nothing declared
|
||||||
|
# it, so autogenerate proposed dropping it (#3275).
|
||||||
|
Index("uq_post_artist_external_id_null_source", "artist_id", "external_post_id",
|
||||||
|
unique=True, postgresql_where=text("source_id IS NULL")),
|
||||||
# Source-bound dedup. Postgres treats NULL != NULL so rows
|
# Source-bound dedup. Postgres treats NULL != NULL so rows
|
||||||
# with source_id IS NULL aren't deduped by this constraint;
|
# with source_id IS NULL aren't deduped by this constraint;
|
||||||
# the partial unique index `uq_post_artist_external_id_null_source`
|
# the partial unique index `uq_post_artist_external_id_null_source`
|
||||||
@@ -35,7 +41,11 @@ class Post(Base):
|
|||||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||||
CheckConstraint(
|
CheckConstraint(
|
||||||
"translation_override IN ('auto', 'force', 'original')",
|
"translation_override IN ('auto', 'force', 'original')",
|
||||||
name="ck_post_translation_override",
|
# Bare name: Base.metadata's naming convention prepends
|
||||||
|
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
||||||
|
# alembic 0088, which renames the four constraints that shipped
|
||||||
|
# that way (#3275).
|
||||||
|
name="translation_override",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -92,3 +102,61 @@ class Post(Base):
|
|||||||
downloaded_at: Mapped[datetime] = mapped_column(
|
downloaded_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# -- Synthetic posts (milestone 388). ----------------------------------
|
||||||
|
# Discord is a delivery CHANNEL, not a publisher: one message is not one
|
||||||
|
# post. So FC authors the post itself, grouping a creator's variant drop
|
||||||
|
# into a single row (services/discord_grouping.py).
|
||||||
|
#
|
||||||
|
# NULL for every post a creator actually wrote — which is all of them until
|
||||||
|
# a grouper runs. Non-NULL names the grouper that authored this row, and is
|
||||||
|
# the ONE flag the UI keys off to say so. The honesty rule is the whole
|
||||||
|
# point: a synthetic post must never present itself as authored, and a
|
||||||
|
# column that is absent-or-a-name makes "was this us?" answerable from the
|
||||||
|
# row rather than inferred from its shape.
|
||||||
|
#
|
||||||
|
# Plain String, no CHECK (rule 36 considered and declined) — same reasoning
|
||||||
|
# as source.error_type and service_seen.kind. There is exactly one grouper
|
||||||
|
# today; a second would be a value, not an invariant.
|
||||||
|
synthesized_by: Mapped[str | None] = mapped_column(String(32), nullable=True)
|
||||||
|
# What it was built from, so the operator can audit a grouping FC invented:
|
||||||
|
# member post ids, message count, and the thresholds in force when the
|
||||||
|
# decision was made. That last part matters — the thresholds are operator-
|
||||||
|
# tunable, so "why did it group these" is unanswerable a month later
|
||||||
|
# without recording the values that produced it.
|
||||||
|
synthesis_details: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
# Set on a MEMBER post, pointing at the synthetic post that absorbed it.
|
||||||
|
# The feed hides absorbed posts (they are the chat lines the synthetic post
|
||||||
|
# replaced); every other surface still reaches them by id, because they
|
||||||
|
# remain the image's true origin and the grouping has to be inspectable.
|
||||||
|
#
|
||||||
|
# Self-FK, ON DELETE SET NULL: deleting a synthetic post un-absorbs its
|
||||||
|
# members and they return to the feed on their own. That is the reversal
|
||||||
|
# path, and it is one DELETE — nothing to undo by hand.
|
||||||
|
absorbed_by_post_id: Mapped[int | None] = mapped_column(
|
||||||
|
ForeignKey("post.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
)
|
||||||
|
# -- An OPEN grouping (milestone 388 E3) -------------------------------
|
||||||
|
# A synthetic post is not sealed at creation: a creator who adds two more
|
||||||
|
# variants the next day extends the existing post rather than starting a
|
||||||
|
# new one. These two columns are what make that possible without the post
|
||||||
|
# either freezing or thrashing the feed.
|
||||||
|
#
|
||||||
|
# `last_grew_at` is when the group last absorbed something. It answers two
|
||||||
|
# questions: how long the group stays JOINABLE (a group closes after a
|
||||||
|
# quiet period — artists reuse characters for years, and a group left open
|
||||||
|
# forever will eventually absorb something it shouldn't), and what the card
|
||||||
|
# shows as "updated N ago". NULL means it has never grown since creation.
|
||||||
|
last_grew_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
# The feed position, and ONLY set when the anti-thrash rule fires — see
|
||||||
|
# discord_grouping.should_resurface. A group that gains one image a day
|
||||||
|
# must not sit permanently at the top of the feed, so growth updates the
|
||||||
|
# post without necessarily moving it; a genuine second wave moves it once.
|
||||||
|
#
|
||||||
|
# NULL on every ordinary post, which is why the feed's sort key can
|
||||||
|
# COALESCE through it without changing where anything else lands.
|
||||||
|
resurfaced_at: Mapped[datetime | None] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=True
|
||||||
|
)
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
"""PostAssociation — "this Patreon post announced that Discord drop".
|
||||||
|
|
||||||
|
Milestone 388, step E5, and the point of the milestone rather than its tail.
|
||||||
|
|
||||||
|
Two of the operator's artists post a deliberately CROPPED fragment on Patreon
|
||||||
|
to signal that the real thing has landed in their Discord. The Patreon post is
|
||||||
|
the announcement; the Discord grouping (milestone 388 E2) is the payload. This
|
||||||
|
row is the link between them.
|
||||||
|
|
||||||
|
## Directional, and NOT a merge
|
||||||
|
|
||||||
|
`announcement` → `payload` is asymmetric on purpose. The teaser announces the
|
||||||
|
drop; the drop does not announce the teaser, and a symmetric "related posts"
|
||||||
|
edge would lose the only thing that makes the pair interesting.
|
||||||
|
|
||||||
|
Nor are the two collapsed into one post. The creator published twice,
|
||||||
|
deliberately, on two platforms with different audiences — flattening that
|
||||||
|
hides the very behaviour being modelled, and would destroy the operator's
|
||||||
|
ability to see that the Patreon post is a teaser at all.
|
||||||
|
|
||||||
|
## Confirm-only, following FC-6.3 (task 737)
|
||||||
|
|
||||||
|
`status` starts at `pending` and nothing is linked until the operator accepts.
|
||||||
|
A wrongly-asserted association tells them two different pieces are one, which
|
||||||
|
is worse than no link: no link leaves them where they already are, a wrong one
|
||||||
|
actively misinforms. Same reason the series matcher writes to a review queue
|
||||||
|
instead of filing posts on its own.
|
||||||
|
|
||||||
|
`status` is a plain String, no CHECK — matching `series_suggestion.status`,
|
||||||
|
which records the same check-existing-enums lesson.
|
||||||
|
|
||||||
|
## Why pHash could not do this, and the correction matters
|
||||||
|
|
||||||
|
The original plan claimed this link was already sitting in `image_provenance`
|
||||||
|
via pHash dedup. It is not. `compute_phash` is `imagehash.phash` at
|
||||||
|
`hash_size=8` — a DCT hash over the WHOLE image, robust to rescaling and
|
||||||
|
recompression but NOT to cropping, because a crop changes the global
|
||||||
|
signature. Cross-platform provenance still links straight re-posts; it does
|
||||||
|
nothing for a cropped teaser and its full version, which is precisely the pair
|
||||||
|
the operator described. Hence a scored proposal rather than a lookup.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import (
|
||||||
|
JSON,
|
||||||
|
DateTime,
|
||||||
|
Float,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
)
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class PostAssociation(Base):
|
||||||
|
__tablename__ = "post_association"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"announcement_post_id", "payload_post_id",
|
||||||
|
name="uq_post_association_pair",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
# The teaser — a real post the creator wrote (Patreon, today).
|
||||||
|
announcement_post_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
# What it announced — a synthetic Discord grouping, today. CASCADE on both
|
||||||
|
# sides: an association to a post that no longer exists is not a fact worth
|
||||||
|
# keeping, and E3's reversal path (delete the grouping) should not leave a
|
||||||
|
# dangling proposal behind.
|
||||||
|
payload_post_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
|
||||||
|
score: Mapped[float] = mapped_column(Float, nullable=False)
|
||||||
|
# Per-signal strengths as scored, so a proposal stays explicable after the
|
||||||
|
# weights or the threshold are tuned. Without it, "why was this suggested"
|
||||||
|
# is unanswerable the moment anything moves.
|
||||||
|
signals: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
# pending | linked | dismissed. A DISMISSED row is kept, not deleted — it
|
||||||
|
# is what stops the matcher proposing the same rejected pair on every
|
||||||
|
# subsequent scan, which is the behaviour that makes a review queue
|
||||||
|
# unusable.
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(16), nullable=False, server_default="pending", index=True
|
||||||
|
)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False,
|
||||||
|
server_default=func.now(), onupdate=func.now(),
|
||||||
|
)
|
||||||
@@ -11,7 +11,7 @@ are pruned by retention.
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime, Float, ForeignKey, String, func
|
from sqlalchemy import DateTime, Float, ForeignKey, Index, String, func
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -20,6 +20,14 @@ from .base import Base
|
|||||||
class PresentationReview(Base):
|
class PresentationReview(Base):
|
||||||
__tablename__ = "presentation_review"
|
__tablename__ = "presentation_review"
|
||||||
|
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
Index("ix_presentation_review_resolved_at", "resolved_at"),
|
||||||
|
# Both FKs to tag were unindexed (#3300); tag_id CASCADEs, so a tag
|
||||||
|
# delete had to scan this table to find its rows.
|
||||||
|
Index("ix_presentation_review_tag_id", "tag_id"),
|
||||||
|
Index("ix_presentation_review_conflict_tag_id", "conflict_tag_id"),
|
||||||
|
)
|
||||||
image_record_id: Mapped[int] = mapped_column(
|
image_record_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,7 +16,14 @@ title is the optional chapter name; stated_part is the optional operator-facing
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime, ForeignKey, Integer, Text, func
|
from sqlalchemy import (
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -25,14 +32,26 @@ from .base import Base
|
|||||||
class SeriesChapter(Base):
|
class SeriesChapter(Base):
|
||||||
__tablename__ = "series_chapter"
|
__tablename__ = "series_chapter"
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# alembic 0047 named the UNIQUE `uq_series_chapter_anchor_page`, not
|
||||||
|
# the `uq_series_chapter_anchor_page_id` a bare `unique=True` would
|
||||||
|
# render (#3275).
|
||||||
|
UniqueConstraint("anchor_page_id", name="uq_series_chapter_anchor_page"),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
series_tag_id: Mapped[int] = mapped_column(
|
series_tag_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
)
|
)
|
||||||
|
# Both the UNIQUE (above) and the FK carry the names 0047 gave them; the
|
||||||
|
# convention would render the FK `fk_series_chapter_anchor_page_id_series_page`.
|
||||||
anchor_page_id: Mapped[int] = mapped_column(
|
anchor_page_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("series_page.id", ondelete="CASCADE"),
|
ForeignKey(
|
||||||
|
"series_page.id",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
name="fk_series_chapter_anchor_page",
|
||||||
|
),
|
||||||
nullable=False,
|
nullable=False,
|
||||||
unique=True,
|
|
||||||
)
|
)
|
||||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
stated_part: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
stated_part: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
|
|||||||
@@ -14,7 +14,14 @@ number parsed from the source post, nullable when unknown.
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime, ForeignKey, Integer, String, func
|
from sqlalchemy import (
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
UniqueConstraint,
|
||||||
|
func,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -23,14 +30,22 @@ from .base import Base
|
|||||||
class SeriesPage(Base):
|
class SeriesPage(Base):
|
||||||
__tablename__ = "series_page"
|
__tablename__ = "series_page"
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# alembic 0005 named this `uq_series_page_image`; a bare `unique=True`
|
||||||
|
# on the column renders `uq_series_page_image_id` under the naming
|
||||||
|
# convention, which is a different object from the one the database
|
||||||
|
# has (#3275).
|
||||||
|
UniqueConstraint("image_id", name="uq_series_page_image"),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
series_tag_id: Mapped[int] = mapped_column(
|
series_tag_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
)
|
)
|
||||||
|
# UNIQUE lives in __table_args__ above, under the name 0005 gave it.
|
||||||
image_id: Mapped[int] = mapped_column(
|
image_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("image_record.id", ondelete="CASCADE"),
|
ForeignKey("image_record.id", ondelete="CASCADE"),
|
||||||
nullable=False,
|
nullable=False,
|
||||||
unique=True,
|
|
||||||
)
|
)
|
||||||
# 'placed' = in the series-global run (page_number set); 'pending' = staged
|
# 'placed' = in the series-global run (page_number set); 'pending' = staged
|
||||||
# from a post awaiting the operator's sort (page_number NULL). (#789 P2)
|
# from a post awaiting the operator's sort (page_number NULL). (#789 P2)
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""service_seen — the learned roster of FabledCurator's own moving parts.
|
||||||
|
|
||||||
|
Nothing else in this application knows what is SUPPOSED to be running.
|
||||||
|
`celery inspect` reports the workers that answer, so a stopped worker is a
|
||||||
|
shorter list rather than a red light, and Postgres and Redis have no
|
||||||
|
representation at all. That is why the only place an operator could see a
|
||||||
|
dead service was Portainer, which knows the intended set (milestone 365).
|
||||||
|
|
||||||
|
This table is the memory that makes an absence observable: every part that
|
||||||
|
has ever checked in, and when it last did. A row that stops advancing is a
|
||||||
|
part that stopped.
|
||||||
|
|
||||||
|
## Why the key is not the hostname
|
||||||
|
|
||||||
|
`_read_workers_sync()` returns celery's worker names, which here are
|
||||||
|
`celery@<container id>`. Those are minted fresh on every deploy. Keyed on
|
||||||
|
them, this table would record a death and a birth every time the stack is
|
||||||
|
updated — and a status page that goes red on every deploy is a status page
|
||||||
|
nobody reads, which is worse than not having one.
|
||||||
|
|
||||||
|
So a celery role is keyed on its **queue set**, which is assigned per role in
|
||||||
|
docker-compose.yml (`CELERY_QUEUES`) and survives container replacement:
|
||||||
|
|
||||||
|
default,import,thumbnail,download -> worker
|
||||||
|
maintenance,scan -> scheduler (celery worker --beat)
|
||||||
|
ml -> ml-worker
|
||||||
|
|
||||||
|
Two replicas of one role share a queue set and are therefore ONE row — which
|
||||||
|
is right, because the question being answered is "is that role being served",
|
||||||
|
not "how many containers exist". The replica count and their hostnames go in
|
||||||
|
`details`, where they can change without the identity changing.
|
||||||
|
|
||||||
|
The GPU agent is keyed on its `agent_id`, the identity its lease protocol
|
||||||
|
already uses (`api/gpu.py`).
|
||||||
|
|
||||||
|
## What is NOT in here
|
||||||
|
|
||||||
|
Postgres and Redis. They are always expected and never learned, and a
|
||||||
|
last-seen for them would be actively misleading — that one answered thirty
|
||||||
|
seconds ago says nothing about now. They are probed live at request time.
|
||||||
|
|
||||||
|
## kind
|
||||||
|
|
||||||
|
Plain `String`, not a Postgres ENUM and not CHECK-gated, matching
|
||||||
|
`gpu_job.status` and `backup_run.status`. The value set here is expected to
|
||||||
|
grow as parts are added, and a constraint swap per new kind (rule 36) would
|
||||||
|
be cost with no invariant behind it.
|
||||||
|
|
||||||
|
celery — a worker role, keyed on its queue set
|
||||||
|
agent — a GPU agent, keyed on its agent_id
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import JSON, DateTime, String, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceSeen(Base):
|
||||||
|
__tablename__ = "service_seen"
|
||||||
|
|
||||||
|
# No indexes beyond the primary key, deliberately. This table holds one row
|
||||||
|
# per moving part — a handful, forever — so every query against it is a
|
||||||
|
# full read of a few rows and an index would be write cost buying nothing
|
||||||
|
# (the lesson of #3301, which removed seven redundant ones).
|
||||||
|
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||||
|
kind: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||||
|
|
||||||
|
# What to call it in the UI. Derived from the queue set where it is
|
||||||
|
# recognised, and falling back to the raw queue list where it is not — a
|
||||||
|
# deployment that slices its queues differently should still show something
|
||||||
|
# true rather than a name this code invented for it.
|
||||||
|
display_name: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
|
||||||
|
first_seen_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||||
|
)
|
||||||
|
last_seen_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now(),
|
||||||
|
)
|
||||||
|
|
||||||
|
# The parts that change without changing identity: replica hostnames,
|
||||||
|
# active task counts, the queues actually being served. Kept as a blob
|
||||||
|
# because it is displayed and never queried — giving it columns would
|
||||||
|
# invite filtering on it, which is what the activity endpoints are for.
|
||||||
|
details: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
@@ -5,7 +5,16 @@ Multiple sources per artist support creators with cross-platform presence.
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text
|
from sqlalchemy import (
|
||||||
|
JSON,
|
||||||
|
Boolean,
|
||||||
|
DateTime,
|
||||||
|
ForeignKey,
|
||||||
|
Integer,
|
||||||
|
String,
|
||||||
|
Text,
|
||||||
|
UniqueConstraint,
|
||||||
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -14,13 +23,27 @@ from .base import Base
|
|||||||
class Source(Base):
|
class Source(Base):
|
||||||
__tablename__ = "source"
|
__tablename__ = "source"
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# alembic 0010. One row per (artist, platform, url): re-adding a source
|
||||||
|
# the artist already has is an update, not a second row. The model had
|
||||||
|
# never declared it (#3275), so autogenerate would have proposed
|
||||||
|
# DROPPING it — the guarantee existed only in the migration chain.
|
||||||
|
#
|
||||||
|
# Named explicitly because the naming convention would render this
|
||||||
|
# `uq_source_artist_id` (uq keys off column_0_name), which is both
|
||||||
|
# wrong about the shape and not what the database actually has.
|
||||||
|
UniqueConstraint(
|
||||||
|
"artist_id", "platform", "url", name="uq_source_artist_platform_url"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
artist_id: Mapped[int] = mapped_column(
|
artist_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
)
|
)
|
||||||
platform: Mapped[str] = mapped_column(String(64), nullable=False)
|
platform: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
url: Mapped[str] = mapped_column(Text, nullable=False)
|
url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||||
|
|
||||||
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
|
||||||
@@ -32,7 +55,7 @@ class Source(Base):
|
|||||||
# by _update_source_health alongside last_error; cleared on 'ok'.
|
# by _update_source_health alongside last_error; cleared on 'ok'.
|
||||||
error_type: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
error_type: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||||
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||||
|
|
||||||
# alembic 0031: sticky deep-scan budget. When > 0, the next N download
|
# alembic 0031: sticky deep-scan budget. When > 0, the next N download
|
||||||
# runs use gallery-dl's full-walk config (skip: True + 1800s timeout);
|
# runs use gallery-dl's full-walk config (skip: True + 1800s timeout);
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ class SubscribeStarFailedMedia(Base):
|
|||||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
)
|
)
|
||||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
first_failed_at: Mapped[datetime] = mapped_column(
|
first_failed_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
|||||||
@@ -15,11 +15,13 @@ from sqlalchemy import (
|
|||||||
Column,
|
Column,
|
||||||
DateTime,
|
DateTime,
|
||||||
ForeignKey,
|
ForeignKey,
|
||||||
|
Index,
|
||||||
Integer,
|
Integer,
|
||||||
String,
|
String,
|
||||||
Table,
|
Table,
|
||||||
false,
|
false,
|
||||||
func,
|
func,
|
||||||
|
text,
|
||||||
)
|
)
|
||||||
from sqlalchemy import (
|
from sqlalchemy import (
|
||||||
Enum as SQLEnum,
|
Enum as SQLEnum,
|
||||||
@@ -67,17 +69,31 @@ image_tag = Table(
|
|||||||
primary_key=True,
|
primary_key=True,
|
||||||
),
|
),
|
||||||
Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True),
|
Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True),
|
||||||
Column("source", String(32), nullable=False, default="manual"),
|
Column("source", String(32), nullable=False, default="manual", server_default="manual"),
|
||||||
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
|
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
|
||||||
|
# The PK is (image_record_id, tag_id), which leads with the WRONG column
|
||||||
|
# for the two things that matter most here (#3300): the gallery's tag
|
||||||
|
# filter (tag_query.py builds `image_tag.c.tag_id == tid`) and the
|
||||||
|
# ON DELETE CASCADE from tag, which has to find a tag's rows to remove
|
||||||
|
# them. Without this index both scan the largest table in the schema.
|
||||||
|
Index("ix_image_tag_tag_id", "tag_id"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class Tag(Base):
|
class Tag(Base):
|
||||||
__tablename__ = "tag"
|
__tablename__ = "tag"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
# alembic 0002. An EXPRESSION index — COALESCE cannot be expressed as a
|
||||||
|
# UniqueConstraint, which is why it only ever existed in a migration (#3275).
|
||||||
|
Index("uq_tag_name_kind_fandom", "name", "kind", text("COALESCE(fandom_id, 0)"),
|
||||||
|
unique=True),
|
||||||
CheckConstraint(
|
CheckConstraint(
|
||||||
"(fandom_id IS NULL) OR (kind = 'character')",
|
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||||
name="ck_tag_fandom_requires_character",
|
# Bare name: Base.metadata's naming convention prepends
|
||||||
|
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
||||||
|
# alembic 0088, which renames the four constraints that shipped
|
||||||
|
# that way (#3275).
|
||||||
|
name="fandom_requires_character",
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -87,6 +103,7 @@ class Tag(Base):
|
|||||||
SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]),
|
SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]),
|
||||||
nullable=False,
|
nullable=False,
|
||||||
default=TagKind.general,
|
default=TagKind.general,
|
||||||
|
server_default="general",
|
||||||
)
|
)
|
||||||
fandom_id: Mapped[int | None] = mapped_column(
|
fandom_id: Mapped[int | None] = mapped_column(
|
||||||
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
|
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ in image_prediction stay unmolested.
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime, ForeignKey, String, func
|
from sqlalchemy import DateTime, ForeignKey, Index, String, func
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -14,10 +14,17 @@ from .base import Base
|
|||||||
class TagAlias(Base):
|
class TagAlias(Base):
|
||||||
__tablename__ = "tag_alias"
|
__tablename__ = "tag_alias"
|
||||||
|
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# Named explicitly: the database calls this ix_tag_alias_canonical, while
|
||||||
|
# a bare index=True on the column would generate ix_tag_alias_canonical_tag_id
|
||||||
|
# and silently propose a drop+create on the next autogenerate (#3275).
|
||||||
|
Index("ix_tag_alias_canonical", "canonical_tag_id"),
|
||||||
|
)
|
||||||
alias_string: Mapped[str] = mapped_column(String(255), primary_key=True)
|
alias_string: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||||
alias_category: Mapped[str] = mapped_column(String(32), primary_key=True)
|
alias_category: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||||
canonical_tag_id: Mapped[int] = mapped_column(
|
canonical_tag_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False
|
||||||
)
|
)
|
||||||
created_at: Mapped[datetime] = mapped_column(
|
created_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ Prevents re-suggestion AND prevents allowlist auto-apply on that image.
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime, ForeignKey, func
|
from sqlalchemy import DateTime, ForeignKey, Index, func
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -14,11 +14,24 @@ from .base import Base
|
|||||||
class TagSuggestionRejection(Base):
|
class TagSuggestionRejection(Base):
|
||||||
__tablename__ = "tag_suggestion_rejection"
|
__tablename__ = "tag_suggestion_rejection"
|
||||||
|
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# Named explicitly; see tag_alias for why (#3275).
|
||||||
|
Index("ix_tag_suggestion_rejection_tag", "tag_id"),
|
||||||
|
)
|
||||||
|
# Both FKs named explicitly. alembic 0003 used a hand-shortened `tsr`
|
||||||
|
# prefix; the convention would render the full table name (#3275).
|
||||||
image_record_id: Mapped[int] = mapped_column(
|
image_record_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
ForeignKey(
|
||||||
|
"image_record.id",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
name="fk_tsr_image_record_id_image_record",
|
||||||
|
),
|
||||||
|
primary_key=True,
|
||||||
)
|
)
|
||||||
tag_id: Mapped[int] = mapped_column(
|
tag_id: Mapped[int] = mapped_column(
|
||||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True
|
ForeignKey("tag.id", ondelete="CASCADE", name="fk_tsr_tag_id_tag"),
|
||||||
|
primary_key=True,
|
||||||
)
|
)
|
||||||
rejected_at: Mapped[datetime] = mapped_column(
|
rejected_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min).
|
|||||||
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from sqlalchemy import DateTime, Integer, String, Text
|
from sqlalchemy import DateTime, Index, Integer, String, Text, text
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
|
||||||
from .base import Base
|
from .base import Base
|
||||||
@@ -24,12 +24,21 @@ from .base import Base
|
|||||||
class TaskRun(Base):
|
class TaskRun(Base):
|
||||||
__tablename__ = "task_run"
|
__tablename__ = "task_run"
|
||||||
|
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# alembic 0016: the three task-history indexes (#3275).
|
||||||
|
Index("ix_task_run_name_started", "task_name", text("started_at DESC")),
|
||||||
|
Index("ix_task_run_queue_started", "queue", text("started_at DESC")),
|
||||||
|
Index("ix_task_run_status_started", "status", text("started_at DESC")),
|
||||||
|
)
|
||||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
celery_task_id: Mapped[str] = mapped_column(
|
celery_task_id: Mapped[str] = mapped_column(
|
||||||
String(64), nullable=False, index=True,
|
String(64), nullable=False, index=True,
|
||||||
)
|
)
|
||||||
queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
# Neither carries index=True: ix_task_run_queue_started and
|
||||||
task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
# ix_task_run_name_started already lead with these columns (#3301).
|
||||||
|
queue: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
task_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
started_at: Mapped[datetime] = mapped_column(
|
started_at: Mapped[datetime] = mapped_column(
|
||||||
DateTime(timezone=True), nullable=False, index=True,
|
DateTime(timezone=True), nullable=False, index=True,
|
||||||
@@ -39,7 +48,9 @@ class TaskRun(Base):
|
|||||||
)
|
)
|
||||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
status: Mapped[str] = mapped_column(
|
status: Mapped[str] = mapped_column(
|
||||||
String(16), nullable=False, default="running", index=True,
|
# No index=True — ix_task_run_status_started leads with `status`.
|
||||||
|
String(16), nullable=False, default="running",
|
||||||
|
server_default="running",
|
||||||
)
|
)
|
||||||
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|||||||
@@ -20,6 +20,9 @@ from sqlalchemy import Select
|
|||||||
from sqlalchemy.exc import IntegrityError
|
from sqlalchemy.exc import IntegrityError
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from ..models import Source
|
||||||
|
from .gallery_dl import ErrorType
|
||||||
|
|
||||||
|
|
||||||
async def get_or_create[T](
|
async def get_or_create[T](
|
||||||
session: AsyncSession,
|
session: AsyncSession,
|
||||||
@@ -50,3 +53,31 @@ async def get_or_create[T](
|
|||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
await sp.rollback()
|
await sp.rollback()
|
||||||
return (await session.execute(select_stmt)).scalar_one(), False
|
return (await session.execute(select_stmt)).scalar_one(), False
|
||||||
|
|
||||||
|
|
||||||
|
# --- shared Source health predicates ----------------------------------------
|
||||||
|
#
|
||||||
|
# The subscriptions rollup, the front-door status ribbon and the list endpoint
|
||||||
|
# all have to agree on what "failing" and "no access" MEAN, or the ribbon says
|
||||||
|
# 3 and the card it links to shows 4. Same reasoning as get_or_create above:
|
||||||
|
# divergent copies of one predicate are how the drift creeps in. Defined here
|
||||||
|
# rather than in source_service because scheduler_service needs them too, and
|
||||||
|
# source_service already imports scheduler_service (the other direction would
|
||||||
|
# be a cycle).
|
||||||
|
|
||||||
|
|
||||||
|
def failing_sources_clause():
|
||||||
|
"""A source is FAILING when its runs are actually erroring.
|
||||||
|
|
||||||
|
Deliberately not `last_error IS NOT NULL` — a tier-limited source clears
|
||||||
|
last_error and keeps a chip, and must never be counted as broken.
|
||||||
|
"""
|
||||||
|
return Source.consecutive_failures > 0
|
||||||
|
|
||||||
|
|
||||||
|
def no_access_sources_clause():
|
||||||
|
"""A source we can't see the content of: the walk works, the tier doesn't
|
||||||
|
grant it (#874 / milestone #387 phase A). Not a failure — kept separate
|
||||||
|
from failing_sources_clause on purpose, and the two are disjoint because
|
||||||
|
an informational class only ever rides an otherwise-OK run."""
|
||||||
|
return Source.error_type == ErrorType.TIER_LIMITED
|
||||||
|
|||||||
@@ -0,0 +1,655 @@
|
|||||||
|
"""Discord drop grouping — FC authors the post that Discord never wrote.
|
||||||
|
|
||||||
|
Milestone 388, step E2.
|
||||||
|
|
||||||
|
Discord is a delivery CHANNEL, not a publisher. A creator drops a set of
|
||||||
|
near-variants — the same piece with different hair colour, accessories, an
|
||||||
|
outfit swap — across a handful of messages, and today each of those messages
|
||||||
|
lands as its own `post` row, so chat lines compete with authored work for the
|
||||||
|
same surface. The fix is not to demote them into a second-class feed; it is to
|
||||||
|
let FC write the post: one row per DROP, its images the drop's images, its body
|
||||||
|
the messages' text in arrival order.
|
||||||
|
|
||||||
|
The result is post-shaped by construction, which is the entire reason to
|
||||||
|
synthesise a `Post` rather than invent a parallel entity — feed, provenance,
|
||||||
|
translation, attachments and series all keep working on it unchanged.
|
||||||
|
|
||||||
|
## The predicate: three axes, ANDed, and the time one does the real work
|
||||||
|
|
||||||
|
**Similarity alone over-groups, and that is the failure that would make this
|
||||||
|
useless.** Any two pieces of the same character by the same artist sit close in
|
||||||
|
SigLIP space; a cosine-only rule collapses a month of one character into a
|
||||||
|
single "post". What makes a variant set a set is that it was dropped TOGETHER.
|
||||||
|
|
||||||
|
same source AND cosine distance <= threshold AND no gap > window
|
||||||
|
|
||||||
|
Two details in there are load-bearing:
|
||||||
|
|
||||||
|
* **Distance is measured to the group's SEED, never to the previous member.**
|
||||||
|
Chaining to the previous member lets a group DRIFT: twenty small steps walk
|
||||||
|
from one piece to a completely different one, each hop individually within
|
||||||
|
threshold. Anchoring on the seed bounds the whole group to one neighbourhood.
|
||||||
|
* **The window is measured between CONSECUTIVE messages, not from the first.**
|
||||||
|
An artist trickling variants out over an evening is one drop; a window
|
||||||
|
anchored on the first message would cut it in half at an arbitrary point.
|
||||||
|
|
||||||
|
## Why this is a post-import sweep and not part of ingest
|
||||||
|
|
||||||
|
The obvious alternative was to migrate Discord to the native post-first
|
||||||
|
ingester (#1266) and group at capture time. **That cannot work**, and the
|
||||||
|
reason is worth recording: the grouping signal is `siglip_embedding`, which is
|
||||||
|
produced ASYNCHRONOUSLY after import (`tasks/ml.py`, the GPU queue backfill).
|
||||||
|
At capture time the embedding does not exist yet, so an ingester has nothing to
|
||||||
|
group on. Grouping is necessarily something that happens once the vectors have
|
||||||
|
caught up — which also means this sweep must be re-runnable and must simply
|
||||||
|
skip what it cannot yet place. It does: a post whose image has no embedding is
|
||||||
|
left alone and picked up on a later run.
|
||||||
|
|
||||||
|
## The honesty rule
|
||||||
|
|
||||||
|
A synthetic post must never pretend an artist authored it. It carries
|
||||||
|
`synthesized_by`, records what it was built from in `synthesis_details`
|
||||||
|
(members, count, and the thresholds in force at the time), and leaves its
|
||||||
|
member posts intact and reachable. Deleting the synthetic post releases the
|
||||||
|
members back into the feed — one DELETE, no repair step. FC invented this
|
||||||
|
grouping; the operator has to be able to see that, inspect it, and undo it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import math
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from sqlalchemy import Select, func, select, update
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from ..models import ImageProvenance, ImageRecord, MLSettings, Post, Source
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# The value that lands in `post.synthesized_by`. One grouper today; a second
|
||||||
|
# would be another value here, which is exactly why the column has no CHECK.
|
||||||
|
DROP_GROUPER = "discord_drop"
|
||||||
|
|
||||||
|
PLATFORM = "discord"
|
||||||
|
|
||||||
|
# Ceiling on member posts examined per source per run. A first sweep over an
|
||||||
|
# established library would otherwise pull every Discord message's 1152-float
|
||||||
|
# vector into memory at once. The sweep is re-runnable and works oldest-first,
|
||||||
|
# so a backlog simply drains over successive runs rather than needing one
|
||||||
|
# heroic pass.
|
||||||
|
MAX_CANDIDATES_PER_SOURCE = 500
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class DropGroup:
|
||||||
|
"""One drop: the member posts, in arrival order, that will become a post."""
|
||||||
|
|
||||||
|
member_ids: list[int] = field(default_factory=list)
|
||||||
|
seed: list[float] | None = None
|
||||||
|
last_at: datetime | None = None
|
||||||
|
|
||||||
|
|
||||||
|
def cosine_distance(a, b) -> float:
|
||||||
|
"""Cosine distance between two embeddings, in the same units pgvector's
|
||||||
|
`cosine_distance` operator returns (0 = identical, 1 = orthogonal).
|
||||||
|
|
||||||
|
Computed in Python rather than SQL because the comparison is against a
|
||||||
|
group seed held in a loop, not against a column — and pure arithmetic keeps
|
||||||
|
numpy off this path entirely. pgvector may hand back a numpy array or a
|
||||||
|
list depending on driver version, so both are coerced.
|
||||||
|
"""
|
||||||
|
va = [float(x) for x in a]
|
||||||
|
vb = [float(x) for x in b]
|
||||||
|
# strict=True: two embeddings of different length is a corrupted row or a
|
||||||
|
# model swap that skipped the re-embed, and silently truncating to the
|
||||||
|
# shorter one would score it as a near match.
|
||||||
|
dot = sum(x * y for x, y in zip(va, vb, strict=True))
|
||||||
|
na = math.sqrt(sum(x * x for x in va))
|
||||||
|
nb = math.sqrt(sum(y * y for y in vb))
|
||||||
|
if na == 0.0 or nb == 0.0:
|
||||||
|
# A zero vector has no direction, so no meaningful distance. Return the
|
||||||
|
# maximum so it can never pull anything into a group.
|
||||||
|
return 1.0
|
||||||
|
return 1.0 - (dot / (na * nb))
|
||||||
|
|
||||||
|
|
||||||
|
def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
||||||
|
"""Ungrouped Discord message-posts, one representative image each, OLDEST
|
||||||
|
FIRST — which is the order `build_groups` requires.
|
||||||
|
|
||||||
|
DISTINCT ON the post picks the lowest-id embedded image as that post's
|
||||||
|
representative: a Discord message carrying several attachments is still one
|
||||||
|
point in the drop, and comparing every attachment would let one incidental
|
||||||
|
image drag an unrelated message into the group.
|
||||||
|
|
||||||
|
The DISTINCT ON is wrapped in a subquery rather than ordered directly,
|
||||||
|
because Postgres requires a DISTINCT ON query's ORDER BY to LEAD with the
|
||||||
|
distinct expression — so the inner query must sort by `post.id`, which is
|
||||||
|
insertion order and not arrival order at all once a backfill has imported
|
||||||
|
anything out of sequence. Sorting outside is what makes the caller's LIMIT
|
||||||
|
take the OLDEST candidates instead of the lowest-numbered ones.
|
||||||
|
"""
|
||||||
|
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||||
|
inner = (
|
||||||
|
select(
|
||||||
|
Post.id.label("post_id"),
|
||||||
|
sort_key.label("occurred_at"),
|
||||||
|
ImageRecord.siglip_embedding.label("embedding"),
|
||||||
|
)
|
||||||
|
.join(ImageRecord, ImageRecord.primary_post_id == Post.id)
|
||||||
|
.where(
|
||||||
|
Post.source_id == source_id,
|
||||||
|
# Never absorb a post FC wrote, and never re-absorb one already
|
||||||
|
# taken — both would build groups out of groups.
|
||||||
|
Post.synthesized_by.is_(None),
|
||||||
|
Post.absorbed_by_post_id.is_(None),
|
||||||
|
ImageRecord.siglip_embedding.is_not(None),
|
||||||
|
sort_key <= not_after,
|
||||||
|
)
|
||||||
|
.distinct(Post.id)
|
||||||
|
.order_by(Post.id, ImageRecord.id)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
select(inner.c.post_id, inner.c.occurred_at, inner.c.embedding)
|
||||||
|
.order_by(inner.c.occurred_at, inner.c.post_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_groups(
|
||||||
|
rows: list[tuple[int, datetime, list[float]]],
|
||||||
|
*,
|
||||||
|
max_distance: float,
|
||||||
|
window: timedelta,
|
||||||
|
) -> list[DropGroup]:
|
||||||
|
"""Walk candidates in arrival order and cut them into drops.
|
||||||
|
|
||||||
|
`rows` must be sorted oldest-first — the whole predicate is about
|
||||||
|
adjacency in time, so an unsorted input would silently produce nonsense
|
||||||
|
rather than fail.
|
||||||
|
"""
|
||||||
|
groups: list[DropGroup] = []
|
||||||
|
current: DropGroup | None = None
|
||||||
|
|
||||||
|
for post_id, occurred_at, embedding in rows:
|
||||||
|
if current is not None:
|
||||||
|
gap_ok = occurred_at - current.last_at <= window
|
||||||
|
# Distance to the SEED, not to the previous member — see the module
|
||||||
|
# docstring on drift.
|
||||||
|
near = cosine_distance(current.seed, embedding) <= max_distance
|
||||||
|
if gap_ok and near:
|
||||||
|
current.member_ids.append(post_id)
|
||||||
|
current.last_at = occurred_at
|
||||||
|
continue
|
||||||
|
groups.append(current)
|
||||||
|
current = DropGroup(
|
||||||
|
member_ids=[post_id], seed=embedding, last_at=occurred_at,
|
||||||
|
)
|
||||||
|
|
||||||
|
if current is not None:
|
||||||
|
groups.append(current)
|
||||||
|
return groups
|
||||||
|
|
||||||
|
|
||||||
|
async def _synthesize(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
source: Source,
|
||||||
|
group: DropGroup,
|
||||||
|
max_distance: float,
|
||||||
|
window_minutes: float,
|
||||||
|
) -> Post | None:
|
||||||
|
"""Write one synthetic post for `group` and absorb its members."""
|
||||||
|
members = (await session.execute(
|
||||||
|
select(Post)
|
||||||
|
.where(Post.id.in_(group.member_ids))
|
||||||
|
.order_by(func.coalesce(Post.post_date, Post.downloaded_at), Post.id)
|
||||||
|
)).scalars().all()
|
||||||
|
if not members:
|
||||||
|
return None
|
||||||
|
|
||||||
|
first = members[0]
|
||||||
|
# Deterministic key, so a re-run cannot mint a second post for the same
|
||||||
|
# drop: the unique (source_id, external_post_id) constraint would reject it
|
||||||
|
# even if the member filter somehow let the drop through twice.
|
||||||
|
external_id = f"fc-drop:{first.external_post_id}"[:128]
|
||||||
|
|
||||||
|
# The messages' own text, in arrival order, IS the post's body — that is
|
||||||
|
# what the operator asked for and it is the only text a drop has. Blank
|
||||||
|
# messages (an attachment with no caption) contribute nothing rather than a
|
||||||
|
# run of empty lines.
|
||||||
|
body = "\n\n".join(m.description.strip() for m in members if m.description and m.description.strip())
|
||||||
|
|
||||||
|
post = Post(
|
||||||
|
source_id=source.id,
|
||||||
|
artist_id=source.artist_id,
|
||||||
|
external_post_id=external_id,
|
||||||
|
# post_title stays NULL DELIBERATELY. A synthesised title is the one
|
||||||
|
# place this feature could accidentally put words in a creator's mouth;
|
||||||
|
# the UI labels the row from `synthesized_by` instead, which cannot be
|
||||||
|
# mistaken for something the artist wrote.
|
||||||
|
post_title=None,
|
||||||
|
post_url=first.post_url,
|
||||||
|
post_date=first.post_date or first.downloaded_at,
|
||||||
|
description=body or None,
|
||||||
|
synthesized_by=DROP_GROUPER,
|
||||||
|
synthesis_details={
|
||||||
|
"member_post_ids": [m.id for m in members],
|
||||||
|
"message_count": len(members),
|
||||||
|
# The thresholds AS THEY WERE. They are operator-tunable, so
|
||||||
|
# without this "why did it group these" is unanswerable later.
|
||||||
|
"max_distance": max_distance,
|
||||||
|
"window_minutes": window_minutes,
|
||||||
|
"grouped_at": datetime.now(UTC).isoformat(),
|
||||||
|
# Growth accumulated since the post last moved in the feed (E3).
|
||||||
|
# Seeded here so the joiner never meets its absence — creation IS
|
||||||
|
# a surfacing, so the count starts at zero.
|
||||||
|
"images_since_surface": 0,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
session.add(post)
|
||||||
|
await session.flush()
|
||||||
|
|
||||||
|
await session.execute(
|
||||||
|
update(Post)
|
||||||
|
.where(Post.id.in_([m.id for m in members]))
|
||||||
|
.values(absorbed_by_post_id=post.id)
|
||||||
|
)
|
||||||
|
|
||||||
|
await _link_member_images(
|
||||||
|
session, post_id=post.id, member_ids=[m.id for m in members],
|
||||||
|
source_id=source.id,
|
||||||
|
)
|
||||||
|
return post
|
||||||
|
|
||||||
|
|
||||||
|
async def _link_member_images(
|
||||||
|
session: AsyncSession, *, post_id: int, member_ids: list[int], source_id: int | None,
|
||||||
|
) -> int:
|
||||||
|
"""Attach every member's images to the synthetic post. Returns how many.
|
||||||
|
|
||||||
|
The feed and detail views already union provenance with `primary_post_id`
|
||||||
|
(post_feed_service._thumbnails_for), so this alone makes the drop's images
|
||||||
|
show up under the post FC wrote — no second render path.
|
||||||
|
|
||||||
|
`primary_post_id` is deliberately NOT rewritten: the message post remains
|
||||||
|
the image's true origin, and the synthetic post is an ADDITIONAL claim on
|
||||||
|
it, which is what keeps the grouping reversible.
|
||||||
|
|
||||||
|
Shared by creation and by the E3 joiner rather than written twice, because
|
||||||
|
the two would otherwise be free to drift on exactly the detail (which post
|
||||||
|
owns the image) that makes a grouping reversible.
|
||||||
|
"""
|
||||||
|
if not member_ids:
|
||||||
|
return 0
|
||||||
|
image_rows = (await session.execute(
|
||||||
|
select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids))
|
||||||
|
)).scalars().all()
|
||||||
|
if not image_rows:
|
||||||
|
return 0
|
||||||
|
await session.execute(
|
||||||
|
pg_insert(ImageProvenance)
|
||||||
|
.values([
|
||||||
|
{"image_record_id": iid, "post_id": post_id, "source_id": source_id}
|
||||||
|
for iid in image_rows
|
||||||
|
])
|
||||||
|
# (image, post) is unique. This is a BACKSTOP against a re-run that
|
||||||
|
# raced itself, not the correctness argument: callers only ever pass
|
||||||
|
# members that were unabsorbed a moment ago, so a conflict here means
|
||||||
|
# concurrency, not a logic error.
|
||||||
|
.on_conflict_do_nothing(constraint="uq_image_provenance_image_post")
|
||||||
|
)
|
||||||
|
return len(image_rows)
|
||||||
|
|
||||||
|
|
||||||
|
async def group_source(
|
||||||
|
session: AsyncSession,
|
||||||
|
source: Source,
|
||||||
|
*,
|
||||||
|
max_distance: float,
|
||||||
|
window_minutes: float,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Group one Discord source's ungrouped messages. Returns posts created."""
|
||||||
|
window = timedelta(minutes=window_minutes)
|
||||||
|
now = now or datetime.now(UTC)
|
||||||
|
# Leave the most recent window alone: a drop that is still arriving would
|
||||||
|
# otherwise be cut in half by whichever sweep happened to land mid-drop,
|
||||||
|
# and the second half would become a separate post claiming to be its own
|
||||||
|
# drop. Waiting one window costs nothing (the sweep re-runs) and is the E2
|
||||||
|
# side of "keep the grouping open"; E3 handles the harder case where a
|
||||||
|
# matching drop resumes after the gap has already passed.
|
||||||
|
rows = (await session.execute(
|
||||||
|
_candidate_stmt(source.id, not_after=now - window)
|
||||||
|
.limit(MAX_CANDIDATES_PER_SOURCE)
|
||||||
|
)).all()
|
||||||
|
if not rows:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
groups = build_groups(
|
||||||
|
[(pid, occurred, emb) for pid, occurred, emb in rows],
|
||||||
|
max_distance=max_distance, window=window,
|
||||||
|
)
|
||||||
|
if len(rows) == MAX_CANDIDATES_PER_SOURCE and len(groups) > 1:
|
||||||
|
# The cap may have fallen INSIDE the last drop, and synthesising a
|
||||||
|
# truncated group would publish a post that claims to be the whole drop
|
||||||
|
# while the rest of it sits one row past the limit. Leave it for the
|
||||||
|
# next run, which starts from the same place and sees the remainder.
|
||||||
|
# Guarded on len > 1 so a single oversized group is not dropped
|
||||||
|
# forever — it would make no progress at all.
|
||||||
|
groups = groups[:-1]
|
||||||
|
|
||||||
|
created = 0
|
||||||
|
for group in groups:
|
||||||
|
post = await _synthesize(
|
||||||
|
session, source=source, group=group,
|
||||||
|
max_distance=max_distance, window_minutes=window_minutes,
|
||||||
|
)
|
||||||
|
if post is not None:
|
||||||
|
created += 1
|
||||||
|
return created
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# E3: an open grouping — a later drop joins its post and updates it.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
#
|
||||||
|
# A synthetic post is not sealed at creation. A creator who adds two more
|
||||||
|
# variants the next day extends the existing post rather than starting a new
|
||||||
|
# one, and its body grows with the new messages. That is what makes chat
|
||||||
|
# capture read as content TRICKLING IN rather than as a stream of separate
|
||||||
|
# arrivals.
|
||||||
|
#
|
||||||
|
# Three hard problems, each answered deliberately below: bridging (a candidate
|
||||||
|
# near two groups), re-surfacing without thrashing the feed, and groups that
|
||||||
|
# stay open forever.
|
||||||
|
|
||||||
|
# How much closer the nearest group must be than the runner-up before a
|
||||||
|
# candidate is assigned to it at all.
|
||||||
|
#
|
||||||
|
# NOT a setting, deliberately. It is not a quality dial the operator would tune
|
||||||
|
# toward a better feed — it expresses "these two are too close to call", and
|
||||||
|
# exposing it would invite turning it to zero, which is precisely the silent
|
||||||
|
# arbitrary choice it exists to prevent. When a candidate is genuinely between
|
||||||
|
# two groups the recoverable answer is to leave it out and let it start its
|
||||||
|
# own; the unrecoverable one is to merge, because a merge rewrites history —
|
||||||
|
# two posts the operator may already have seen become one, and anything
|
||||||
|
# pointing at the absorbed post dangles.
|
||||||
|
AMBIGUITY_MARGIN = 0.02
|
||||||
|
|
||||||
|
|
||||||
|
def should_resurface(
|
||||||
|
*,
|
||||||
|
images_since_surface: int,
|
||||||
|
last_surface_at: datetime,
|
||||||
|
now: datetime,
|
||||||
|
min_images: int,
|
||||||
|
cooldown: timedelta,
|
||||||
|
) -> bool:
|
||||||
|
"""Has this group grown enough, and waited long enough, to move in the feed?
|
||||||
|
|
||||||
|
An updated post SHOULD be visible — that is the point of keeping it open —
|
||||||
|
but a group gaining one image a day must not sit permanently at the top.
|
||||||
|
Both conditions have to hold: enough new images that the update is worth an
|
||||||
|
interruption, and enough time since the last one that a steady drip cannot
|
||||||
|
chain bumps together.
|
||||||
|
"""
|
||||||
|
if images_since_surface < min_images:
|
||||||
|
return False
|
||||||
|
return now - last_surface_at >= cooldown
|
||||||
|
|
||||||
|
|
||||||
|
async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None:
|
||||||
|
"""The embedding a group is measured against — its FIRST member's image.
|
||||||
|
|
||||||
|
Derived rather than stored, and derived by the same definition `build_groups`
|
||||||
|
used (earliest member, lowest-id embedded image). Storing it at creation
|
||||||
|
would have meant a backfill for groups already written and two definitions
|
||||||
|
free to disagree; this way there is one.
|
||||||
|
"""
|
||||||
|
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||||
|
return (await session.execute(
|
||||||
|
select(ImageRecord.siglip_embedding)
|
||||||
|
.join(Post, ImageRecord.primary_post_id == Post.id)
|
||||||
|
.where(
|
||||||
|
Post.absorbed_by_post_id == post_id,
|
||||||
|
ImageRecord.siglip_embedding.is_not(None),
|
||||||
|
)
|
||||||
|
.order_by(sort_key, Post.id, ImageRecord.id)
|
||||||
|
.limit(1)
|
||||||
|
)).scalar_one_or_none()
|
||||||
|
|
||||||
|
|
||||||
|
async def open_groups(
|
||||||
|
session: AsyncSession, source_id: int, *, now: datetime, close_after: timedelta,
|
||||||
|
) -> list[tuple[Post, list[float]]]:
|
||||||
|
"""This source's synthetic posts that are still accepting members.
|
||||||
|
|
||||||
|
Openness is DERIVED, not stored: a group is open if it grew — or started —
|
||||||
|
within `close_after`. A group left open forever would eventually absorb
|
||||||
|
something it shouldn't, because artists reuse characters for years; a
|
||||||
|
stored `closed_at` would need a sweep to set it and a repair path to ever
|
||||||
|
change the policy. This way the policy IS the query.
|
||||||
|
"""
|
||||||
|
cutoff = now - close_after
|
||||||
|
posts = (await session.execute(
|
||||||
|
select(Post).where(
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.synthesized_by == DROP_GROUPER,
|
||||||
|
# A synthetic post that was itself absorbed is not a thing today
|
||||||
|
# (nothing absorbs one), but joining into one would nest groups.
|
||||||
|
Post.absorbed_by_post_id.is_(None),
|
||||||
|
func.coalesce(Post.last_grew_at, Post.post_date, Post.downloaded_at) >= cutoff,
|
||||||
|
)
|
||||||
|
)).scalars().all()
|
||||||
|
|
||||||
|
out: list[tuple[Post, list[float]]] = []
|
||||||
|
for post in posts:
|
||||||
|
seed = await _group_seed(session, post.id)
|
||||||
|
if seed is not None:
|
||||||
|
out.append((post, seed))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def assign_to_group(
|
||||||
|
embedding: list[float],
|
||||||
|
groups: list[tuple[Post, list[float]]],
|
||||||
|
*,
|
||||||
|
max_distance: float,
|
||||||
|
) -> Post | None:
|
||||||
|
"""Pick the one group this image belongs to, or None to leave it alone.
|
||||||
|
|
||||||
|
Returns None in two cases that mean different things and are deliberately
|
||||||
|
treated the same: nothing is close enough (so E2 will start a new group
|
||||||
|
from it), or two groups are BOTH close and too near each other to choose
|
||||||
|
between (so E2 will start a new group from it). The second is the bridging
|
||||||
|
case, and letting it start its own post is the recoverable failure —
|
||||||
|
merging two existing posts is not.
|
||||||
|
"""
|
||||||
|
# key= on the distance ALONE. Sorting bare tuples falls through to the
|
||||||
|
# second element when two distances tie, and `Post` has no ordering — so a
|
||||||
|
# perfectly symmetric bridge (the exact case this function exists for)
|
||||||
|
# would raise TypeError instead of declining to choose.
|
||||||
|
scored = sorted(
|
||||||
|
((cosine_distance(seed, embedding), post) for post, seed in groups),
|
||||||
|
key=lambda pair: pair[0],
|
||||||
|
)
|
||||||
|
within = [(d, p) for d, p in scored if d <= max_distance]
|
||||||
|
if not within:
|
||||||
|
return None
|
||||||
|
if len(within) >= 2 and (within[1][0] - within[0][0]) < AMBIGUITY_MARGIN:
|
||||||
|
return None
|
||||||
|
return within[0][1]
|
||||||
|
|
||||||
|
|
||||||
|
async def _absorb_into(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
group: Post,
|
||||||
|
member_ids: list[int],
|
||||||
|
source_id: int | None,
|
||||||
|
now: datetime,
|
||||||
|
min_images: int,
|
||||||
|
cooldown: timedelta,
|
||||||
|
) -> int:
|
||||||
|
"""Extend an existing synthetic post with new members. Returns images added."""
|
||||||
|
members = (await session.execute(
|
||||||
|
select(Post)
|
||||||
|
.where(Post.id.in_(member_ids))
|
||||||
|
.order_by(func.coalesce(Post.post_date, Post.downloaded_at), Post.id)
|
||||||
|
)).scalars().all()
|
||||||
|
if not members:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
added_images = await _link_member_images(
|
||||||
|
session, post_id=group.id, member_ids=[m.id for m in members],
|
||||||
|
source_id=source_id,
|
||||||
|
)
|
||||||
|
await session.execute(
|
||||||
|
update(Post)
|
||||||
|
.where(Post.id.in_([m.id for m in members]))
|
||||||
|
.values(absorbed_by_post_id=group.id)
|
||||||
|
)
|
||||||
|
|
||||||
|
# The new messages' text joins the body, in arrival order, exactly as at
|
||||||
|
# creation — the post's body is the drop's text and the drop just grew.
|
||||||
|
new_text = "\n\n".join(
|
||||||
|
m.description.strip() for m in members if m.description and m.description.strip()
|
||||||
|
)
|
||||||
|
if new_text:
|
||||||
|
group.description = f"{group.description}\n\n{new_text}" if group.description else new_text
|
||||||
|
|
||||||
|
details = dict(group.synthesis_details or {})
|
||||||
|
existing_ids = list(details.get("member_post_ids") or [])
|
||||||
|
details["member_post_ids"] = existing_ids + [
|
||||||
|
m.id for m in members if m.id not in existing_ids
|
||||||
|
]
|
||||||
|
details["message_count"] = len(details["member_post_ids"])
|
||||||
|
since = int(details.get("images_since_surface") or 0) + added_images
|
||||||
|
details["last_grew_at"] = now.isoformat()
|
||||||
|
|
||||||
|
# The feed position moves only when the anti-thrash rule fires. Measured
|
||||||
|
# from the last time the post actually MOVED (resurfaced_at), falling back
|
||||||
|
# to when the drop started — creation is itself a surfacing.
|
||||||
|
last_surface = group.resurfaced_at or group.post_date or group.downloaded_at
|
||||||
|
if should_resurface(
|
||||||
|
images_since_surface=since, last_surface_at=last_surface, now=now,
|
||||||
|
min_images=min_images, cooldown=cooldown,
|
||||||
|
):
|
||||||
|
group.resurfaced_at = now
|
||||||
|
since = 0
|
||||||
|
details["images_since_surface"] = since
|
||||||
|
|
||||||
|
group.synthesis_details = details
|
||||||
|
group.last_grew_at = now
|
||||||
|
return added_images
|
||||||
|
|
||||||
|
|
||||||
|
async def join_open_groups(
|
||||||
|
session: AsyncSession,
|
||||||
|
source: Source,
|
||||||
|
*,
|
||||||
|
max_distance: float,
|
||||||
|
window_minutes: float,
|
||||||
|
close_after_hours: float,
|
||||||
|
resurface_min_images: int,
|
||||||
|
resurface_cooldown_hours: float,
|
||||||
|
now: datetime | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Offer this source's ungrouped messages to its open groups.
|
||||||
|
|
||||||
|
Runs BEFORE `group_source` in the sweep: a message that belongs to an
|
||||||
|
existing drop must join it rather than found a rival post, and whichever
|
||||||
|
runs first wins that message.
|
||||||
|
"""
|
||||||
|
now = now or datetime.now(UTC)
|
||||||
|
window = timedelta(minutes=window_minutes)
|
||||||
|
groups = await open_groups(
|
||||||
|
session, source.id, now=now, close_after=timedelta(hours=close_after_hours),
|
||||||
|
)
|
||||||
|
if not groups:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Same quarantine as E2: a drop still arriving is left for the next run.
|
||||||
|
rows = (await session.execute(
|
||||||
|
_candidate_stmt(source.id, not_after=now - window)
|
||||||
|
.limit(MAX_CANDIDATES_PER_SOURCE)
|
||||||
|
)).all()
|
||||||
|
if not rows:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
claimed: dict[int, list[int]] = {}
|
||||||
|
for post_id, _occurred_at, embedding in rows:
|
||||||
|
target = assign_to_group(embedding, groups, max_distance=max_distance)
|
||||||
|
if target is not None:
|
||||||
|
claimed.setdefault(target.id, []).append(post_id)
|
||||||
|
|
||||||
|
by_id = {post.id: post for post, _seed in groups}
|
||||||
|
joined = 0
|
||||||
|
for group_id, member_ids in claimed.items():
|
||||||
|
joined += await _absorb_into(
|
||||||
|
session, group=by_id[group_id], member_ids=member_ids,
|
||||||
|
source_id=source.id, now=now,
|
||||||
|
min_images=resurface_min_images,
|
||||||
|
cooldown=timedelta(hours=resurface_cooldown_hours),
|
||||||
|
)
|
||||||
|
return joined
|
||||||
|
|
||||||
|
|
||||||
|
async def sweep(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||||
|
"""Group every enabled Discord source. No-op when the switch is off.
|
||||||
|
|
||||||
|
Two passes per source, and the ORDER is load-bearing: offer new messages to
|
||||||
|
the groups that are still open (E3) BEFORE founding new ones (E2). Whichever
|
||||||
|
runs first claims a message, and a variant that belongs to yesterday's drop
|
||||||
|
must extend that post rather than found a rival to it.
|
||||||
|
"""
|
||||||
|
settings = await MLSettings.load(session)
|
||||||
|
if not settings.discord_grouping_enabled:
|
||||||
|
return {
|
||||||
|
"enabled": False, "sources": 0, "posts_created": 0, "images_joined": 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
sources = (await session.execute(
|
||||||
|
select(Source).where(
|
||||||
|
Source.platform == PLATFORM,
|
||||||
|
Source.enabled.is_(True),
|
||||||
|
)
|
||||||
|
)).scalars().all()
|
||||||
|
|
||||||
|
max_distance = float(settings.discord_group_max_distance)
|
||||||
|
window_minutes = float(settings.discord_group_window_minutes)
|
||||||
|
|
||||||
|
created = 0
|
||||||
|
joined = 0
|
||||||
|
for source in sources:
|
||||||
|
joined += await join_open_groups(
|
||||||
|
session, source,
|
||||||
|
max_distance=max_distance,
|
||||||
|
window_minutes=window_minutes,
|
||||||
|
close_after_hours=float(settings.discord_group_close_after_hours),
|
||||||
|
resurface_min_images=int(settings.discord_group_resurface_min_images),
|
||||||
|
resurface_cooldown_hours=float(
|
||||||
|
settings.discord_group_resurface_cooldown_hours
|
||||||
|
),
|
||||||
|
now=now,
|
||||||
|
)
|
||||||
|
created += await group_source(
|
||||||
|
session, source,
|
||||||
|
max_distance=max_distance,
|
||||||
|
window_minutes=window_minutes,
|
||||||
|
now=now,
|
||||||
|
)
|
||||||
|
log.info(
|
||||||
|
"discord drop grouping: %d source(s), %d synthetic post(s) created, "
|
||||||
|
"%d image(s) joined to open groups",
|
||||||
|
len(sources), created, joined,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"enabled": True, "sources": len(sources),
|
||||||
|
"posts_created": created, "images_joined": joined,
|
||||||
|
}
|
||||||
@@ -34,7 +34,9 @@ from .gallery_dl import (
|
|||||||
GalleryDLService,
|
GalleryDLService,
|
||||||
SourceConfig,
|
SourceConfig,
|
||||||
extract_errors_warnings,
|
extract_errors_warnings,
|
||||||
|
is_informational,
|
||||||
truncate_log,
|
truncate_log,
|
||||||
|
walk_completed,
|
||||||
)
|
)
|
||||||
from .importer import Importer
|
from .importer import Importer
|
||||||
from .platforms import auth_type_for
|
from .platforms import auth_type_for
|
||||||
@@ -552,11 +554,13 @@ class DownloadService:
|
|||||||
# page is no longer double-counted. new_overrides (read fresh above)
|
# page is no longer double-counted. new_overrides (read fresh above)
|
||||||
# carries the ingester's committed value forward untouched.
|
# carries the ingester's committed value forward untouched.
|
||||||
|
|
||||||
completed = (
|
# Shared with the result path so the two halves can't disagree about
|
||||||
dl_result.success
|
# what "finished" means. Note it admits an INFORMATIONAL error_type: a
|
||||||
and dl_result.error_type is None
|
# fully-paywalled creator's backfill really did reach the bottom, and
|
||||||
and dl_result.return_code == 0
|
# treating it as unfinished would re-walk that wall every chunk until
|
||||||
)
|
# the stall counter tripped — the creator we can see least becoming the
|
||||||
|
# one we fetch most.
|
||||||
|
completed = walk_completed(dl_result)
|
||||||
if completed:
|
if completed:
|
||||||
new_overrides["_backfill_state"] = "complete"
|
new_overrides["_backfill_state"] = "complete"
|
||||||
new_overrides.pop("_backfill_cursor", None)
|
new_overrides.pop("_backfill_cursor", None)
|
||||||
@@ -625,8 +629,17 @@ class DownloadService:
|
|||||||
if status == "ok":
|
if status == "ok":
|
||||||
source.consecutive_failures = 0
|
source.consecutive_failures = 0
|
||||||
source.last_error = None
|
source.last_error = None
|
||||||
# alembic 0032 — clear the failure-class chip on success.
|
# alembic 0032 — clear the failure-class chip on success, EXCEPT an
|
||||||
source.error_type = None
|
# informational class. tier_limited rides an otherwise-successful
|
||||||
|
# run: failures stay 0 and last_error stays clear (the run did not
|
||||||
|
# fail and must not earn a backoff), but "there is content here we
|
||||||
|
# aren't allowed to see" is a durable fact about the SOURCE, not
|
||||||
|
# about this run. Clearing it here is what left FailingSourcesCard's
|
||||||
|
# `tier_limited` palette entry unreachable — the chip was wiped by
|
||||||
|
# the very success that produced it.
|
||||||
|
source.error_type = (
|
||||||
|
error_type if is_informational(error_type) else None
|
||||||
|
)
|
||||||
elif status == "error":
|
elif status == "error":
|
||||||
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
source.consecutive_failures = (source.consecutive_failures or 0) + 1
|
||||||
source.last_error = error_message
|
source.last_error = error_message
|
||||||
|
|||||||
@@ -261,6 +261,72 @@ def make_run_stats(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
# --- tier-gated classification, shared by BOTH backends ---------------------
|
||||||
|
#
|
||||||
|
# These three live together because the native ingester and the gallery-dl
|
||||||
|
# subprocess must reach the same verdict from the same number. They did not:
|
||||||
|
# gallery-dl classified TIER_LIMITED while ingest_core counted gated posts and
|
||||||
|
# threw the count away, so the platforms FC owns reported a paywalled creator as
|
||||||
|
# a silent one (#874 follow-up). One predicate, spread into both, rather than
|
||||||
|
# the condition re-derived per backend.
|
||||||
|
|
||||||
|
|
||||||
|
def classify_tier_gated(tier_gated_count: int) -> ErrorType | None:
|
||||||
|
"""TIER_LIMITED when a walk saw tier-gated posts and nothing else failed.
|
||||||
|
|
||||||
|
Deliberately NOT conditioned on `downloaded == 0`. A creator whose top-tier
|
||||||
|
posts we cannot see is tier-limited even in a week we did get their cheaper
|
||||||
|
ones — the fact the operator needs ("there is content here you are not
|
||||||
|
paying for") is true either way. gallery-dl has classified it this way since
|
||||||
|
the paywall-as-"needs attention" complaint (see `_categorize_error`), and the
|
||||||
|
native path now matches rather than inventing a stricter rule.
|
||||||
|
|
||||||
|
Callers must apply this only AFTER the real error categories (auth, rate
|
||||||
|
limit, drift, …) have had their turn; tier-gating is the weakest signal and
|
||||||
|
must never mask a genuine failure.
|
||||||
|
"""
|
||||||
|
return ErrorType.TIER_LIMITED if tier_gated_count else None
|
||||||
|
|
||||||
|
|
||||||
|
def tier_gated_message(count: int) -> str:
|
||||||
|
"""The one wording for the tier-gated verdict, so the two backends can't
|
||||||
|
describe the same state differently in the Logs UI."""
|
||||||
|
return (
|
||||||
|
f"Subscription tier does not grant access to "
|
||||||
|
f"{count} post{'s' if count != 1 else ''}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# `Source.error_type` doubles as the failure-class chip, and a status of "ok"
|
||||||
|
# CLEARS it (alembic 0032). TIER_LIMITED breaks that assumption: it rides an
|
||||||
|
# otherwise-successful run, so without an exemption the chip is wiped the moment
|
||||||
|
# it is set and `FailingSourcesCard`'s `tier_limited` palette entry can never
|
||||||
|
# render. Informational classes are the exemption — they describe the source,
|
||||||
|
# not a failure of the run.
|
||||||
|
INFORMATIONAL_ERROR_TYPES = frozenset({ErrorType.TIER_LIMITED.value})
|
||||||
|
|
||||||
|
|
||||||
|
def is_informational(error_type) -> bool:
|
||||||
|
"""True for a class that reports a state rather than a failure. Accepts an
|
||||||
|
ErrorType or the plain string persisted on Source.error_type."""
|
||||||
|
return error_type is not None and str(error_type) in INFORMATIONAL_ERROR_TYPES
|
||||||
|
|
||||||
|
|
||||||
|
def walk_completed(result: DownloadResult) -> bool:
|
||||||
|
"""Did this walk reach the bottom cleanly?
|
||||||
|
|
||||||
|
The backfill lifecycle's completion test. An informational error_type still
|
||||||
|
counts as complete: a fully-paywalled creator's backfill DID finish, and
|
||||||
|
treating it as unfinished re-walks the same wall until the stall counter
|
||||||
|
trips — the creator we can see least becoming the one we fetch most.
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
result.success
|
||||||
|
and result.return_code == 0
|
||||||
|
and (result.error_type is None or is_informational(result.error_type))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class GalleryDLService:
|
class GalleryDLService:
|
||||||
"""Service for executing gallery-dl downloads."""
|
"""Service for executing gallery-dl downloads."""
|
||||||
|
|
||||||
@@ -531,12 +597,12 @@ class GalleryDLService:
|
|||||||
line for line in combined.split("\n")
|
line for line in combined.split("\n")
|
||||||
if "][warning]" in line and "not allowed to view post" in line
|
if "][warning]" in line and "not allowed to view post" in line
|
||||||
]
|
]
|
||||||
if tier_gated_lines:
|
# Same predicate + wording the native path uses, so the two backends
|
||||||
|
# can't drift on what counts as tier-gated or how it reads.
|
||||||
count = len(tier_gated_lines)
|
count = len(tier_gated_lines)
|
||||||
return (
|
gated = classify_tier_gated(count)
|
||||||
ErrorType.TIER_LIMITED,
|
if gated is not None:
|
||||||
f"Subscription tier does not grant access to {count} post{'s' if count != 1 else ''}",
|
return (gated, tier_gated_message(count))
|
||||||
)
|
|
||||||
|
|
||||||
# Partial-success: the subprocess exited non-zero (typically because
|
# Partial-success: the subprocess exited non-zero (typically because
|
||||||
# the wall-clock timeout fired mid-walk), but it had downloaded ≥1
|
# the wall-clock timeout fired mid-walk), but it had downloaded ≥1
|
||||||
|
|||||||
@@ -35,7 +35,13 @@ from collections.abc import Callable
|
|||||||
from sqlalchemy import delete, func, select, text
|
from sqlalchemy import delete, func, select, text
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
|
||||||
from .gallery_dl import DownloadResult, ErrorType, make_run_stats
|
from .gallery_dl import (
|
||||||
|
DownloadResult,
|
||||||
|
ErrorType,
|
||||||
|
classify_tier_gated,
|
||||||
|
make_run_stats,
|
||||||
|
tier_gated_message,
|
||||||
|
)
|
||||||
from .native_ingest_common import NativeAuthError, NativeDriftError
|
from .native_ingest_common import NativeAuthError, NativeDriftError
|
||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
@@ -245,6 +251,12 @@ class Ingester:
|
|||||||
per_item_failures=errors,
|
per_item_failures=errors,
|
||||||
quarantined_count=quarantined,
|
quarantined_count=quarantined,
|
||||||
dead_lettered_count=dead_lettered,
|
dead_lettered_count=dead_lettered,
|
||||||
|
# #874 follow-up: the native path counted gated posts but
|
||||||
|
# never reported them, so DownloadDetailModal's "Tier-gated"
|
||||||
|
# field read 0 on every native walk while gallery-dl's read
|
||||||
|
# true. A paywalled creator was indistinguishable from a
|
||||||
|
# silent one.
|
||||||
|
tier_gated_count=gated_skipped,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -464,6 +476,10 @@ class Ingester:
|
|||||||
"errors": errors,
|
"errors": errors,
|
||||||
"quarantined": quarantined,
|
"quarantined": quarantined,
|
||||||
"posts": posts_processed,
|
"posts": posts_processed,
|
||||||
|
# Ticks during the walk, not only at finalization: a
|
||||||
|
# deep backfill on a creator we've lost access to is
|
||||||
|
# otherwise a long run of zeros with no explanation.
|
||||||
|
"gated": gated_skipped,
|
||||||
})
|
})
|
||||||
|
|
||||||
if early_out:
|
if early_out:
|
||||||
@@ -564,17 +580,33 @@ class Ingester:
|
|||||||
error_type=ErrorType.API_DRIFT, error_message=msg,
|
error_type=ErrorType.API_DRIFT, error_message=msg,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Normal success: reached the bottom, or a tick that early-outed. rc 0 +
|
# Normal success: reached the bottom, or a tick that early-outed. A
|
||||||
# error_type None is REQUIRED for a backfill/recovery walk that reached
|
# zero-download walk still returns success here — a re-confirming walk
|
||||||
# the bottom to be marked COMPLETE by
|
# that found nothing new genuinely completed. A tick that early-outed
|
||||||
# download_service._apply_backfill_lifecycle — so we return None even
|
# also lands here; ticks never set backfill state so the lifecycle is a
|
||||||
# when downloaded == 0 (a re-confirming walk that found nothing new still
|
# no-op for them.
|
||||||
# completed). success=True maps to status "ok" regardless. A tick that
|
#
|
||||||
# early-outed also returns here; ticks never set backfill state so the
|
# success=True and return_code=0 are load-bearing, not cosmetic. They
|
||||||
# lifecycle is a no-op for them.
|
# are what make this a COMPLETE walk for
|
||||||
|
# download_service._apply_backfill_lifecycle (via walk_completed) and
|
||||||
|
# what map it to status "ok", so a walk that fetched nothing doesn't
|
||||||
|
# accrue consecutive_failures or a backoff it hasn't earned.
|
||||||
|
#
|
||||||
|
# #874 follow-up: "nothing new" and "everything sat behind a tier you
|
||||||
|
# don't hold" are different facts, and returning None for both made a
|
||||||
|
# paywalled creator indistinguishable from a silent one. TIER_LIMITED is
|
||||||
|
# classified LAST — every real failure has already returned above —
|
||||||
|
# because tier-gating is the weakest signal and must never mask a
|
||||||
|
# genuine error. It is informational, so walk_completed still counts
|
||||||
|
# this walk as finished (see that predicate for why re-walking a
|
||||||
|
# paywalled creator forever is the bug being avoided).
|
||||||
|
gated_error = classify_tier_gated(gated_skipped)
|
||||||
return _result(
|
return _result(
|
||||||
success=True, return_code=0,
|
success=True, return_code=0,
|
||||||
error_type=None, error_message=None,
|
error_type=gated_error,
|
||||||
|
error_message=(
|
||||||
|
tier_gated_message(gated_skipped) if gated_error else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
# -- failure mapping (adapter overrides) -------------------------------
|
# -- failure mapping (adapter overrides) -------------------------------
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
"""The learned membership roster: what the account actually subscribes to.
|
||||||
|
|
||||||
|
Milestone 387, phase C. Sibling of `service_roster` (milestone 365) and built
|
||||||
|
on the same insight — an absence is only observable against a record of
|
||||||
|
presence. There, a stopped worker; here, a subscription that lapsed.
|
||||||
|
|
||||||
|
## Nothing calls this yet
|
||||||
|
|
||||||
|
`touch_membership` is written before its caller because the caller (the sweep,
|
||||||
|
C3) needs a client seam (C2) that needs Patreon's real response characterised
|
||||||
|
from a captured sample (C0), and that capture needs the operator's browser
|
||||||
|
session. The write side does not depend on any of it: an upsert keyed on
|
||||||
|
(platform, external_campaign_id) is the same regardless of what the payload
|
||||||
|
turns out to look like, and `details` carries whatever C0 finds.
|
||||||
|
|
||||||
|
## Why the whitelist lives here and not in the column
|
||||||
|
|
||||||
|
`platform_membership.status` is an unconstrained String holding the PLATFORM's
|
||||||
|
own word — `active_patron`, not some normalised FC value. The mapping from
|
||||||
|
those words to FC's meaning is a read-site concern and belongs in code that can
|
||||||
|
be corrected without a migration, because the vocabulary comes from whatever
|
||||||
|
each platform says and will be discovered per platform rather than designed up
|
||||||
|
front. `MEMBERSHIP_STATUS` below is a place for that knowledge to accumulate as
|
||||||
|
platforms are characterised; it is deliberately empty of guesses today.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from sqlalchemy import func
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from ..models import PlatformMembership
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Platform word -> whether the account currently has paid access.
|
||||||
|
#
|
||||||
|
# EMPTY ON PURPOSE. Every entry here must come from a characterised response
|
||||||
|
# (step C0), not from what the API docs or a plausible guess suggest — that is
|
||||||
|
# the whole point of project rule 130, and inventing `active_patron` before
|
||||||
|
# seeing it in a real payload is exactly the failure it names. Populate per
|
||||||
|
# platform as each is characterised.
|
||||||
|
#
|
||||||
|
# Unknown words are NOT an error: an unrecognised status means the roster
|
||||||
|
# records evidence it cannot yet interpret, which is a better state than
|
||||||
|
# dropping the row or asserting a meaning for it.
|
||||||
|
MEMBERSHIP_STATUS: dict[str, dict[str, bool]] = {}
|
||||||
|
|
||||||
|
|
||||||
|
def has_paid_access(platform: str, status: str | None) -> bool | None:
|
||||||
|
"""Does this status mean the account currently pays for access?
|
||||||
|
|
||||||
|
Returns None for a status this code has not been taught, which callers must
|
||||||
|
treat as "unknown" rather than as False. The difference matters: False says
|
||||||
|
the operator has lost access, and asserting that from an unrecognised word
|
||||||
|
would tell them to cancel a source they are still paying for.
|
||||||
|
"""
|
||||||
|
if status is None:
|
||||||
|
return None
|
||||||
|
entry = MEMBERSHIP_STATUS.get(platform, {})
|
||||||
|
return entry.get(status)
|
||||||
|
|
||||||
|
|
||||||
|
async def touch_membership(
|
||||||
|
session: AsyncSession,
|
||||||
|
*,
|
||||||
|
platform: str,
|
||||||
|
external_campaign_id: str,
|
||||||
|
display_name: str | None = None,
|
||||||
|
url: str | None = None,
|
||||||
|
status: str | None = None,
|
||||||
|
tier_names: list | None = None,
|
||||||
|
amount_cents: int | None = None,
|
||||||
|
currency: str | None = None,
|
||||||
|
details: dict | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Record that this membership was observed just now.
|
||||||
|
|
||||||
|
Upsert rather than read-modify-write, for the same reason as
|
||||||
|
`service_roster.touch_service`: a sweep may overlap its own previous run,
|
||||||
|
and the last writer is simply the most recent sighting.
|
||||||
|
|
||||||
|
`first_seen_at` is deliberately NOT in the update set. It is the one field
|
||||||
|
that answers "has this ever been true", which is what makes a membership's
|
||||||
|
later DISAPPEARANCE readable as a lapse rather than indistinguishable from
|
||||||
|
a creator FC never knew about. Every other column is last-writer-wins,
|
||||||
|
including status — a membership that goes from active to former must move.
|
||||||
|
"""
|
||||||
|
stmt = pg_insert(PlatformMembership).values(
|
||||||
|
platform=platform,
|
||||||
|
external_campaign_id=external_campaign_id,
|
||||||
|
display_name=display_name,
|
||||||
|
url=url,
|
||||||
|
status=status,
|
||||||
|
tier_names=tier_names,
|
||||||
|
amount_cents=amount_cents,
|
||||||
|
currency=currency,
|
||||||
|
details=details or {},
|
||||||
|
)
|
||||||
|
stmt = stmt.on_conflict_do_update(
|
||||||
|
constraint="uq_platform_membership_platform_campaign",
|
||||||
|
set_={
|
||||||
|
"display_name": stmt.excluded.display_name,
|
||||||
|
"url": stmt.excluded.url,
|
||||||
|
"status": stmt.excluded.status,
|
||||||
|
"tier_names": stmt.excluded.tier_names,
|
||||||
|
"amount_cents": stmt.excluded.amount_cents,
|
||||||
|
"currency": stmt.excluded.currency,
|
||||||
|
"details": stmt.excluded.details,
|
||||||
|
"last_seen_at": func.now(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await session.execute(stmt)
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
"""The announcement matcher: which Patreon post announced which Discord drop.
|
||||||
|
|
||||||
|
Milestone 388, step E5.
|
||||||
|
|
||||||
|
Two of the operator's artists post a deliberately CROPPED fragment on Patreon
|
||||||
|
to signal that the real thing has landed in their Discord. This service
|
||||||
|
proposes those pairs, and proposes only — the operator accepts or dismisses,
|
||||||
|
following the FC-6.3 series matcher (task 737) rather than linking on its own.
|
||||||
|
|
||||||
|
## Why confirm-only is not caution for its own sake
|
||||||
|
|
||||||
|
A wrongly-asserted association tells the operator that two different pieces are
|
||||||
|
one. That is strictly worse than no link at all: no link leaves them exactly
|
||||||
|
where they already were, a wrong one actively misinforms and then propagates
|
||||||
|
into whatever reads the association. So the matcher's job is to make a SHORT
|
||||||
|
list worth reading, not a long list worth trusting.
|
||||||
|
|
||||||
|
## Signals, and the one deliberately NOT built
|
||||||
|
|
||||||
|
1. **Time proximity.** The Patreon post exists in order to announce the drop,
|
||||||
|
so the two are minutes-to-hours apart. Nearly free, and strong.
|
||||||
|
2. **The post says so.** These announcements routinely name Discord or carry
|
||||||
|
an invite link, which is close to a declaration.
|
||||||
|
|
||||||
|
3. **Crop-to-source matching is HELD, on the plan's own instruction** — it is
|
||||||
|
real work with real false-positive risk, and it is only worth building once
|
||||||
|
1 and 2 are shown to be insufficient against the operator's actual artists.
|
||||||
|
Nothing here should be read as evidence it is unnecessary; it is deferred,
|
||||||
|
and the thing that would justify it is an empty review queue on a pair the
|
||||||
|
operator can see with their own eyes.
|
||||||
|
|
||||||
|
Note also that a naive whole-image SigLIP similarity is NOT that signal. A
|
||||||
|
cropped teaser and its full version are exactly the pair a whole-image
|
||||||
|
comparison handles worst, so adding one as a "bonus" would mostly add noise
|
||||||
|
while looking like progress.
|
||||||
|
|
||||||
|
## Creator identity comes free, so E4 is not actually a prerequisite
|
||||||
|
|
||||||
|
The plan listed E4 (creator identity across the two channels) as a dependency.
|
||||||
|
It is not one for the pairs that matter today: a Patreon `Source` and a Discord
|
||||||
|
`Source` the operator has added under the same `Artist` already share
|
||||||
|
`Post.artist_id`, and the synthetic grouping inherits it (E2). E4 EXTENDS this
|
||||||
|
to creators whose association FC has to learn rather than being told; it is not
|
||||||
|
needed to represent an association FC already knows.
|
||||||
|
|
||||||
|
## A correction the plan carried, worth restating
|
||||||
|
|
||||||
|
`link_extract.py` does exist and does capture off-platform links — but only
|
||||||
|
file hosts (`SUPPORTED_HOSTS` is mega/gdrive/mediafire/dropbox/pixeldrain).
|
||||||
|
`host_for()` returns None for a Discord URL, so no `ExternalLink` row is ever
|
||||||
|
written for one. The declaration signal therefore reads the post body itself
|
||||||
|
rather than the extracted-links table the plan assumed it could use.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from sqlalchemy import func, or_, select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from ..models import ImportSettings, Post, PostAssociation
|
||||||
|
from ..utils.text import html_to_plain
|
||||||
|
from .discord_grouping import DROP_GROUPER
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Additive weights, summing to 1.0. Kept as constants rather than settings —
|
||||||
|
# the sensitivity knob that matters is the threshold, and per-signal weights
|
||||||
|
# are an over-tune (same call as series_match_service.WEIGHTS).
|
||||||
|
#
|
||||||
|
# THE RELATIONSHIP TO THE THRESHOLD IS THE DESIGN. No single weight may reach
|
||||||
|
# the default threshold, which is what makes "time proximity alone is never
|
||||||
|
# enough" arithmetic rather than aspirational: on a busy day an artist posts
|
||||||
|
# several times, and a matcher that could pair on proximity alone would turn
|
||||||
|
# every busy day into false pairs. A guard test pins this.
|
||||||
|
WEIGHTS = {"proximity": 0.55, "declared": 0.45}
|
||||||
|
|
||||||
|
# A Discord INVITE in the body is close to a declaration; the bare word is
|
||||||
|
# weaker but still meaningful, because these posts are short and on-topic.
|
||||||
|
_INVITE = re.compile(r"discord\.(?:gg|com/invite)/", re.I)
|
||||||
|
_MENTION = re.compile(r"\bdiscord\b", re.I)
|
||||||
|
|
||||||
|
DECLARED_INVITE = 1.0
|
||||||
|
DECLARED_MENTION = 0.6
|
||||||
|
|
||||||
|
MAX_CANDIDATES = 25
|
||||||
|
|
||||||
|
|
||||||
|
def proximity_signal(gap: timedelta, window: timedelta) -> float:
|
||||||
|
"""1.0 when the two posts are simultaneous, decaying linearly to 0 at the
|
||||||
|
window's edge. Linear rather than a step, so a pair an hour outside a
|
||||||
|
hand-tuned window degrades instead of vanishing."""
|
||||||
|
if window <= timedelta(0):
|
||||||
|
return 0.0
|
||||||
|
seconds = abs(gap.total_seconds())
|
||||||
|
if seconds >= window.total_seconds():
|
||||||
|
return 0.0
|
||||||
|
return round(1.0 - (seconds / window.total_seconds()), 4)
|
||||||
|
|
||||||
|
|
||||||
|
def declared_signal(description: str | None) -> float:
|
||||||
|
"""Does the announcement say, in its own body, that this is about Discord?
|
||||||
|
|
||||||
|
The invite is matched against the RAW body and the bare mention against the
|
||||||
|
stripped text, which is not fussiness — post bodies are HTML, and these
|
||||||
|
creators put the invite in an anchor's `href`. `html_to_plain` discards
|
||||||
|
attributes, so stripping first would have thrown away the strongest form of
|
||||||
|
the signal and left only whatever the link text happened to say.
|
||||||
|
|
||||||
|
The mention still reads stripped text, so `\bdiscord\b` is matched against
|
||||||
|
prose rather than against markup and URLs, where it would fire on any link
|
||||||
|
that merely passes through a discord domain.
|
||||||
|
"""
|
||||||
|
if not description:
|
||||||
|
return 0.0
|
||||||
|
if _INVITE.search(description):
|
||||||
|
return DECLARED_INVITE
|
||||||
|
if _MENTION.search(html_to_plain(description) or ""):
|
||||||
|
return DECLARED_MENTION
|
||||||
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def weighted_score(signals: dict) -> float:
|
||||||
|
return round(sum(WEIGHTS[k] * signals.get(k, 0.0) for k in WEIGHTS), 4)
|
||||||
|
|
||||||
|
|
||||||
|
def _post_time(post: Post) -> datetime:
|
||||||
|
return post.post_date or post.downloaded_at
|
||||||
|
|
||||||
|
|
||||||
|
class PostAssociationService:
|
||||||
|
def __init__(self, session: AsyncSession):
|
||||||
|
self.session = session
|
||||||
|
|
||||||
|
async def _decided(self, announcement_id: int) -> set[int]:
|
||||||
|
"""Payload posts already proposed for this announcement, in ANY status.
|
||||||
|
|
||||||
|
Dismissed pairs are included deliberately: re-proposing a pair the
|
||||||
|
operator has already rejected on every subsequent scan is the single
|
||||||
|
behaviour that makes a review queue get ignored.
|
||||||
|
"""
|
||||||
|
rows = (await self.session.execute(
|
||||||
|
select(PostAssociation.payload_post_id)
|
||||||
|
.where(PostAssociation.announcement_post_id == announcement_id)
|
||||||
|
)).scalars().all()
|
||||||
|
return set(rows)
|
||||||
|
|
||||||
|
async def _candidate_groups(
|
||||||
|
self, announcement: Post, *, window: timedelta,
|
||||||
|
) -> list[Post]:
|
||||||
|
"""Synthetic Discord groupings by the SAME artist, inside the window.
|
||||||
|
|
||||||
|
Same-artist is the identity signal and it is free (see the module
|
||||||
|
docstring on E4). It is also a hard filter rather than a scored one:
|
||||||
|
two different creators posting minutes apart is a coincidence, not
|
||||||
|
evidence, and letting it score at all would mean a busy hour across the
|
||||||
|
library could out-vote everything else.
|
||||||
|
"""
|
||||||
|
at = _post_time(announcement)
|
||||||
|
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||||
|
return (await self.session.execute(
|
||||||
|
select(Post)
|
||||||
|
.where(
|
||||||
|
Post.artist_id == announcement.artist_id,
|
||||||
|
Post.synthesized_by == DROP_GROUPER,
|
||||||
|
Post.id != announcement.id,
|
||||||
|
sort_key >= at - window,
|
||||||
|
sort_key <= at + window,
|
||||||
|
)
|
||||||
|
.order_by(sort_key)
|
||||||
|
.limit(MAX_CANDIDATES)
|
||||||
|
)).scalars().all()
|
||||||
|
|
||||||
|
async def match_post(
|
||||||
|
self, announcement_id: int, *, threshold: float, window_hours: float,
|
||||||
|
) -> int:
|
||||||
|
"""Score one announcement against nearby groupings. Returns proposals made."""
|
||||||
|
announcement = await self.session.get(Post, announcement_id)
|
||||||
|
if announcement is None or announcement.synthesized_by is not None:
|
||||||
|
# A synthetic post cannot announce anything — FC wrote it.
|
||||||
|
return 0
|
||||||
|
|
||||||
|
window = timedelta(hours=window_hours)
|
||||||
|
declared = declared_signal(announcement.description)
|
||||||
|
already = await self._decided(announcement_id)
|
||||||
|
|
||||||
|
made = 0
|
||||||
|
for group in await self._candidate_groups(announcement, window=window):
|
||||||
|
if group.id in already:
|
||||||
|
continue
|
||||||
|
signals = {
|
||||||
|
"proximity": proximity_signal(
|
||||||
|
_post_time(group) - _post_time(announcement), window,
|
||||||
|
),
|
||||||
|
"declared": declared,
|
||||||
|
}
|
||||||
|
score = weighted_score(signals)
|
||||||
|
if score < threshold:
|
||||||
|
continue
|
||||||
|
self.session.add(PostAssociation(
|
||||||
|
announcement_post_id=announcement.id,
|
||||||
|
payload_post_id=group.id,
|
||||||
|
score=score,
|
||||||
|
signals=signals,
|
||||||
|
status="pending",
|
||||||
|
))
|
||||||
|
made += 1
|
||||||
|
return made
|
||||||
|
|
||||||
|
async def list_pending(self) -> list[dict]:
|
||||||
|
rows = (await self.session.execute(
|
||||||
|
select(PostAssociation)
|
||||||
|
.where(PostAssociation.status == "pending")
|
||||||
|
.order_by(PostAssociation.score.desc(), PostAssociation.id.desc())
|
||||||
|
)).scalars().all()
|
||||||
|
return [
|
||||||
|
{
|
||||||
|
"id": a.id,
|
||||||
|
"announcement_post_id": a.announcement_post_id,
|
||||||
|
"payload_post_id": a.payload_post_id,
|
||||||
|
"score": a.score,
|
||||||
|
"signals": a.signals,
|
||||||
|
}
|
||||||
|
for a in rows
|
||||||
|
]
|
||||||
|
|
||||||
|
async def accept(self, association_id: int) -> dict | None:
|
||||||
|
a = await self.session.get(PostAssociation, association_id)
|
||||||
|
if a is None:
|
||||||
|
return None
|
||||||
|
a.status = "linked"
|
||||||
|
return {"id": a.id, "status": a.status}
|
||||||
|
|
||||||
|
async def dismiss(self, association_id: int) -> dict | None:
|
||||||
|
a = await self.session.get(PostAssociation, association_id)
|
||||||
|
if a is None:
|
||||||
|
return None
|
||||||
|
# Kept, not deleted — the row is what remembers the rejection.
|
||||||
|
a.status = "dismissed"
|
||||||
|
return {"id": a.id, "status": a.status}
|
||||||
|
|
||||||
|
async def linked_for(self, post_ids: list[int]) -> dict[int, list[dict]]:
|
||||||
|
"""Accepted links touching these posts, keyed by post id, BOTH ways.
|
||||||
|
|
||||||
|
A post is either end of the relationship, and each end wants the other
|
||||||
|
one: the teaser wants "the full set is over here", the grouping wants
|
||||||
|
"this is what announced me". One query, both directions.
|
||||||
|
"""
|
||||||
|
if not post_ids:
|
||||||
|
return {}
|
||||||
|
rows = (await self.session.execute(
|
||||||
|
select(PostAssociation).where(
|
||||||
|
PostAssociation.status == "linked",
|
||||||
|
or_(
|
||||||
|
PostAssociation.announcement_post_id.in_(post_ids),
|
||||||
|
PostAssociation.payload_post_id.in_(post_ids),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)).scalars().all()
|
||||||
|
out: dict[int, list[dict]] = {}
|
||||||
|
for a in rows:
|
||||||
|
if a.announcement_post_id in post_ids:
|
||||||
|
out.setdefault(a.announcement_post_id, []).append(
|
||||||
|
{"role": "announces", "post_id": a.payload_post_id, "id": a.id}
|
||||||
|
)
|
||||||
|
if a.payload_post_id in post_ids:
|
||||||
|
out.setdefault(a.payload_post_id, []).append(
|
||||||
|
{"role": "announced_by", "post_id": a.announcement_post_id, "id": a.id}
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
async def rescan(session: AsyncSession, *, now: datetime | None = None) -> dict:
|
||||||
|
"""Score every recent non-synthetic post against nearby groupings."""
|
||||||
|
settings = await ImportSettings.load(session)
|
||||||
|
if not settings.discord_link_enabled:
|
||||||
|
return {"enabled": False, "scanned": 0, "proposed": 0}
|
||||||
|
|
||||||
|
now = now or datetime.now(UTC)
|
||||||
|
window_hours = float(settings.discord_link_window_hours)
|
||||||
|
# Only look at announcements that could still have a partner in range —
|
||||||
|
# a full-library rescan is the manual button's job, not the sweep's.
|
||||||
|
horizon = now - timedelta(hours=window_hours * 2)
|
||||||
|
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||||
|
ids = (await session.execute(
|
||||||
|
select(Post.id).where(
|
||||||
|
Post.synthesized_by.is_(None),
|
||||||
|
Post.absorbed_by_post_id.is_(None),
|
||||||
|
sort_key >= horizon,
|
||||||
|
)
|
||||||
|
)).scalars().all()
|
||||||
|
|
||||||
|
svc = PostAssociationService(session)
|
||||||
|
proposed = 0
|
||||||
|
for pid in ids:
|
||||||
|
proposed += await svc.match_post(
|
||||||
|
pid,
|
||||||
|
threshold=float(settings.discord_link_threshold),
|
||||||
|
window_hours=window_hours,
|
||||||
|
)
|
||||||
|
log.info(
|
||||||
|
"discord announcement matcher: scanned %d post(s), proposed %d pair(s)",
|
||||||
|
len(ids), proposed,
|
||||||
|
)
|
||||||
|
return {"enabled": True, "scanned": len(ids), "proposed": proposed}
|
||||||
@@ -41,8 +41,40 @@ THUMBNAIL_LIMIT = 6
|
|||||||
|
|
||||||
|
|
||||||
def _sort_key():
|
def _sort_key():
|
||||||
"""Postgres COALESCE expression used in ORDER BY and WHERE clauses."""
|
"""Postgres COALESCE expression used in ORDER BY and WHERE clauses.
|
||||||
return func.coalesce(Post.post_date, Post.downloaded_at)
|
|
||||||
|
`resurfaced_at` leads (milestone 388 E3). A synthetic post stays OPEN — a
|
||||||
|
creator who adds variants the next day extends the existing post — so such
|
||||||
|
a post has two dates, and which one orders the feed is a real decision:
|
||||||
|
|
||||||
|
* ordering by when the drop STARTED buries a group that grows a week later
|
||||||
|
under a week of other posts, so the operator never sees the new content —
|
||||||
|
which defeats keeping the group open at all;
|
||||||
|
* ordering by every growth lets a group that gains one image a day sit
|
||||||
|
permanently at the top, so chat out-competes authored posts for the front
|
||||||
|
page — the opposite of "post pacing stays front and centre".
|
||||||
|
|
||||||
|
So the feed orders by neither directly. `resurfaced_at` moves only when the
|
||||||
|
anti-thrash rule fires (discord_grouping.should_resurface: enough new
|
||||||
|
images AND enough time since the last move), which means a drip-feed
|
||||||
|
updates IN PLACE and a genuine second wave resurfaces exactly once.
|
||||||
|
|
||||||
|
It is NULL on every ordinary post, so this COALESCE cannot move anything
|
||||||
|
that is not a grouping. Used identically in ORDER BY and in the cursor's
|
||||||
|
WHERE, which is what keeps pagination stable across the change.
|
||||||
|
"""
|
||||||
|
return func.coalesce(Post.resurfaced_at, Post.post_date, Post.downloaded_at)
|
||||||
|
|
||||||
|
|
||||||
|
def _post_sort_value(post: Post):
|
||||||
|
"""The Python twin of `_sort_key()`, for building a cursor from a loaded row.
|
||||||
|
|
||||||
|
Kept next to it on purpose: these two are one expression in two languages,
|
||||||
|
and the failure when they disagree is not an error but a quiet one — rows
|
||||||
|
skipped or repeated at page boundaries, which reads as a backend bug
|
||||||
|
anywhere but here.
|
||||||
|
"""
|
||||||
|
return post.resurfaced_at or post.post_date or post.downloaded_at
|
||||||
|
|
||||||
|
|
||||||
class PostFeedService:
|
class PostFeedService:
|
||||||
@@ -86,6 +118,13 @@ class PostFeedService:
|
|||||||
.join(Artist, Post.artist_id == Artist.id)
|
.join(Artist, Post.artist_id == Artist.id)
|
||||||
.outerjoin(Source, Post.source_id == Source.id)
|
.outerjoin(Source, Post.source_id == Source.id)
|
||||||
)
|
)
|
||||||
|
# Absorbed posts are the individual chat messages a synthetic post
|
||||||
|
# replaced (milestone 388 E2). They stay in the table — they are the
|
||||||
|
# images' true origin and the grouping has to be auditable — but the
|
||||||
|
# feed shows the post FC authored, not the dozen lines it was built
|
||||||
|
# from. `around` and `get_post` deliberately do NOT apply this: reaching
|
||||||
|
# a member by id is how you inspect a grouping.
|
||||||
|
stmt = stmt.where(Post.absorbed_by_post_id.is_(None))
|
||||||
if artist_id is not None:
|
if artist_id is not None:
|
||||||
stmt = stmt.where(Post.artist_id == artist_id)
|
stmt = stmt.where(Post.artist_id == artist_id)
|
||||||
if platform is not None:
|
if platform is not None:
|
||||||
@@ -127,15 +166,19 @@ class PostFeedService:
|
|||||||
# Far edge in the travel direction: oldest row going older,
|
# Far edge in the travel direction: oldest row going older,
|
||||||
# newest row going newer (rows is descending for display).
|
# newest row going newer (rows is descending for display).
|
||||||
edge_post = rows[-1][0] if direction == "older" else rows[0][0]
|
edge_post = rows[-1][0] if direction == "older" else rows[0][0]
|
||||||
edge_key = edge_post.post_date or edge_post.downloaded_at
|
# Must match _sort_key() exactly, including resurfaced_at's
|
||||||
|
# precedence: a cursor built from a different expression than the
|
||||||
|
# ORDER BY silently skips or repeats rows at every page boundary.
|
||||||
|
edge_key = _post_sort_value(edge_post)
|
||||||
next_cursor = encode_cursor(edge_key, edge_post.id)
|
next_cursor = encode_cursor(edge_key, edge_post.id)
|
||||||
|
|
||||||
post_ids = [p.id for p, _, _ in rows]
|
post_ids = [p.id for p, _, _ in rows]
|
||||||
thumbs_map = await self._thumbnails_for(post_ids)
|
thumbs_map = await self._thumbnails_for(post_ids)
|
||||||
atts_map = await self._attachments_for(post_ids)
|
atts_map = await self._attachments_for(post_ids)
|
||||||
|
links_map = await self._links_for(post_ids)
|
||||||
|
|
||||||
items = [
|
items = [
|
||||||
self._to_dict(post, artist, source, thumbs_map, atts_map)
|
self._to_dict(post, artist, source, thumbs_map, atts_map, links_map)
|
||||||
for post, artist, source in rows
|
for post, artist, source in rows
|
||||||
]
|
]
|
||||||
return {"items": items, "next_cursor": next_cursor}
|
return {"items": items, "next_cursor": next_cursor}
|
||||||
@@ -161,7 +204,7 @@ class PostFeedService:
|
|||||||
if anchor is None:
|
if anchor is None:
|
||||||
return None
|
return None
|
||||||
anchor_post, anchor_artist, anchor_source = anchor
|
anchor_post, anchor_artist, anchor_source = anchor
|
||||||
anchor_key = anchor_post.post_date or anchor_post.downloaded_at
|
anchor_key = _post_sort_value(anchor_post)
|
||||||
anchor_cursor = encode_cursor(anchor_key, anchor_post.id)
|
anchor_cursor = encode_cursor(anchor_key, anchor_post.id)
|
||||||
|
|
||||||
older = await self.scroll(
|
older = await self.scroll(
|
||||||
@@ -176,6 +219,7 @@ class PostFeedService:
|
|||||||
atts_map = await self._attachments_for([anchor_post.id])
|
atts_map = await self._attachments_for([anchor_post.id])
|
||||||
anchor_item = self._to_dict(
|
anchor_item = self._to_dict(
|
||||||
anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map,
|
anchor_post, anchor_artist, anchor_source, thumbs_map, atts_map,
|
||||||
|
await self._links_for([anchor_post.id]),
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"items": newer["items"] + [anchor_item] + older["items"],
|
"items": newer["items"] + [anchor_item] + older["items"],
|
||||||
@@ -199,7 +243,10 @@ class PostFeedService:
|
|||||||
# the default arg.
|
# the default arg.
|
||||||
thumbs_map = await self._thumbnails_for([post.id], limit=None)
|
thumbs_map = await self._thumbnails_for([post.id], limit=None)
|
||||||
atts_map = await self._attachments_for([post.id])
|
atts_map = await self._attachments_for([post.id])
|
||||||
item = self._to_dict(post, artist, source, thumbs_map, atts_map)
|
item = self._to_dict(
|
||||||
|
post, artist, source, thumbs_map, atts_map,
|
||||||
|
await self._links_for([post.id]),
|
||||||
|
)
|
||||||
item["description_full"] = html_to_plain(post.description)
|
item["description_full"] = html_to_plain(post.description)
|
||||||
# Full (uncapped) translated description for the detail view (#143).
|
# Full (uncapped) translated description for the detail view (#143).
|
||||||
item["description_translated_full"] = post.description_translated
|
item["description_translated_full"] = post.description_translated
|
||||||
@@ -365,9 +412,25 @@ class PostFeedService:
|
|||||||
})
|
})
|
||||||
return out
|
return out
|
||||||
|
|
||||||
|
async def _links_for(self, post_ids: list[int]) -> dict[int, list[dict]]:
|
||||||
|
"""Accepted announcement links touching these posts (#388 E5).
|
||||||
|
|
||||||
|
Only ACCEPTED ones. A pending proposal is a question for the review
|
||||||
|
queue, not a claim to render beside the artwork — showing one here
|
||||||
|
would assert a link the operator has not agreed to, which is the exact
|
||||||
|
failure the confirm-only design exists to prevent.
|
||||||
|
"""
|
||||||
|
# Imported here rather than at module scope: post_association_service
|
||||||
|
# imports discord_grouping, which imports the models, and the feed
|
||||||
|
# service is imported by the API at startup. A local import keeps that
|
||||||
|
# chain out of the import graph for a purely optional read.
|
||||||
|
from .post_association_service import PostAssociationService
|
||||||
|
|
||||||
|
return await PostAssociationService(self.session).linked_for(post_ids)
|
||||||
|
|
||||||
def _to_dict(
|
def _to_dict(
|
||||||
self, post: Post, artist: Artist, source: Source | None,
|
self, post: Post, artist: Artist, source: Source | None,
|
||||||
thumbs_map: dict, atts_map: dict,
|
thumbs_map: dict, atts_map: dict, links_map: dict | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
plain_full = html_to_plain(post.description) if post.description else None
|
plain_full = html_to_plain(post.description) if post.description else None
|
||||||
if plain_full is None:
|
if plain_full is None:
|
||||||
@@ -400,6 +463,27 @@ class PostFeedService:
|
|||||||
"translated_source_lang": post.translated_source_lang,
|
"translated_source_lang": post.translated_source_lang,
|
||||||
# Sticky per-post translation choice (auto/force/original, #155).
|
# Sticky per-post translation choice (auto/force/original, #155).
|
||||||
"translation_override": post.translation_override,
|
"translation_override": post.translation_override,
|
||||||
|
# Milestone 388 E2. Non-null means FC AUTHORED this post by grouping
|
||||||
|
# a creator's drop — the UI must say so wherever the post appears,
|
||||||
|
# and `synthesis` carries what it was built from so the operator can
|
||||||
|
# audit a grouping FC invented. Null for every real post; the two
|
||||||
|
# keys are always present so the frontend never branches on absence.
|
||||||
|
"synthesized_by": post.synthesized_by,
|
||||||
|
"synthesis": post.synthesis_details,
|
||||||
|
# #388 E3. A grouping stays open, so the card can say "updated N
|
||||||
|
# ago" — which is the whole signal that chat content is trickling
|
||||||
|
# in. NULL means it has not grown since it was created.
|
||||||
|
"last_grew_at": post.last_grew_at.isoformat() if post.last_grew_at else None,
|
||||||
|
# Accepted links only (#388 E5): [{role, post_id, id}], where role
|
||||||
|
# is "announces" (this post is the teaser) or "announced_by" (this
|
||||||
|
# post is the drop). Always a list so the UI never branches on
|
||||||
|
# absence.
|
||||||
|
"associations": (links_map or {}).get(post.id, []),
|
||||||
|
# Non-null on a chat message a synthetic post absorbed. The feed
|
||||||
|
# filters these out, but `around`/`get_post` still reach them, and
|
||||||
|
# the UI uses this to explain why a post it linked to is not in the
|
||||||
|
# stream.
|
||||||
|
"absorbed_by_post_id": post.absorbed_by_post_id,
|
||||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||||
"source": (
|
"source": (
|
||||||
{"id": source.id, "platform": source.platform}
|
{"id": source.id, "platform": source.platform}
|
||||||
|
|||||||
@@ -8,12 +8,13 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import func, select
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
from sqlalchemy.orm import selectinload
|
from sqlalchemy.orm import selectinload
|
||||||
|
|
||||||
from ..models import AppSetting, Artist, ImportSettings, Source
|
from ..models import AppSetting, Artist, ImportSettings, Source
|
||||||
|
from .db_helpers import failing_sources_clause, no_access_sources_clause
|
||||||
|
|
||||||
MIN_INTERVAL_SECONDS = 60
|
MIN_INTERVAL_SECONDS = 60
|
||||||
MAX_INTERVAL_SECONDS = 86400
|
MAX_INTERVAL_SECONDS = 86400
|
||||||
@@ -219,10 +220,38 @@ async def scheduler_status(session: AsyncSession) -> dict:
|
|||||||
|
|
||||||
cooldowns = await active_platform_cooldowns(session)
|
cooldowns = await active_platform_cooldowns(session)
|
||||||
|
|
||||||
|
# Ingestion health for the front-door ribbon (#387 B3). Counted over ENABLED
|
||||||
|
# sources rather than the auto_check subset walked above: a source that is
|
||||||
|
# erroring or paywalled is worth surfacing whether or not a schedule happens
|
||||||
|
# to poll it. Two scalar COUNTs, not a second pass over `rows`.
|
||||||
|
#
|
||||||
|
# Both predicates are the shared ones, so the ribbon and the surfaces it
|
||||||
|
# links to cannot disagree about what they are counting.
|
||||||
|
failing_sources = (await session.execute(
|
||||||
|
select(func.count()).select_from(Source)
|
||||||
|
.where(Source.enabled.is_(True), failing_sources_clause())
|
||||||
|
)).scalar_one()
|
||||||
|
no_access_sources = (await session.execute(
|
||||||
|
select(func.count()).select_from(Source)
|
||||||
|
.where(Source.enabled.is_(True), no_access_sources_clause())
|
||||||
|
)).scalar_one()
|
||||||
|
# #387 B4: lets the front door tell "nothing configured yet" (a fresh
|
||||||
|
# install — show the on-ramp) apart from "configured, still fetching" (a
|
||||||
|
# first run in progress — show what's running). Telling someone to add a
|
||||||
|
# source when they already have three and are mid-backfill is worse than
|
||||||
|
# saying nothing. Deliberately NOT auto_sources, which counts only what is
|
||||||
|
# on a schedule: a source with auto_check off still means "configured".
|
||||||
|
total_sources = (await session.execute(
|
||||||
|
select(func.count()).select_from(Source).where(Source.enabled.is_(True))
|
||||||
|
)).scalar_one()
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"last_tick_at": last_tick_at,
|
"last_tick_at": last_tick_at,
|
||||||
"next_due_at": next_due_at.isoformat() if next_due_at else None,
|
"next_due_at": next_due_at.isoformat() if next_due_at else None,
|
||||||
"due_now": due_now,
|
"due_now": due_now,
|
||||||
"auto_sources": len(rows),
|
"auto_sources": len(rows),
|
||||||
|
"failing_sources": failing_sources,
|
||||||
|
"no_access_sources": no_access_sources,
|
||||||
|
"total_sources": total_sources,
|
||||||
"platform_cooldowns": {p: dt.isoformat() for p, dt in cooldowns.items()},
|
"platform_cooldowns": {p: dt.isoformat() for p, dt in cooldowns.items()},
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,175 @@
|
|||||||
|
"""The learned roster: which of FabledCurator's parts have checked in, and when.
|
||||||
|
|
||||||
|
Milestone 365. `celery inspect` answers "who is here"; this answers "who is
|
||||||
|
missing", which nothing in the application could do before — see
|
||||||
|
`models/service_seen.py` for why the identity is a queue set and not a
|
||||||
|
worker hostname.
|
||||||
|
|
||||||
|
## Who does the observing, and why it is the web process
|
||||||
|
|
||||||
|
Three candidates, and the choice matters more than the code:
|
||||||
|
|
||||||
|
* **A celery beat sweep.** Rejected. If the scheduler dies, the sweep stops,
|
||||||
|
every row goes stale, and the page reports that everything is down when one
|
||||||
|
thing is. An alarm that cannot distinguish "one part died" from "the
|
||||||
|
observer died" is worse than no alarm.
|
||||||
|
* **A background task in web.** Rejected on a detail of how this deploys:
|
||||||
|
hypercorn runs `--workers 4`, so a `before_serving` loop would be FOUR
|
||||||
|
concurrent inspect loops hammering the broker, forever, per container.
|
||||||
|
* **Refresh on demand, rate-limited by the data itself.** Taken. Whichever web
|
||||||
|
process happens to serve a health request refreshes the roster if it is
|
||||||
|
older than REFRESH_TTL, and otherwise reads what is already there.
|
||||||
|
|
||||||
|
The third has the property the other two lack: **the observer is the thing
|
||||||
|
serving the page.** If web is down you get a browser error rather than a
|
||||||
|
confidently green page, which is the honest failure. It also self-limits
|
||||||
|
without coordination — the TTL lives in the row everybody can see.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from ..models import ServiceSeen
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# How stale the roster may be before a health request refreshes it. Comfortably
|
||||||
|
# under the staleness thresholds that decide a service is missing, so the
|
||||||
|
# verdict is never limited by how often anyone looked.
|
||||||
|
REFRESH_TTL_SECONDS = 20.0
|
||||||
|
|
||||||
|
# celery inspect is a broker round trip and this sits on a request path, so it
|
||||||
|
# gets a deadline (rule 156). A broker that has stopped answering must make the
|
||||||
|
# roster stale — which is a true statement about the system — not hang the one
|
||||||
|
# page that exists to explain it.
|
||||||
|
INSPECT_TIMEOUT_SECONDS = 2.0
|
||||||
|
|
||||||
|
# Queue set -> the name an operator recognises. Sorted-tuple keys, because the
|
||||||
|
# order celery reports them in is not guaranteed.
|
||||||
|
#
|
||||||
|
# A deployment that slices CELERY_QUEUES differently falls through to the raw
|
||||||
|
# queue list rather than being given a name this table invented for it: a
|
||||||
|
# wrong-but-confident label on a status page is worse than an ugly true one.
|
||||||
|
ROLE_NAMES: dict[tuple[str, ...], str] = {
|
||||||
|
("default", "download", "import", "thumbnail"): "Worker",
|
||||||
|
("maintenance", "scan"): "Scheduler",
|
||||||
|
("ml",): "ML worker",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def role_display_name(queues: tuple[str, ...]) -> str:
|
||||||
|
known = ROLE_NAMES.get(queues)
|
||||||
|
if known:
|
||||||
|
return known
|
||||||
|
return "Worker (" + ", ".join(queues) + ")"
|
||||||
|
|
||||||
|
|
||||||
|
def _inspect_celery_sync() -> dict[tuple[str, ...], dict]:
|
||||||
|
"""celery inspect, grouped by queue set rather than by worker.
|
||||||
|
|
||||||
|
Returns {queue_set: {"hostnames": [...], "active": int}}. Two replicas of
|
||||||
|
one role collapse into one entry on purpose — the question is whether the
|
||||||
|
role is being served, not how many containers exist.
|
||||||
|
"""
|
||||||
|
from ..celery_app import celery as celery_app
|
||||||
|
|
||||||
|
insp = celery_app.control.inspect(timeout=INSPECT_TIMEOUT_SECONDS)
|
||||||
|
active_queues = insp.active_queues() or {}
|
||||||
|
active_tasks = insp.active() or {}
|
||||||
|
|
||||||
|
grouped: dict[tuple[str, ...], dict] = {}
|
||||||
|
for hostname, queues in active_queues.items():
|
||||||
|
key = tuple(sorted({q["name"] for q in queues}))
|
||||||
|
entry = grouped.setdefault(key, {"hostnames": [], "active": 0})
|
||||||
|
entry["hostnames"].append(hostname)
|
||||||
|
entry["active"] += len(active_tasks.get(hostname, []))
|
||||||
|
for entry in grouped.values():
|
||||||
|
entry["hostnames"].sort()
|
||||||
|
return grouped
|
||||||
|
|
||||||
|
|
||||||
|
async def touch_service(
|
||||||
|
session: AsyncSession, *, key: str, kind: str, display_name: str, details: dict
|
||||||
|
) -> None:
|
||||||
|
"""Record that a part checked in just now.
|
||||||
|
|
||||||
|
Upsert rather than read-modify-write: several web processes and several
|
||||||
|
agents can be doing this at once, and the last writer is simply the most
|
||||||
|
recent sighting. `first_seen_at` is deliberately NOT updated — it is the
|
||||||
|
one field that answers "has this ever run", which the learned-roster design
|
||||||
|
depends on.
|
||||||
|
"""
|
||||||
|
stmt = pg_insert(ServiceSeen).values(
|
||||||
|
key=key, kind=kind, display_name=display_name, details=details,
|
||||||
|
)
|
||||||
|
stmt = stmt.on_conflict_do_update(
|
||||||
|
index_elements=[ServiceSeen.key],
|
||||||
|
set_={
|
||||||
|
"kind": stmt.excluded.kind,
|
||||||
|
"display_name": stmt.excluded.display_name,
|
||||||
|
"details": stmt.excluded.details,
|
||||||
|
"last_seen_at": func.now(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
await session.execute(stmt)
|
||||||
|
|
||||||
|
|
||||||
|
async def refresh_celery_roster(session: AsyncSession) -> None:
|
||||||
|
"""Inspect the broker and record what answered. Never raises.
|
||||||
|
|
||||||
|
A failure here means the roster does not advance, and the rows going stale
|
||||||
|
is then a TRUE report about a broker nobody can reach. Letting the
|
||||||
|
exception out would instead break the health endpoint, which is the one
|
||||||
|
thing that must keep answering when the stack is unwell.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
grouped = await asyncio.wait_for(
|
||||||
|
asyncio.to_thread(_inspect_celery_sync),
|
||||||
|
timeout=INSPECT_TIMEOUT_SECONDS * 2,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
log.warning("service roster: celery inspect failed; roster not refreshed", exc_info=True)
|
||||||
|
return
|
||||||
|
|
||||||
|
for queues, entry in grouped.items():
|
||||||
|
await touch_service(
|
||||||
|
session,
|
||||||
|
key="celery:" + ",".join(queues),
|
||||||
|
kind="celery",
|
||||||
|
display_name=role_display_name(queues),
|
||||||
|
details={
|
||||||
|
"queues": list(queues),
|
||||||
|
"hostnames": entry["hostnames"],
|
||||||
|
"replicas": len(entry["hostnames"]),
|
||||||
|
"active": entry["active"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def refresh_if_stale(session: AsyncSession) -> None:
|
||||||
|
"""Refresh the celery roster if nobody has for REFRESH_TTL_SECONDS.
|
||||||
|
|
||||||
|
Rate-limited by the data rather than by a lock: the gate is the newest
|
||||||
|
last_seen_at across the celery rows, which every web process can see. Two
|
||||||
|
processes racing through the gate costs one redundant inspect and writes
|
||||||
|
the same values twice, so the benign outcome needs no coordination to
|
||||||
|
prevent.
|
||||||
|
"""
|
||||||
|
newest = (
|
||||||
|
await session.execute(
|
||||||
|
select(func.max(ServiceSeen.last_seen_at)).where(ServiceSeen.kind == "celery")
|
||||||
|
)
|
||||||
|
).scalar_one_or_none()
|
||||||
|
|
||||||
|
if newest is not None:
|
||||||
|
age = (await session.execute(select(func.now()))).scalar_one() - newest
|
||||||
|
if age.total_seconds() < REFRESH_TTL_SECONDS:
|
||||||
|
return
|
||||||
|
|
||||||
|
await refresh_celery_roster(session)
|
||||||
@@ -10,12 +10,15 @@ from sqlalchemy.ext.asyncio import AsyncSession
|
|||||||
|
|
||||||
from ..models import (
|
from ..models import (
|
||||||
Artist,
|
Artist,
|
||||||
|
DownloadEvent,
|
||||||
ImageProvenance,
|
ImageProvenance,
|
||||||
ImageRecord,
|
ImageRecord,
|
||||||
ImportSettings,
|
ImportSettings,
|
||||||
Post,
|
Post,
|
||||||
Source,
|
Source,
|
||||||
)
|
)
|
||||||
|
from .db_helpers import failing_sources_clause
|
||||||
|
from .gallery_dl import ErrorType
|
||||||
from .platforms import known_platform_keys
|
from .platforms import known_platform_keys
|
||||||
from .scheduler_service import compute_next_check_at
|
from .scheduler_service import compute_next_check_at
|
||||||
|
|
||||||
@@ -84,6 +87,11 @@ class SourceRecord:
|
|||||||
# plan #704: cumulative posts processed across the walk's chunks — live
|
# plan #704: cumulative posts processed across the walk's chunks — live
|
||||||
# progress for the badge.
|
# progress for the badge.
|
||||||
backfill_posts: int
|
backfill_posts: int
|
||||||
|
# Milestone #387 A3: posts the last walk skipped because the account can't
|
||||||
|
# view them. Lives on the EVENT (run_stats.tier_gated_count), not the
|
||||||
|
# source, so it is joined in by `list()` only — None everywhere else, which
|
||||||
|
# the UI renders as the bare no-access state with no fabricated number.
|
||||||
|
tier_gated_count: int | None = None
|
||||||
|
|
||||||
def to_dict(self) -> dict:
|
def to_dict(self) -> dict:
|
||||||
return {
|
return {
|
||||||
@@ -107,6 +115,7 @@ class SourceRecord:
|
|||||||
"backfill_bypass_seen": self.backfill_bypass_seen,
|
"backfill_bypass_seen": self.backfill_bypass_seen,
|
||||||
"backfill_recapture": self.backfill_recapture,
|
"backfill_recapture": self.backfill_recapture,
|
||||||
"backfill_posts": self.backfill_posts,
|
"backfill_posts": self.backfill_posts,
|
||||||
|
"tier_gated_count": self.tier_gated_count,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -159,8 +168,39 @@ class SourceService:
|
|||||||
async def _load_settings(self) -> ImportSettings:
|
async def _load_settings(self) -> ImportSettings:
|
||||||
return await ImportSettings.load(self.session)
|
return await ImportSettings.load(self.session)
|
||||||
|
|
||||||
|
async def _tier_gated_counts(self, source_ids: list[int]) -> dict[int, int]:
|
||||||
|
"""Latest walk's tier-gated post count, per source, in ONE query.
|
||||||
|
|
||||||
|
Selects the `run_stats` sub-object rather than whole `metadata` blobs:
|
||||||
|
those carry truncated stdout/stderr up to 500KB each, and pulling one
|
||||||
|
per source to read a single integer would make the subscriptions list
|
||||||
|
pay for the Logs view. DISTINCT ON + ORDER BY takes the newest event per
|
||||||
|
source (Postgres-only, like the rest of this codebase).
|
||||||
|
|
||||||
|
Callers pass only the sources that actually need it — the count is
|
||||||
|
meaningless for a source that isn't tier-gated.
|
||||||
|
"""
|
||||||
|
if not source_ids:
|
||||||
|
return {}
|
||||||
|
rows = (await self.session.execute(
|
||||||
|
select(
|
||||||
|
DownloadEvent.source_id,
|
||||||
|
DownloadEvent.metadata_["run_stats"],
|
||||||
|
)
|
||||||
|
.where(DownloadEvent.source_id.in_(source_ids))
|
||||||
|
.distinct(DownloadEvent.source_id)
|
||||||
|
.order_by(DownloadEvent.source_id, DownloadEvent.started_at.desc())
|
||||||
|
)).all()
|
||||||
|
counts: dict[int, int] = {}
|
||||||
|
for source_id, run_stats in rows:
|
||||||
|
n = (run_stats or {}).get("tier_gated_count") or 0
|
||||||
|
if n:
|
||||||
|
counts[source_id] = int(n)
|
||||||
|
return counts
|
||||||
|
|
||||||
def _build_record(
|
def _build_record(
|
||||||
self, source: Source, artist: Artist, settings: ImportSettings,
|
self, source: Source, artist: Artist, settings: ImportSettings,
|
||||||
|
gated_counts: dict[int, int] | None = None,
|
||||||
) -> SourceRecord:
|
) -> SourceRecord:
|
||||||
nxt = compute_next_check_at(source, artist, settings)
|
nxt = compute_next_check_at(source, artist, settings)
|
||||||
co = source.config_overrides or {}
|
co = source.config_overrides or {}
|
||||||
@@ -185,6 +225,7 @@ class SourceService:
|
|||||||
backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
|
backfill_bypass_seen=bool(co.get("_backfill_bypass_seen")),
|
||||||
backfill_recapture=bool(co.get("_backfill_recapture")),
|
backfill_recapture=bool(co.get("_backfill_recapture")),
|
||||||
backfill_posts=int(co.get("_backfill_posts", 0)),
|
backfill_posts=int(co.get("_backfill_posts", 0)),
|
||||||
|
tier_gated_count=(gated_counts or {}).get(source.id),
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _row_to_record(self, source: Source) -> SourceRecord:
|
async def _row_to_record(self, source: Source) -> SourceRecord:
|
||||||
@@ -210,14 +251,21 @@ class SourceService:
|
|||||||
stmt = stmt.where(~Source.url.like("sidecar:%"))
|
stmt = stmt.where(~Source.url.like("sidecar:%"))
|
||||||
if failing:
|
if failing:
|
||||||
# Worst-first so the rollup card surfaces the loudest failures.
|
# Worst-first so the rollup card surfaces the loudest failures.
|
||||||
stmt = stmt.where(Source.consecutive_failures > 0).order_by(
|
# Shared clause: the front-door ribbon counts with the same one, so
|
||||||
|
# it can never report a number this list then contradicts.
|
||||||
|
stmt = stmt.where(failing_sources_clause()).order_by(
|
||||||
Source.consecutive_failures.desc(), Artist.name.asc(),
|
Source.consecutive_failures.desc(), Artist.name.asc(),
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
stmt = stmt.order_by(Artist.name.asc(), Source.id.asc())
|
stmt = stmt.order_by(Artist.name.asc(), Source.id.asc())
|
||||||
rows = (await self.session.execute(stmt)).all()
|
rows = (await self.session.execute(stmt)).all()
|
||||||
settings = await self._load_settings()
|
settings = await self._load_settings()
|
||||||
return [self._build_record(s, a, settings) for s, a in rows]
|
# Only tier-gated rows need the join — on a healthy library that is an
|
||||||
|
# empty list and _tier_gated_counts short-circuits without a query.
|
||||||
|
gated_counts = await self._tier_gated_counts(
|
||||||
|
[s.id for s, _a in rows if s.error_type == ErrorType.TIER_LIMITED]
|
||||||
|
)
|
||||||
|
return [self._build_record(s, a, settings, gated_counts) for s, a in rows]
|
||||||
|
|
||||||
async def get(self, source_id: int) -> SourceRecord | None:
|
async def get(self, source_id: int) -> SourceRecord | None:
|
||||||
source = (await self.session.execute(
|
source = (await self.session.execute(
|
||||||
|
|||||||
@@ -1131,3 +1131,76 @@ def vacuum_analyze() -> dict:
|
|||||||
done.append(table)
|
done.append(table)
|
||||||
log.info("vacuum_analyze complete: %s", done)
|
log.info("vacuum_analyze complete: %s", done)
|
||||||
return {"vacuumed": done}
|
return {"vacuumed": done}
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.maintenance.group_discord_drops",
|
||||||
|
soft_time_limit=1800, time_limit=2100,
|
||||||
|
)
|
||||||
|
def group_discord_drops() -> str:
|
||||||
|
"""Milestone 388 E2: group Discord message-posts into the drops FC authors.
|
||||||
|
|
||||||
|
Lives on the MAINTENANCE lane, not the ml lane, even though it reads SigLIP
|
||||||
|
vectors — it does no inference and imports no ML library, and the ml-worker
|
||||||
|
is an OPTIONAL container (B3). Routing it to 'ml' would silently disable
|
||||||
|
grouping on every stack that runs a GPU agent and drops that container,
|
||||||
|
which is the same trap gpu_queue.py was moved here to avoid.
|
||||||
|
|
||||||
|
Async body under its own loop, per the _async_session contract: the sweep
|
||||||
|
needs pgvector column reads and the shared services are async.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from ..services.discord_grouping import sweep
|
||||||
|
from ._async_session import async_session_factory
|
||||||
|
|
||||||
|
async def _run() -> dict:
|
||||||
|
async_factory, engine = async_session_factory()
|
||||||
|
try:
|
||||||
|
async with async_factory() as session:
|
||||||
|
result = await sweep(session)
|
||||||
|
await session.commit()
|
||||||
|
return result
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
res = asyncio.run(_run())
|
||||||
|
if not res["enabled"]:
|
||||||
|
return "disabled"
|
||||||
|
return (
|
||||||
|
f"sources={res['sources']} created={res['posts_created']} "
|
||||||
|
f"joined={res['images_joined']}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.maintenance.match_post_associations",
|
||||||
|
soft_time_limit=900, time_limit=1200,
|
||||||
|
)
|
||||||
|
def match_post_associations() -> str:
|
||||||
|
"""Milestone 388 E5: propose which Patreon post announced which Discord drop.
|
||||||
|
|
||||||
|
Proposes only — every pair lands in a review queue and nothing is linked
|
||||||
|
until the operator accepts. Maintenance lane for the same reason as the
|
||||||
|
grouper: no inference, no ML library, and it must not depend on the
|
||||||
|
optional ml-worker being present.
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from ..services.post_association_service import rescan
|
||||||
|
from ._async_session import async_session_factory
|
||||||
|
|
||||||
|
async def _run() -> dict:
|
||||||
|
async_factory, engine = async_session_factory()
|
||||||
|
try:
|
||||||
|
async with async_factory() as session:
|
||||||
|
result = await rescan(session)
|
||||||
|
await session.commit()
|
||||||
|
return result
|
||||||
|
finally:
|
||||||
|
await engine.dispose()
|
||||||
|
|
||||||
|
res = asyncio.run(_run())
|
||||||
|
if not res["enabled"]:
|
||||||
|
return "disabled"
|
||||||
|
return f"scanned={res['scanned']} proposed={res['proposed']}"
|
||||||
|
|||||||
+15
-8
@@ -198,14 +198,21 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
|||||||
refresh from being undone.
|
refresh from being undone.
|
||||||
- **`pull: true` on the scheduled path only** is the mechanism: a moved base
|
- **`pull: true` on the scheduled path only** is the mechanism: a moved base
|
||||||
tag changes the `FROM` layer's cache key and everything above it rebuilds.
|
tag changes the `FROM` layer's cache key and everything above it rebuilds.
|
||||||
**It does not currently make the unmoved case free.** Measured on the first
|
It did not always make the unmoved case free. Measured on the first real
|
||||||
real fire (run 4934, 2026-08-30): every content step reported `CACHED` and
|
fire (run 4934, 2026-08-30): every content step reported `CACHED` and the
|
||||||
the bases resolved to unchanged digests, yet all three `:latest` tags got a
|
bases resolved to unchanged digests, yet all three `:latest` tags got a NEW
|
||||||
NEW manifest digest, because buildkit mints a fresh image config per run and
|
manifest digest, because buildkit stamps a fresh image config per run and
|
||||||
republishes identical layers under it. So `:latest` is rewritten weekly
|
republishes identical layers under it — so `:latest` was rewritten weekly
|
||||||
whether or not anything changed, and `:c-<sha>` is handed a new manifest to
|
whether or not anything changed, and a digest change stopped meaning
|
||||||
diverge from on the same cadence — a digest change stops meaning anything.
|
anything (#3265).
|
||||||
Tracked as #3265; the likely fix is a deterministic `SOURCE_DATE_EPOCH`.
|
- **`SOURCE_DATE_EPOCH` is what makes it free.** Set on each build step from
|
||||||
|
`artifacts.sh epoch <artifact>` — the unix timestamp of the same commit
|
||||||
|
`revision` and `version` name, so all three are views of one `newest()`
|
||||||
|
lookup and cannot drift into disagreeing. With the config's `created` field
|
||||||
|
and history timestamps pinned to the content rather than to the wall clock,
|
||||||
|
identical source produces an identical manifest digest and the push is a
|
||||||
|
registry no-op. That restores the property the whole scheme rests on: a
|
||||||
|
channel tag's digest changes when, and only when, its content does.
|
||||||
Separately not caught: a Debian package update inside the `apt-get install`
|
Separately not caught: a Debian package update inside the `apt-get install`
|
||||||
layer while the base tag stands still — a lag rather than a hole, since the
|
layer while the base tag stands still — a lag rather than a hole, since the
|
||||||
official python/cuda images rebuild with those updates baked in.
|
official python/cuda images rebuild with those updates baked in.
|
||||||
|
|||||||
+69
-20
@@ -1,12 +1,21 @@
|
|||||||
# Base compose stack. Uses ${VAR:-default} interpolation throughout so the
|
# Base compose stack, and the install path. Uses ${VAR:-default} throughout so
|
||||||
# stack boots with zero config — sane dev defaults baked in. For production
|
# the stack boots with zero config — but those defaults are DEV defaults, and
|
||||||
# deployments, override the defaults via shell env vars or a .env file:
|
# two of them (DB_PASSWORD, SECRET_KEY) are published in this file. Copy
|
||||||
|
# .env.example to .env and set them before running this anywhere real.
|
||||||
#
|
#
|
||||||
# DB_PASSWORD=...real... SECRET_KEY=...real... docker compose up
|
# To run FabledCurator:
|
||||||
#
|
#
|
||||||
# The dev override (docker-compose.override.yml) is auto-merged when you
|
# docker compose -f docker-compose.yml up -d
|
||||||
# run `docker compose up` from this directory and switches images to
|
#
|
||||||
# local builds + DEBUG logging.
|
# The -f is load-bearing. Without it Compose auto-merges
|
||||||
|
# docker-compose.override.yml, which replaces every image: with a local
|
||||||
|
# build: and turns on DEBUG logging — the contributor path. Naming this file
|
||||||
|
# explicitly skips the override and pulls the published :latest images.
|
||||||
|
#
|
||||||
|
# FabledCurator has no authentication. Whatever can reach ${PORT} is an
|
||||||
|
# administrator, including over the stored Patreon/SubscribeStar/Pixiv session
|
||||||
|
# cookies. Do not publish this port beyond a network you trust — see
|
||||||
|
# "Before you expose it" in README.md.
|
||||||
|
|
||||||
# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a
|
# Rolling-deploy safety (Swarm / `docker stack deploy`): update one task at a
|
||||||
# time, START the new task before stopping the old (zero-downtime via the ingress
|
# time, START the new task before stopping the old (zero-downtime via the ingress
|
||||||
@@ -74,7 +83,25 @@ services:
|
|||||||
retries: 5
|
retries: 5
|
||||||
|
|
||||||
web:
|
web:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
# :latest, NOT :dev — this file IS the install path.
|
||||||
|
#
|
||||||
|
# `docker compose up -d` merges docker-compose.override.yml, which sets
|
||||||
|
# build: for all five app services, and a build: wins over image:. So a
|
||||||
|
# contributor never pulls this tag and is unaffected by what it says.
|
||||||
|
#
|
||||||
|
# The tag is consulted only on `docker compose -f docker-compose.yml up -d`
|
||||||
|
# — the documented production path, which skips the override. That is a
|
||||||
|
# stranger installing the product, and they must land on the stable channel.
|
||||||
|
#
|
||||||
|
# :latest is main, which IS production (rule 147). :dev is the rolling
|
||||||
|
# bleeding-edge channel we work out of, republished several times a day with
|
||||||
|
# no stability promise. This file pinned :dev on all five services until
|
||||||
|
# 2026-08-31 (#3270), so the documented install shipped development builds.
|
||||||
|
# It went unnoticed because nobody who works on the project takes this path:
|
||||||
|
# the operator deploys from a swarm stack file, contributors get the
|
||||||
|
# override. Do not "fix" this back to :dev while debugging — use the
|
||||||
|
# override, or -f with an explicit tag on the command line.
|
||||||
|
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||||
command: ["web"]
|
command: ["web"]
|
||||||
# Graceful shutdown: give the container time to drain in-flight work on a
|
# Graceful shutdown: give the container time to drain in-flight work on a
|
||||||
# deploy (docker SIGTERMs, then SIGKILLs after this window — default is only
|
# deploy (docker SIGTERMs, then SIGKILLs after this window — default is only
|
||||||
@@ -105,24 +132,46 @@ services:
|
|||||||
CELERY_BROKER_URL: redis://redis:6379/0
|
CELERY_BROKER_URL: redis://redis:6379/0
|
||||||
CELERY_RESULT_BACKEND: redis://redis:6379/0
|
CELERY_RESULT_BACKEND: redis://redis:6379/0
|
||||||
SECRET_KEY: ${SECRET_KEY:-dev_secret_key_not_for_production_change_me}
|
SECRET_KEY: ${SECRET_KEY:-dev_secret_key_not_for_production_change_me}
|
||||||
EXTENSION_API_KEY: ${EXTENSION_API_KEY:-}
|
|
||||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||||
|
# First boot only. FabledCurator refuses to start until the credential
|
||||||
|
# encryption key at /images/secrets/credential_key.b64 exists, and
|
||||||
|
# refuses to create one unless told to — auto-creating is
|
||||||
|
# indistinguishable from a restore that lost ./images/secrets, where it
|
||||||
|
# would mint a key that decrypts nothing and leave an instance that looks
|
||||||
|
# healthy while every paywalled download fails.
|
||||||
|
#
|
||||||
|
# Passed through EXPLICITLY because a variable in `.env` is only used for
|
||||||
|
# ${...} interpolation; it does not reach the container unless it is
|
||||||
|
# named here. Defaulted to empty so the refusal stands for everyone who
|
||||||
|
# has not opted in — the app tests for exactly "1".
|
||||||
|
#
|
||||||
|
# Set it in .env for one `up`, then remove it. See .env.example.
|
||||||
|
CURATOR_BOOTSTRAP_NEW_KEY: ${CURATOR_BOOTSTRAP_NEW_KEY:-}
|
||||||
volumes:
|
volumes:
|
||||||
- ./images:/images
|
- ./images:/images
|
||||||
- ./import:/import
|
- ./import:/import
|
||||||
# FC-5 legacy migration: bind-mount the host's ImageRepo images dir
|
# /import is a staging area for scripting a one-off ingest of a library
|
||||||
# under /import (FC's existing filesystem scan picks them up). Read-only
|
# you already have on disk. Drop files in ./import, or bind-mount an
|
||||||
# is sufficient — FC copies into /images during the scan. The worker +
|
# existing directory under it as below, then trigger the scan:
|
||||||
# scheduler services see the same /import via their own mounts below
|
#
|
||||||
# because of /import volume reuse. Edit the host path to match your
|
# curl -X POST http://localhost:8080/api/import/trigger
|
||||||
# install before running Settings → Maintenance → Legacy migration.
|
#
|
||||||
# - /var/lib/imagerepo/images:/import/imagerepo:ro
|
# Read-only is sufficient — FC copies into /images during the scan. The
|
||||||
|
# worker + scheduler services mount the same /import so the scan can run
|
||||||
|
# on whichever lane picks it up.
|
||||||
|
#
|
||||||
|
# Deliberately has no UI. The manual-scan surface was retired 2026-07-02
|
||||||
|
# once imports arrived via subscriptions + the extension, and the call
|
||||||
|
# not to restore it stands (operator, 2026-09-02): folder ingestion
|
||||||
|
# brings complexity the product does not need. The endpoint stays as an
|
||||||
|
# unsupported escape hatch; the supported way in is Subscriptions.
|
||||||
|
# - /srv/media/my-library:/import/my-library:ro
|
||||||
depends_on:
|
depends_on:
|
||||||
postgres: { condition: service_healthy }
|
postgres: { condition: service_healthy }
|
||||||
redis: { condition: service_healthy }
|
redis: { condition: service_healthy }
|
||||||
|
|
||||||
worker:
|
worker:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||||
command: ["worker"]
|
command: ["worker"]
|
||||||
# Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
|
# Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
|
||||||
stop_grace_period: 90s
|
stop_grace_period: 90s
|
||||||
@@ -142,7 +191,7 @@ services:
|
|||||||
redis: { condition: service_healthy }
|
redis: { condition: service_healthy }
|
||||||
|
|
||||||
scheduler:
|
scheduler:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||||
command: ["scheduler"]
|
command: ["scheduler"]
|
||||||
# Quick maintenance/scan lane + beat — short tasks, modest drain window.
|
# Quick maintenance/scan lane + beat — short tasks, modest drain window.
|
||||||
stop_grace_period: 60s
|
stop_grace_period: 60s
|
||||||
@@ -163,7 +212,7 @@ services:
|
|||||||
# 30-min backup or a multi-chunk audit can never starve the 5-min recovery
|
# 30-min backup or a multi-chunk audit can never starve the 5-min recovery
|
||||||
# sweeps / vacuum (operator-flagged 2026-06-07). One slot — these are heavy.
|
# sweeps / vacuum (operator-flagged 2026-06-07). One slot — these are heavy.
|
||||||
maintenance-long:
|
maintenance-long:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||||
command: ["worker"]
|
command: ["worker"]
|
||||||
# Longest lane (DB backups, library audits, translation backfill) — give it
|
# Longest lane (DB backups, library audits, translation backfill) — give it
|
||||||
# the most room to finish a chunk gracefully. Chunked + idempotent, so a job
|
# the most room to finish a chunk gracefully. Chunked + idempotent, so a job
|
||||||
@@ -184,7 +233,7 @@ services:
|
|||||||
redis: { condition: service_healthy }
|
redis: { condition: service_healthy }
|
||||||
|
|
||||||
ml-worker:
|
ml-worker:
|
||||||
image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev
|
image: git.fabledsword.com/bvandeusen/fabledcurator-ml:latest
|
||||||
command: ["ml-worker"]
|
command: ["ml-worker"]
|
||||||
# A single GPU inference pass can run tens of seconds — let it finish.
|
# A single GPU inference pass can run tens of seconds — let it finish.
|
||||||
stop_grace_period: 120s
|
stop_grace_period: 120s
|
||||||
|
|||||||
@@ -1,5 +1,24 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32">
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 32 32" role="img" aria-label="FabledCurator">
|
||||||
|
<title>FabledCurator</title>
|
||||||
|
<!-- Hand-drawn, NOT traced from the source raster. This renders at 16px in a
|
||||||
|
browser tab and 22px in the nav (TopNav.vue), and a traced mark carries
|
||||||
|
hundreds of sub-pixel wobble nodes that read as fuzz at those sizes and
|
||||||
|
cannot be tidied afterwards.
|
||||||
|
|
||||||
|
Two elements only. The full logo's glove, cufflink, fleur-de-lis finials,
|
||||||
|
outer ring and sparkle rays are all deliberately ABSENT rather than drawn
|
||||||
|
and lost: measured at 16px, ring + frame + star merged into an
|
||||||
|
indistinct blob, and the whole logo was unreadable mush. The frame won
|
||||||
|
over a plain ring because it carries the meaning — a framed work is what
|
||||||
|
curation looks like — and it is the full logo's own centrepiece.
|
||||||
|
|
||||||
|
Colours are theme tokens (frontend/src/theme/fabled-tokens.js): obsidian
|
||||||
|
plate, accent gold. The plate is kept here (unlike logo.svg) so the tab
|
||||||
|
icon is self-contained against any browser chrome; on the nav it is
|
||||||
|
invisible because it matches --fc-chrome-rgb exactly. -->
|
||||||
<rect width="32" height="32" rx="6" fill="#14171A"/>
|
<rect width="32" height="32" rx="6" fill="#14171A"/>
|
||||||
<path d="M8 9 L8 23 L24 23 L24 9 L20 9 L16 13 L12 9 Z"
|
<rect x="6.2" y="4.2" width="19.6" height="23.6" rx="1.4"
|
||||||
fill="#A87338" stroke="#E8E4D8" stroke-width="1" stroke-linejoin="round"/>
|
fill="none" stroke="#A87338" stroke-width="2.4"/>
|
||||||
|
<path d="M16 9.6 Q16.75 15.25 22.4 16 Q16.75 16.75 16 22.4 Q15.25 16.75 9.6 16 Q15.25 15.25 16 9.6 Z"
|
||||||
|
fill="#A87338"/>
|
||||||
</svg>
|
</svg>
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 263 B After Width: | Height: | Size: 1.4 KiB |
@@ -0,0 +1,165 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 1254 1254" role="img" aria-label="FabledCurator">
|
||||||
|
<title>FabledCurator</title>
|
||||||
|
<!-- Traced from the source raster, then re-painted from the design tokens:
|
||||||
|
gold -> accent.curator #A87338, glove -> text.parchment #E8E4D8. The
|
||||||
|
original plate was a warm brown (#1B1105), NOT the app obsidian, so it
|
||||||
|
is dropped entirely — the mark sits on whatever surface hosts it and the
|
||||||
|
frame interior shows that surface through. Do not reintroduce a
|
||||||
|
background rect. Too detailed below ~48px: use favicon.svg there. -->
|
||||||
|
<g transform="translate(0,1254) scale(0.1,-0.1)" fill="#A87338" stroke="none">
|
||||||
|
<path d="M5900 12310 c-1934 -145 -3621 -1109 -4645 -2655 -1080 -1631 -1218
|
||||||
|
-3942 -340 -5694 978 -1952 3013 -3175 5285 -3176 389 0 474 5 760 41 2363
|
||||||
|
299 4374 1950 4995 4099 307 1062 261 2393 -120 3503 -693 2017 -2533 3520
|
||||||
|
-4690 3831 -347 50 -934 74 -1245 51z m745 -220 c2580 -167 4716 -2034 5170
|
||||||
|
-4520 289 -1587 -50 -3083 -969 -4280 -1592 -2071 -4404 -2832 -6832 -1847
|
||||||
|
-379 153 -528 230 -532 272 -7 83 -97 361 -202 625 -317 800 -808 1466 -1325
|
||||||
|
1796 -175 112 -163 111 -325 31 -151 -75 -153 -80 -55 -136 594 -337 1122
|
||||||
|
-1023 1484 -1923 101 -252 103 -250 -106 -108 -1075 724 -1897 1902 -2207
|
||||||
|
3160 -281 1138 -174 2503 279 3565 505 1183 1429 2155 2611 2748 734 368 1453
|
||||||
|
560 2324 620 114 8 541 6 685 -3z M6222 11867 c-54 -109 -138 -190 -252 -244
|
||||||
|
-44 -21 -80 -41 -80 -44 0 -4 37 -24 83 -46 129 -63 217 -155 265 -279 25 -63
|
||||||
|
38 -62 66 7 57 141 180 255 329 303 30 10 19 20 -56 54 -134 60 -227 154 -275
|
||||||
|
278 -27 68 -33 65 -80 -29z M5470 11590 c-2341 -327 -4149 -2143 -4370 -4389
|
||||||
|
-11 -117 -8 -125 46 -136 50 -9 60 4 68 93 120 1359 922 2698 2099 3503 671
|
||||||
|
459 1567 790 2288 846 88 7 102 16 97 65 -4 42 -34 45 -228 18z M6843 11599
|
||||||
|
c-36 -36 -8 -89 47 -89 105 0 456 -65 709 -130 776 -202 1571 -624 2098 -1112
|
||||||
|
62 -57 63 -58 122 -58 102 0 97 10 -74 158 -726 626 -1646 1053 -2605 1206
|
||||||
|
-219 35 -282 41 -297 25z M7855 10996 c-63 -64 -96 -114 -109 -170 -7 -27 -9
|
||||||
|
-28 -50 -23 -61 9 -109 -8 -145 -49 -48 -54 -42 -67 43 -108 185 -89 270 -345
|
||||||
|
181 -544 -28 -63 -28 -63 2 -99 39 -48 43 -49 86 -10 34 32 36 36 27 68 -61
|
||||||
|
210 106 482 311 506 27 3 54 12 60 19 34 42 -49 139 -127 149 -50 7 -50 7 -62
|
||||||
|
76 -17 99 -100 249 -137 249 -9 0 -44 -29 -80 -64z M7369 10610 c-60 -10 -136
|
||||||
|
-49 -234 -120 -215 -155 -399 -202 -543 -140 -52 23 -112 78 -112 105 0 7 -7
|
||||||
|
21 -16 29 -42 43 -114 -13 -114 -88 0 -97 102 -161 311 -197 198 -34 306 -24
|
||||||
|
490 47 46 18 46 18 92 -30 44 -46 88 -60 78 -25 -49 169 188 238 269 78 35
|
||||||
|
-68 25 -84 -32 -49 -45 27 -121 27 -148 0 -25 -25 -17 -46 24 -60 18 -6 48
|
||||||
|
-26 66 -45 36 -37 107 -65 166 -65 62 0 111 131 94 252 -29 203 -204 341 -391
|
||||||
|
308z M5710 10589 c-215 -36 -300 -314 -163 -533 17 -27 17 -29 -1 -40 -116
|
||||||
|
-68 -166 -195 -120 -304 57 -138 229 -168 289 -52 38 74 -4 170 -74 170 -12 0
|
||||||
|
-21 7 -21 15 0 20 52 20 90 0 131 -68 45 -361 -118 -405 -74 -19 -87 -61 -46
|
||||||
|
-142 67 -132 71 -285 20 -683 -20 -154 -43 -332 -52 -395 -8 -63 -17 -127 -20
|
||||||
|
-143 -5 -23 -1 -30 26 -45 18 -9 54 -34 80 -54 36 -27 51 -34 57 -25 10 13 11
|
||||||
|
22 68 457 35 267 88 663 140 1039 42 307 23 421 -85 513 -40 35 -51 58 -27 58
|
||||||
|
28 0 107 -61 129 -99 41 -72 49 -56 64 126 6 68 4 84 -14 120 -57 110 -16 243
|
||||||
|
74 243 34 0 30 -23 -6 -37 -76 -29 -82 -173 -8 -173 34 0 34 13 3 -225 -14
|
||||||
|
-104 -36 -278 -50 -385 -14 -107 -48 -357 -76 -555 -27 -198 -57 -412 -65
|
||||||
|
-475 -8 -63 -31 -232 -51 -375 -20 -143 -36 -282 -36 -310 -1 -53 58 -214 78
|
||||||
|
-215 6 0 18 58 28 133 10 72 38 281 62 462 25 182 61 445 80 585 19 140 55
|
||||||
|
408 80 595 25 187 55 408 66 490 11 83 24 154 29 159 5 5 109 -5 242 -24 556
|
||||||
|
-77 880 -123 1128 -161 146 -22 337 -51 425 -64 88 -13 322 -49 520 -79 198
|
||||||
|
-30 513 -78 700 -106 187 -28 350 -53 363 -56 31 -6 31 -3 -33 -565 -48 -428
|
||||||
|
-137 -1243 -180 -1659 -16 -145 -64 -594 -110 -1020 -14 -124 -36 -337 -50
|
||||||
|
-475 -14 -137 -28 -256 -30 -264 -6 -17 -45 -14 -298 24 -98 15 -184 24 -192
|
||||||
|
21 -22 -8 -61 -94 -49 -106 16 -16 632 -106 644 -94 9 9 41 267 75 614 17 170
|
||||||
|
33 325 100 955 24 231 58 553 75 715 31 294 142 1311 180 1636 34 298 33 317
|
||||||
|
-20 326 -58 10 -1143 178 -1443 224 -277 41 -274 41 -325 4 -53 -38 -53 -38
|
||||||
|
-105 17 -51 54 -44 52 -692 143 -510 72 -621 87 -789 107 -101 12 -150 22
|
||||||
|
-153 31 -9 23 4 25 118 13 62 -7 115 -8 119 -4 5 4 -34 58 -85 119 -180 214
|
||||||
|
-321 273 -565 233z M8170 10491 c-170 -56 -291 -270 -240 -425 24 -72 147 -96
|
||||||
|
240 -47 25 14 52 28 60 31 28 13 16 44 -25 68 -39 22 -52 23 -127 6 -59 -13
|
||||||
|
-10 69 63 106 130 67 273 -61 202 -181 -29 -49 22 -51 92 -3 47 31 43 32 122
|
||||||
|
-28 194 -147 622 -235 773 -158 95 49 107 170 20 215 -35 18 -45 14 -89 -37
|
||||||
|
-123 -141 -436 -34 -643 218 -163 200 -315 279 -448 235z M9741 10040 c-90
|
||||||
|
-24 -163 -63 -286 -155 -130 -97 -132 -93 62 -124 67 -11 141 -25 163 -31 51
|
||||||
|
-14 74 -2 91 45 17 46 39 46 39 -1 0 -39 -12 -67 -42 -101 -33 -36 -77 -313
|
||||||
|
-50 -313 5 0 19 11 32 24 39 42 170 75 170 43 0 -7 -19 -18 -43 -24 -126 -34
|
||||||
|
-187 -164 -216 -458 -6 -60 -20 -191 -31 -290 -44 -392 -101 -929 -140 -1310
|
||||||
|
-22 -220 -49 -479 -60 -575 -11 -96 -29 -265 -40 -375 -11 -110 -27 -260 -36
|
||||||
|
-333 -21 -174 0 -313 57 -386 23 -29 24 -46 4 -46 -9 0 -30 20 -47 45 -53 75
|
||||||
|
-64 52 -53 -118 11 -185 11 -221 -4 -254 -17 -42 -45 -45 -36 -4 15 67 -48 99
|
||||||
|
-235 120 -47 5 -104 13 -126 16 -53 9 -61 -4 -24 -40 32 -30 39 -55 17 -55 -8
|
||||||
|
0 -22 10 -33 21 -52 59 -288 135 -352 115 -21 -7 -147 -177 -147 -199 0 -15
|
||||||
|
13 -14 90 7 148 40 315 -9 315 -94 0 -33 26 -44 70 -31 44 13 80 52 80 87 0
|
||||||
|
40 24 27 80 -42 210 -258 551 -349 702 -188 95 101 116 316 45 455 -37 74 -44
|
||||||
|
99 -28 99 4 0 29 18 53 40 141 124 92 363 -84 411 -159 42 -259 -179 -118
|
||||||
|
-259 16 -9 30 -23 30 -29 0 -24 -49 -13 -83 17 -118 103 -48 355 120 431 89
|
||||||
|
40 102 73 59 150 -91 164 -95 351 -22 1044 26 248 58 564 72 704 36 360 77
|
||||||
|
503 176 616 54 62 49 93 -29 160 -89 76 -130 251 -80 342 21 40 89 77 123 69
|
||||||
|
33 -9 31 -36 -2 -36 -133 0 -118 -214 18 -251 187 -50 305 240 154 380 -33 31
|
||||||
|
-33 36 10 84 129 143 137 385 17 509 -89 91 -252 127 -402 88z M6187 10008
|
||||||
|
c-8 -14 -34 -172 -61 -383 -26 -198 -70 -522 -97 -720 -27 -198 -65 -477 -84
|
||||||
|
-620 -19 -143 -44 -325 -55 -405 -24 -176 -45 -353 -60 -520 -16 -173 -57
|
||||||
|
-544 -80 -728 -44 -338 -51 -425 -38 -438 7 -7 85 -24 173 -38 88 -14 372 -62
|
||||||
|
630 -105 536 -91 1299 -211 1336 -211 14 0 63 16 110 35 288 117 571 67 630
|
||||||
|
-112 13 -40 2 -37 248 -73 236 -35 226 -37 235 63 4 39 29 272 56 517 27 245
|
||||||
|
61 553 75 685 14 132 35 321 45 420 11 99 40 369 64 600 25 231 66 596 91 810
|
||||||
|
73 615 84 738 71 752 -6 6 -225 43 -486 82 -261 39 -608 92 -770 116 -952 146
|
||||||
|
-1345 203 -1770 260 -102 14 -199 27 -217 30 -26 5 -35 1 -46 -17z m598 -459
|
||||||
|
c116 -16 521 -75 900 -130 380 -54 850 -122 1045 -149 207 -30 360 -56 368
|
||||||
|
-63 10 -10 3 -92 -33 -412 -25 -220 -61 -537 -80 -705 -19 -168 -53 -465 -75
|
||||||
|
-660 -22 -195 -52 -456 -66 -580 -57 -525 -86 -746 -97 -757 -10 -10 -31 -10
|
||||||
|
-102 2 -90 15 -410 64 -1795 277 -412 63 -731 116 -739 123 -11 11 -7 58 23
|
||||||
|
296 20 156 67 529 106 829 38 300 90 700 115 890 25 190 66 505 91 700 48 377
|
||||||
|
50 383 98 375 17 -3 126 -19 241 -36z M7696 8744 c-121 -421 -482 -746 -868
|
||||||
|
-782 -60 -5 -81 -28 -38 -39 40 -10 160 -72 217 -112 265 -181 435 -496 468
|
||||||
|
-863 10 -102 24 -100 59 10 136 415 458 696 859 747 64 8 60 25 -17 58 -350
|
||||||
|
153 -580 495 -627 935 -14 130 -26 140 -53 46z M10390 9624 c-16 -42 -12 -105
|
||||||
|
8 -128 64 -71 251 -371 366 -584 279 -520 462 -1104 531 -1696 16 -142 23
|
||||||
|
-156 69 -156 63 0 66 16 37 242 -109 836 -404 1569 -903 2241 -86 115 -93 121
|
||||||
|
-108 81z M5657 6918 c-37 -73 -67 -146 -67 -162 0 -15 -20 -179 -45 -364 -37
|
||||||
|
-271 -43 -338 -33 -348 7 -7 153 -35 323 -64 171 -28 443 -75 605 -105 434
|
||||||
|
-78 1022 -175 1166 -191 60 -6 60 -6 110 36 27 24 50 49 52 56 4 16 8 16 -283
|
||||||
|
59 -680 102 -1828 300 -1839 317 -8 14 2 111 79 731 26 211 22 213 -68 35z
|
||||||
|
M1124 6843 c-33 -140 -181 -299 -326 -351 -63 -22 -61 -35 10 -61 126 -46 245
|
||||||
|
-167 298 -301 33 -84 45 -87 67 -16 40 125 154 250 283 308 89 40 91 49 17 77
|
||||||
|
-146 53 -255 176 -318 359 -8 21 -24 13 -31 -15z M11345 6824 c-8 -20 -15 -40
|
||||||
|
-15 -44 0 -21 -105 -162 -146 -195 -46 -38 -168 -105 -191 -105 -36 0 -3 -30
|
||||||
|
75 -67 132 -63 209 -146 268 -291 29 -71 39 -71 68 0 65 159 179 273 319 318
|
||||||
|
50 16 47 32 -12 53 -125 44 -246 162 -300 292 -35 82 -45 89 -66 39z M5443
|
||||||
|
6629 c-40 -46 -53 -68 -53 -93 0 -91 -103 -184 -173 -158 -47 18 -113 -55
|
||||||
|
-109 -120 2 -19 -4 -31 -19 -38 -30 -16 -64 -97 -94 -223 -30 -125 -23 -179
|
||||||
|
36 -250 105 -128 397 -130 570 -5 44 32 69 31 69 -3 0 -27 73 -101 91 -94 8 3
|
||||||
|
23 20 34 38 46 73 165 102 297 73 140 -31 251 -97 418 -251 162 -147 309 -196
|
||||||
|
461 -152 152 45 250 183 206 289 -34 82 -227 104 -274 31 -21 -32 25 -83 77
|
||||||
|
-83 40 0 39 -25 -1 -39 -139 -48 -317 64 -229 144 55 50 -44 81 -105 33 -23
|
||||||
|
-18 -30 -19 -48 -9 -129 81 -219 110 -511 162 -172 31 -207 32 -315 2 -22 -5
|
||||||
|
-23 -4 -16 26 3 17 5 31 3 31 -2 0 -46 9 -98 20 -142 30 -175 27 -217 -15 -19
|
||||||
|
-19 -39 -35 -44 -35 -19 0 -8 60 21 115 17 31 30 69 30 84 0 14 4 42 9 61 27
|
||||||
|
105 19 150 -20 104 -14 -17 -82 -54 -99 -54 -20 0 -10 27 16 41 70 37 123 151
|
||||||
|
148 319 20 134 15 138 -61 49z M1115 5844 c-19 -19 -18 -36 22 -269 67 -398
|
||||||
|
241 -929 406 -1239 49 -93 63 -104 105 -87 46 19 46 34 -3 131 -114 228 -240
|
||||||
|
575 -314 865 -46 182 -111 501 -111 543 0 62 -65 96 -105 56z M11312 5848 c-7
|
||||||
|
-7 -12 -26 -12 -43 0 -51 -48 -312 -86 -463 -293 -1183 -1186 -2343 -2304
|
||||||
|
-2992 -1397 -811 -3104 -938 -4572 -339 -97 40 -97 40 -148 14 -87 -44 -66
|
||||||
|
-59 249 -176 1105 -412 2330 -434 3476 -62 1535 498 2757 1657 3282 3113 115
|
||||||
|
322 240 900 203 945 -15 18 -71 20 -88 3z M7260 5633 c-76 -68 -38 -250 66
|
||||||
|
-313 18 -12 42 -19 51 -16 14 5 68 86 91 138 2 4 -15 18 -37 33 -59 37 -55 55
|
||||||
|
12 48 49 -5 57 -3 87 24 45 40 36 55 -53 86 -103 37 -176 37 -217 0z M8484
|
||||||
|
5146 c-81 -19 -146 -53 -223 -119 -68 -58 -115 -143 -461 -831 -106 -210 -141
|
||||||
|
-249 -325 -361 -267 -161 -503 -324 -564 -389 -53 -56 54 -28 549 145 554 194
|
||||||
|
522 173 700 469 198 331 438 660 628 862 72 78 72 78 -3 147 -86 79 -183 104
|
||||||
|
-301 77z M1620 3910 c0 -50 807 -1065 1163 -1465 56 -62 47 -20 -24 114 -246
|
||||||
|
463 -531 859 -786 1088 -180 163 -353 291 -353 263z M3579 2997 c-316 -89
|
||||||
|
-350 -543 -51 -680 162 -75 385 23 452 198 102 269 -137 557 -401 482z m168
|
||||||
|
-72 c194 -58 258 -317 117 -474 -174 -193 -475 -70 -474 194 1 200 173 335
|
||||||
|
357 280z M3596 2764 c-7 -61 -22 -96 -61 -143 -33 -39 -26 -51 31 -51 54 0 77
|
||||||
|
-10 125 -57 41 -39 44 -38 54 32 8 58 27 99 62 137 32 33 23 48 -28 48 -52 0
|
||||||
|
-97 20 -133 59 -40 41 -42 40 -50 -25z"/>
|
||||||
|
</g>
|
||||||
|
<g transform="translate(0,1254) scale(0.1,-0.1)" fill="#E8E4D8" stroke="none">
|
||||||
|
<path d="M5355 7946 c-40 -18 -64 -47 -110 -131 -159 -293 -388 -522 -783
|
||||||
|
-786 -172 -115 -170 -113 -244 -233 -34 -56 -172 -252 -308 -436 -398 -541
|
||||||
|
-402 -548 -645 -950 -208 -345 -310 -504 -368 -572 -80 -95 -241 -195 -372
|
||||||
|
-232 -138 -39 -138 -39 -177 -91 -35 -46 -47 -54 -155 -98 -229 -94 -393 -169
|
||||||
|
-393 -181 0 -7 16 -20 36 -30 240 -114 591 -436 850 -781 309 -409 618 -1048
|
||||||
|
770 -1588 14 -51 31 -95 37 -99 6 -3 49 26 96 66 170 145 370 281 630 431 137
|
||||||
|
79 156 94 159 118 4 32 -31 141 -99 310 -27 67 -49 137 -49 157 0 52 25 142
|
||||||
|
43 152 32 18 45 38 52 84 8 53 37 86 135 154 393 269 1017 301 1726 90 280
|
||||||
|
-84 322 -75 522 114 186 175 346 293 697 513 188 118 167 88 470 687 290 576
|
||||||
|
315 619 473 838 59 81 117 162 130 179 69 98 -43 209 -213 209 -172 0 -415
|
||||||
|
-121 -554 -277 -89 -98 -241 -341 -241 -384 0 -6 -38 -51 -84 -100 -47 -49
|
||||||
|
-113 -127 -147 -174 -50 -66 -75 -91 -119 -114 -72 -40 -300 -201 -314 -223
|
||||||
|
-9 -14 -4 -25 26 -55 44 -46 88 -111 88 -129 0 -26 -25 -14 -100 46 -97 78
|
||||||
|
-205 134 -317 164 -104 27 -117 34 -277 133 -112 70 -521 308 -851 496 -402
|
||||||
|
230 -627 313 -945 350 -97 11 -105 14 -108 35 -4 24 16 29 48 12 13 -7 33 -6
|
||||||
|
64 2 60 16 212 -3 344 -42 113 -34 126 -29 77 30 -34 40 -36 48 -31 89 3 26
|
||||||
|
10 96 15 156 13 172 62 341 126 435 28 41 63 96 77 121 17 29 70 82 142 143
|
||||||
|
355 299 509 573 510 906 0 301 -179 559 -339 486z m-1542 -4944 c201 -96 278
|
||||||
|
-319 183 -527 -125 -273 -529 -286 -663 -21 -169 334 154 703 480 548z M7135
|
||||||
|
5383 c-50 -44 -145 -90 -203 -99 -35 -5 -64 -22 -126 -70 -111 -88 -300 -177
|
||||||
|
-561 -265 -64 -21 -110 -42 -110 -50 0 -11 130 -93 248 -155 40 -21 153 -55
|
||||||
|
264 -81 17 -4 53 14 130 65 60 38 138 86 174 106 54 29 82 55 147 134 44 53
|
||||||
|
114 134 156 179 90 96 88 84 28 138 -27 24 -56 62 -66 84 -25 53 -34 55 -81
|
||||||
|
14z"/>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 12 KiB |
@@ -5,9 +5,13 @@
|
|||||||
<img src="/favicon.svg" alt="" class="fc-brand__glyph" width="22" height="22" />
|
<img src="/favicon.svg" alt="" class="fc-brand__glyph" width="22" height="22" />
|
||||||
<span class="fc-brand__text">FabledCurator</span>
|
<span class="fc-brand__text">FabledCurator</span>
|
||||||
</RouterLink>
|
</RouterLink>
|
||||||
<span class="fc-health" :title="health.label">
|
<RouterLink
|
||||||
|
:to="{ name: 'settings', query: { tab: 'system' } }"
|
||||||
|
class="fc-health" :title="health.label"
|
||||||
|
:aria-label="`System health: ${health.label}`"
|
||||||
|
>
|
||||||
<v-icon size="x-small" :color="health.color">{{ health.icon }}</v-icon>
|
<v-icon size="x-small" :color="health.color">{{ health.icon }}</v-icon>
|
||||||
</span>
|
</RouterLink>
|
||||||
<PipelineStatusChip />
|
<PipelineStatusChip />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
@@ -64,13 +68,15 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed, onBeforeUnmount, onMounted, ref } from 'vue'
|
import { computed, onBeforeUnmount, onMounted, onUnmounted, ref } from 'vue'
|
||||||
import { useRoute } from 'vue-router'
|
import { useRoute } from 'vue-router'
|
||||||
import router, { FRONT_DOOR } from '../router.js'
|
import router, { FRONT_DOOR } from '../router.js'
|
||||||
import { useSystemStore } from '../stores/system.js'
|
import { useSystemStore } from '../stores/system.js'
|
||||||
|
import { useSystemHealthStore } from '../stores/systemHealth.js'
|
||||||
import PipelineStatusChip from './PipelineStatusChip.vue'
|
import PipelineStatusChip from './PipelineStatusChip.vue'
|
||||||
|
|
||||||
const system = useSystemStore()
|
const system = useSystemStore()
|
||||||
|
const healthStore = useSystemHealthStore()
|
||||||
|
|
||||||
// Publish the nav's REAL height as --fc-nav-h so full-height workspaces
|
// Publish the nav's REAL height as --fc-nav-h so full-height workspaces
|
||||||
// (Explore/Subscriptions) and sticky sub-headers pin to it exactly instead of a
|
// (Explore/Subscriptions) and sticky sub-headers pin to it exactly instead of a
|
||||||
@@ -116,15 +122,55 @@ const settingsRoute = computed(() =>
|
|||||||
navRoutes.value.find(r => r.name === 'settings') || null
|
navRoutes.value.find(r => r.name === 'settings') || null
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// The dot beside the brand, and the only ambient signal that something in the
|
||||||
|
// stack has stopped (milestone 365).
|
||||||
|
//
|
||||||
|
// It used to read /api/health — a no-DB liveness check that proves the WEB
|
||||||
|
// container is serving and nothing else. Green there while the worker was dead
|
||||||
|
// is exactly what it looked like, and a green dot next to the product name is
|
||||||
|
// read as "everything is fine". It now reflects the whole-stack verdict.
|
||||||
|
//
|
||||||
|
// Deliberately re-using this element rather than adding a second indicator:
|
||||||
|
// there were already three partial surfaces (this, the pipeline chip, the
|
||||||
|
// Settings Activity tab) and a fourth would have made the question harder to
|
||||||
|
// answer, not easier. This is the one that already occupied the slot.
|
||||||
const health = computed(() => {
|
const health = computed(() => {
|
||||||
if (system.healthy === null) {
|
const overall = healthStore.overall
|
||||||
|
if (overall === null) {
|
||||||
return { icon: 'mdi-circle-outline', color: 'on-surface', label: 'checking…' }
|
return { icon: 'mdi-circle-outline', color: 'on-surface', label: 'checking…' }
|
||||||
}
|
}
|
||||||
if (system.healthy === true) {
|
if (overall === 'ok') {
|
||||||
return { icon: 'mdi-circle', color: 'success', label: 'healthy' }
|
return { icon: 'mdi-circle', color: 'success', label: 'All parts running' }
|
||||||
}
|
}
|
||||||
return { icon: 'mdi-alert-circle', color: 'error', label: 'unreachable' }
|
// Name what is wrong in the tooltip. "Something is unhealthy" sends someone
|
||||||
|
// hunting; "Scheduler has not checked in for 6 min" does not.
|
||||||
|
const worst = healthStore.problems[0]
|
||||||
|
const others = healthStore.problems.length - 1
|
||||||
|
const suffix = others > 0 ? ` (+${others} more)` : ''
|
||||||
|
if (overall === 'down') {
|
||||||
|
return {
|
||||||
|
icon: 'mdi-alert-circle', color: 'error',
|
||||||
|
label: (worst?.detail || 'A part has stopped') + suffix,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (overall === 'stale') {
|
||||||
|
return {
|
||||||
|
icon: 'mdi-alert', color: 'warning',
|
||||||
|
label: (worst?.detail || 'A part is quiet') + suffix,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { icon: 'mdi-help-circle-outline', color: 'on-surface', label: 'Health unknown' }
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const HEALTH_POLL_MS = 15_000
|
||||||
|
let healthTimer = null
|
||||||
|
onMounted(() => {
|
||||||
|
healthStore.refresh()
|
||||||
|
healthTimer = setInterval(() => {
|
||||||
|
if (!document.hidden) healthStore.refresh()
|
||||||
|
}, HEALTH_POLL_MS)
|
||||||
|
})
|
||||||
|
onUnmounted(() => { if (healthTimer) clearInterval(healthTimer) })
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
@@ -237,7 +283,14 @@ const health = computed(() => {
|
|||||||
display: flex;
|
display: flex;
|
||||||
align-items: center;
|
align-items: center;
|
||||||
flex-shrink: 0;
|
flex-shrink: 0;
|
||||||
|
/* A RouterLink since milestone 365 — it is the path to the Settings System
|
||||||
|
tab, not just an indicator. Reset the anchor so turning a span into a
|
||||||
|
link changed nothing about how the nav reads. */
|
||||||
|
text-decoration: none;
|
||||||
|
color: inherit;
|
||||||
|
border-radius: 50%;
|
||||||
}
|
}
|
||||||
|
.fc-health:hover { background: rgb(var(--v-theme-on-surface) / 0.12); }
|
||||||
.fc-nav-right {
|
.fc-nav-right {
|
||||||
flex: 1 1 0;
|
flex: 1 1 0;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
<template>
|
||||||
|
<div class="fc-empty">
|
||||||
|
<!-- The brand mark's second home (#387 A-side logo work, operator-confirmed).
|
||||||
|
It earns its place HERE and not on a populated feed: this is the first
|
||||||
|
screen a fresh install shows anyone, and it gives the on-ramp something
|
||||||
|
to be composed around rather than a bare line of prose plus buttons.
|
||||||
|
Large on purpose — logo.svg stops reading below ~48px, which is why the
|
||||||
|
22px nav slot has its own mark. No wrapping card: the file deliberately
|
||||||
|
carries no plate and is meant to sit on the page surface. Vendored
|
||||||
|
locally, so this renders on an install with no network — which is
|
||||||
|
exactly the state this screen appears in. -->
|
||||||
|
<img src="/logo.svg" alt="" class="fc-empty__mark" width="150" height="150" />
|
||||||
|
|
||||||
|
<!-- FIRST RUN: sources exist, nothing has landed yet. Telling someone to
|
||||||
|
add a source when they already have three and are mid-backfill is
|
||||||
|
worse than saying nothing at all. -->
|
||||||
|
<template v-if="hasSources">
|
||||||
|
<p class="fc-empty__lead">Nothing has arrived yet.</p>
|
||||||
|
<p class="fc-empty__sub">
|
||||||
|
{{ sourceCountLabel }} configured. The first check can take a while —
|
||||||
|
deep history is fetched in chunks.
|
||||||
|
</p>
|
||||||
|
<div class="fc-empty__actions">
|
||||||
|
<v-btn
|
||||||
|
size="small" variant="tonal" color="accent"
|
||||||
|
prepend-icon="mdi-progress-download"
|
||||||
|
:to="{ path: '/subscriptions', query: { tab: 'downloads' } }"
|
||||||
|
>See what's running</v-btn>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<!-- FRESH INSTALL: the on-ramp, in the order the steps actually depend on
|
||||||
|
each other — a source cannot fetch anything without a credential. -->
|
||||||
|
<template v-else>
|
||||||
|
<p class="fc-empty__lead">Nothing here yet.</p>
|
||||||
|
<p class="fc-empty__sub">
|
||||||
|
FabledCurator follows the creators you subscribe to and files what they
|
||||||
|
post. Two steps to start.
|
||||||
|
</p>
|
||||||
|
<div class="fc-empty__actions">
|
||||||
|
<v-btn
|
||||||
|
size="small" variant="tonal" color="accent" prepend-icon="mdi-key-variant"
|
||||||
|
:to="{ path: '/subscriptions', query: { tab: 'settings' } }"
|
||||||
|
>Add a credential</v-btn>
|
||||||
|
<v-btn
|
||||||
|
size="small" variant="text" prepend-icon="mdi-plus"
|
||||||
|
:to="{ path: '/subscriptions' }"
|
||||||
|
>Add a source</v-btn>
|
||||||
|
</div>
|
||||||
|
<p class="fc-empty__hint">
|
||||||
|
A credential comes first — a source can't fetch anything without your
|
||||||
|
logged-in session.
|
||||||
|
</p>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted } from 'vue'
|
||||||
|
import { storeToRefs } from 'pinia'
|
||||||
|
|
||||||
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
|
|
||||||
|
const store = useSourcesStore()
|
||||||
|
const { scheduleStatus: status } = storeToRefs(store)
|
||||||
|
|
||||||
|
// Absent status is treated as "fresh install", which is the safe way round:
|
||||||
|
// the on-ramp is useful to a first-run operator and merely redundant to an
|
||||||
|
// established one, whereas "see what's running" shown to someone with nothing
|
||||||
|
// configured is a dead end.
|
||||||
|
const hasSources = computed(() => (status.value?.total_sources || 0) > 0)
|
||||||
|
|
||||||
|
const sourceCountLabel = computed(() => {
|
||||||
|
const n = status.value?.total_sources || 0
|
||||||
|
return `${n} ${n === 1 ? 'source' : 'sources'}`
|
||||||
|
})
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
// The ribbon may already have loaded this on the front door; in Browse's
|
||||||
|
// Posts tab nothing has. Failure is swallowed — an empty feed must still
|
||||||
|
// explain itself when the status call is unavailable (rule #164).
|
||||||
|
if (!status.value) store.loadScheduleStatus().catch(() => {})
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-empty {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
text-align: center;
|
||||||
|
gap: 6px;
|
||||||
|
padding: 48px 16px 32px;
|
||||||
|
}
|
||||||
|
.fc-empty__mark {
|
||||||
|
/* Sits back a little: it frames the message rather than competing with it. */
|
||||||
|
opacity: 0.9;
|
||||||
|
margin-bottom: 8px;
|
||||||
|
}
|
||||||
|
.fc-empty__lead {
|
||||||
|
font-size: 1.05rem;
|
||||||
|
color: rgb(var(--v-theme-on-surface));
|
||||||
|
}
|
||||||
|
.fc-empty__sub, .fc-empty__hint {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
max-width: 34rem;
|
||||||
|
}
|
||||||
|
.fc-empty__hint { font-size: 0.8rem; margin-top: 4px; }
|
||||||
|
.fc-empty__actions {
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
justify-content: center;
|
||||||
|
gap: 8px;
|
||||||
|
margin-top: 10px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
<template>
|
||||||
|
<!-- Renders nothing at all until it has something true to say. A ribbon that
|
||||||
|
shows a skeleton or an error on the front door would make the app look
|
||||||
|
broken every cold load; the feed is the point, this is an aside. -->
|
||||||
|
<div v-if="status" class="fc-ribbon">
|
||||||
|
<span class="fc-ribbon__item">
|
||||||
|
<v-icon size="x-small">mdi-clock-outline</v-icon>
|
||||||
|
Checked {{ lastCheckedLabel }}
|
||||||
|
</span>
|
||||||
|
|
||||||
|
<RouterLink
|
||||||
|
v-if="failing" class="fc-ribbon__item fc-ribbon__item--err"
|
||||||
|
:to="{ path: '/subscriptions', query: { status: 'errors' } }"
|
||||||
|
>
|
||||||
|
<v-icon size="x-small">mdi-alert-circle-outline</v-icon>
|
||||||
|
{{ failing }} {{ failing === 1 ? 'source is' : 'sources are' }} failing
|
||||||
|
</RouterLink>
|
||||||
|
|
||||||
|
<!-- The reason this ribbon exists. Phase A made "we can't see this
|
||||||
|
creator's posts" a durable fact; without a line here it stays buried
|
||||||
|
three clicks into Subscriptions, which is exactly where nobody looks
|
||||||
|
until they already suspect something. -->
|
||||||
|
<RouterLink
|
||||||
|
v-if="noAccess" class="fc-ribbon__item fc-ribbon__item--gated"
|
||||||
|
:to="{ path: '/subscriptions', query: { status: 'no_access' } }"
|
||||||
|
>
|
||||||
|
<v-icon size="x-small">mdi-lock-outline</v-icon>
|
||||||
|
{{ noAccess }} you can't see
|
||||||
|
</RouterLink>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { computed, onMounted } from 'vue'
|
||||||
|
import { storeToRefs } from 'pinia'
|
||||||
|
|
||||||
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
|
import { formatRelative } from '../../utils/date.js'
|
||||||
|
|
||||||
|
const store = useSourcesStore()
|
||||||
|
const { scheduleStatus: status } = storeToRefs(store)
|
||||||
|
|
||||||
|
const failing = computed(() => status.value?.failing_sources || 0)
|
||||||
|
const noAccess = computed(() => status.value?.no_access_sources || 0)
|
||||||
|
const lastCheckedLabel = computed(() =>
|
||||||
|
formatRelative(status.value?.last_tick_at, { nullText: 'never' }),
|
||||||
|
)
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
// Swallowed on purpose: this is an aside on the front door, and the feed
|
||||||
|
// must render whether or not the status call succeeds (rule #164 — a
|
||||||
|
// genuinely-external-ish fact degrades to absent, it never gates the page).
|
||||||
|
store.loadScheduleStatus().catch(() => {})
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-ribbon {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
gap: 4px 14px;
|
||||||
|
padding: 2px 0 10px;
|
||||||
|
font-size: 0.78rem;
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
.fc-ribbon__item {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
color: inherit;
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
/* Only the actionable items take a colour, so a healthy instance reads as one
|
||||||
|
quiet grey line rather than a status dashboard. */
|
||||||
|
.fc-ribbon__item--err { color: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-ribbon__item--gated { color: rgb(var(--v-theme-info)); }
|
||||||
|
.fc-ribbon__item--err:hover,
|
||||||
|
.fc-ribbon__item--gated:hover { text-decoration: underline; }
|
||||||
|
</style>
|
||||||
@@ -6,6 +6,19 @@
|
|||||||
<v-chip size="x-small" variant="tonal">
|
<v-chip size="x-small" variant="tonal">
|
||||||
{{ post.source?.platform ?? 'filesystem import' }}
|
{{ post.source?.platform ?? 'filesystem import' }}
|
||||||
</v-chip>
|
</v-chip>
|
||||||
|
<!-- The honesty marker (#388 E2). Discord doesn't publish posts, so FC
|
||||||
|
groups a creator's drop and writes the post itself. This chip is the
|
||||||
|
only thing standing between "FC assembled this" and the card reading
|
||||||
|
as something the artist authored — it must never be conditional on
|
||||||
|
anything but the flag, and it says what it was built FROM so the
|
||||||
|
claim is checkable rather than just disclosed. -->
|
||||||
|
<v-chip
|
||||||
|
v-if="synthesized" size="x-small" variant="outlined"
|
||||||
|
class="fc-post-card__synthetic" :title="synthesisTitle"
|
||||||
|
>
|
||||||
|
<v-icon icon="mdi-auto-fix" size="x-small" start />
|
||||||
|
grouped by FabledCurator
|
||||||
|
</v-chip>
|
||||||
<RouterLink
|
<RouterLink
|
||||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||||
class="fc-post-card__artist"
|
class="fc-post-card__artist"
|
||||||
@@ -14,6 +27,12 @@
|
|||||||
<span v-if="totalImages" class="fc-post-card__meta">
|
<span v-if="totalImages" class="fc-post-card__meta">
|
||||||
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
||||||
</span>
|
</span>
|
||||||
|
<!-- Only on a grouping that has actually grown. An ordinary post can
|
||||||
|
never show this, and a grouping that has not grown says nothing —
|
||||||
|
an "updated" label that is always present teaches you to ignore it. -->
|
||||||
|
<span v-if="grewAt" class="fc-post-card__meta fc-post-card__grew">
|
||||||
|
· updated {{ grewRelative }}
|
||||||
|
</span>
|
||||||
<v-spacer />
|
<v-spacer />
|
||||||
<PostSeriesMenu :post="post" />
|
<PostSeriesMenu :post="post" />
|
||||||
<v-btn
|
<v-btn
|
||||||
@@ -59,6 +78,13 @@
|
|||||||
|
|
||||||
<div class="fc-post-card__text">
|
<div class="fc-post-card__text">
|
||||||
<h3 v-if="displayTitle" class="fc-post-card__title">{{ displayTitle }}</h3>
|
<h3 v-if="displayTitle" class="fc-post-card__title">{{ displayTitle }}</h3>
|
||||||
|
<!-- A synthetic post has no title on purpose (inventing one is the one
|
||||||
|
place this feature could put words in a creator's mouth), so name
|
||||||
|
it for what it is rather than leaking `fc-drop:<message id>`. -->
|
||||||
|
<h3
|
||||||
|
v-else-if="synthesized"
|
||||||
|
class="fc-post-card__title fc-post-card__title--missing"
|
||||||
|
>{{ synthesisTitle }}</h3>
|
||||||
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
<h3 v-else class="fc-post-card__title fc-post-card__title--missing">
|
||||||
Post {{ post.external_post_id }}
|
Post {{ post.external_post_id }}
|
||||||
</h3>
|
</h3>
|
||||||
@@ -102,6 +128,25 @@
|
|||||||
@click="toggleDesc"
|
@click="toggleDesc"
|
||||||
>{{ descExpanded ? 'Show less' : 'Show more' }}</button>
|
>{{ descExpanded ? 'Show less' : 'Show more' }}</button>
|
||||||
|
|
||||||
|
<!-- #388 E5: the accepted announcement link, both directions. Only
|
||||||
|
ACCEPTED ones reach the payload, so anything rendered here is
|
||||||
|
something the operator agreed to — never a proposal. -->
|
||||||
|
<!-- A location with no name/path is "the current route, with these
|
||||||
|
query params" — so this works from Latest or Browse alike without
|
||||||
|
the card reaching for useRoute(), which it has no other reason to
|
||||||
|
know about. -->
|
||||||
|
<div v-if="associations.length" class="fc-post-card__assoc">
|
||||||
|
<RouterLink
|
||||||
|
v-for="a in associations" :key="a.id"
|
||||||
|
:to="{ query: { post_id: a.post_id } }"
|
||||||
|
class="fc-post-card__assoc-link"
|
||||||
|
>
|
||||||
|
<v-icon icon="mdi-link-variant" size="x-small" />
|
||||||
|
{{ a.role === 'announces'
|
||||||
|
? 'The full set is in Discord' : 'Announced on Patreon' }}
|
||||||
|
</RouterLink>
|
||||||
|
</div>
|
||||||
|
|
||||||
<!-- Off-platform file-host links (mega/gdrive/…) found in the body.
|
<!-- Off-platform file-host links (mega/gdrive/…) found in the body.
|
||||||
Shown once expanded; clickable now, auto-downloaded by the worker
|
Shown once expanded; clickable now, auto-downloaded by the worker
|
||||||
slice. -->
|
slice. -->
|
||||||
@@ -163,6 +208,18 @@ const images = computed(() => props.post.thumbnails || [])
|
|||||||
const totalImages = computed(() => images.value.length + (props.post.thumbnails_more || 0))
|
const totalImages = computed(() => images.value.length + (props.post.thumbnails_more || 0))
|
||||||
const plainTitle = computed(() => toPlainText(props.post.post_title))
|
const plainTitle = computed(() => toPlainText(props.post.post_title))
|
||||||
|
|
||||||
|
// #388 E2. Non-null `synthesized_by` means FC authored this row by grouping a
|
||||||
|
// creator's drop; `synthesis` carries what it was built from. Read defensively
|
||||||
|
// — a post fetched before the field existed, or any surface that composes a
|
||||||
|
// post dict by hand, must degrade to "not synthetic" rather than throw.
|
||||||
|
const synthesized = computed(() => Boolean(props.post.synthesized_by))
|
||||||
|
const messageCount = computed(() => props.post.synthesis?.message_count ?? 0)
|
||||||
|
const synthesisTitle = computed(() => {
|
||||||
|
const n = messageCount.value
|
||||||
|
if (!n) return 'Grouped from Discord'
|
||||||
|
return `Grouped from ${n} Discord message${n === 1 ? '' : 's'}`
|
||||||
|
})
|
||||||
|
|
||||||
const hero = computed(() => images.value[0])
|
const hero = computed(() => images.value[0])
|
||||||
// The thumbnail strip spans the hero's full width (CSS grid, equal columns),
|
// The thumbnail strip spans the hero's full width (CSS grid, equal columns),
|
||||||
// rather than a fixed 3-cell cap. Show up to RAIL_MAX cells; when there are
|
// rather than a fixed 3-cell cap. Show up to RAIL_MAX cells; when there are
|
||||||
@@ -189,16 +246,29 @@ const moreCount = computed(() => {
|
|||||||
const railCols = computed(() => rail.value.length + (moreCount.value > 0 ? 1 : 0))
|
const railCols = computed(() => rail.value.length + (moreCount.value > 0 ? 1 : 0))
|
||||||
|
|
||||||
const sortDateIso = computed(() => props.post.post_date || props.post.downloaded_at)
|
const sortDateIso = computed(() => props.post.post_date || props.post.downloaded_at)
|
||||||
|
|
||||||
|
// #388 E3. A grouping stays OPEN, so its own date and its latest activity are
|
||||||
|
// different facts. The card keeps showing when the drop STARTED — that is the
|
||||||
|
// post's identity — and reports growth separately, because "this post is from
|
||||||
|
// Tuesday but gained images this morning" is the whole signal that chat
|
||||||
|
// content is trickling in.
|
||||||
|
const grewAt = computed(() => (synthesized.value ? props.post.last_grew_at : null))
|
||||||
|
// #388 E5. Accepted links only — a pending proposal lives in the review queue,
|
||||||
|
// never beside the artwork. Defaults to [] so a post dict from before the
|
||||||
|
// feature (or composed by hand) renders without a link rather than throwing.
|
||||||
|
const associations = computed(() => props.post.associations || [])
|
||||||
|
const grewRelative = computed(() => (grewAt.value ? relativeFrom(grewAt.value) : ''))
|
||||||
const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString())
|
const absoluteDate = computed(() => new Date(sortDateIso.value).toLocaleString())
|
||||||
const relativeDate = computed(() => {
|
function relativeFrom (iso) {
|
||||||
const then = new Date(sortDateIso.value).getTime()
|
const then = new Date(iso).getTime()
|
||||||
const diff = (Date.now() - then) / 1000
|
const diff = (Date.now() - then) / 1000
|
||||||
if (diff < 60) return `${Math.floor(diff)}s ago`
|
if (diff < 60) return `${Math.floor(diff)}s ago`
|
||||||
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
if (diff < 3600) return `${Math.floor(diff / 60)}m ago`
|
||||||
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
if (diff < 86400) return `${Math.floor(diff / 3600)}h ago`
|
||||||
if (diff < 86400 * 30) return `${Math.floor(diff / 86400)}d ago`
|
if (diff < 86400 * 30) return `${Math.floor(diff / 86400)}d ago`
|
||||||
return new Date(sortDateIso.value).toLocaleDateString()
|
return new Date(iso).toLocaleDateString()
|
||||||
})
|
}
|
||||||
|
const relativeDate = computed(() => relativeFrom(sortDateIso.value))
|
||||||
|
|
||||||
// --- images → post-scoped modal ---------------------------------------
|
// --- images → post-scoped modal ---------------------------------------
|
||||||
async function fullImageIds () {
|
async function fullImageIds () {
|
||||||
@@ -337,6 +407,27 @@ function formatBytes (n) {
|
|||||||
font-weight: 600;
|
font-weight: 600;
|
||||||
}
|
}
|
||||||
.fc-post-card__artist:hover { color: rgb(var(--v-theme-accent)); }
|
.fc-post-card__artist:hover { color: rgb(var(--v-theme-accent)); }
|
||||||
|
|
||||||
|
/* Quiet, not decorative: the marker has to be legible on every card without
|
||||||
|
turning a synthetic post into the loudest thing in the feed. */
|
||||||
|
.fc-post-card__synthetic {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Growth is news, so it gets the accent the rest of the meta line doesn't —
|
||||||
|
but it is still the meta line, not a badge competing with the artwork. */
|
||||||
|
.fc-post-card__grew { color: rgb(var(--v-theme-accent)); }
|
||||||
|
|
||||||
|
.fc-post-card__assoc { margin-top: 8px; }
|
||||||
|
.fc-post-card__assoc-link {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
font-size: 0.8125rem;
|
||||||
|
color: rgb(var(--v-theme-accent));
|
||||||
|
text-decoration: none;
|
||||||
|
}
|
||||||
|
.fc-post-card__assoc-link:hover { text-decoration: underline; }
|
||||||
.fc-post-card__date,
|
.fc-post-card__date,
|
||||||
.fc-post-card__meta { white-space: nowrap; }
|
.fc-post-card__meta { white-space: nowrap; }
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,134 @@
|
|||||||
|
<template>
|
||||||
|
<MaintenanceTile
|
||||||
|
icon="mdi-image-multiple"
|
||||||
|
title="Discord drop grouping"
|
||||||
|
blurb="Group a creator's variant drop into one post FC writes itself."
|
||||||
|
>
|
||||||
|
<div v-if="store.settings">
|
||||||
|
<div class="text-caption fc-muted mb-3">
|
||||||
|
Discord is a delivery channel, not a publisher — one message isn't one
|
||||||
|
post. When this is on, FC groups a creator's variant drop (same piece,
|
||||||
|
different hair colour or outfit) into a single post it authors, with the
|
||||||
|
messages' text as the body. Grouped posts are always marked as FC's own,
|
||||||
|
and deleting one returns its messages to the feed unchanged.
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-switch
|
||||||
|
v-model="local.discord_grouping_enabled" color="accent" hide-details
|
||||||
|
density="compact" label="Group Discord drops into posts"
|
||||||
|
@update:model-value="onSave"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<v-row class="mt-2">
|
||||||
|
<v-col cols="12" sm="6">
|
||||||
|
<SettingNumberField
|
||||||
|
v-model="local.discord_group_max_distance"
|
||||||
|
label="Max visual distance" :min="0" :max="1" :step="0.01"
|
||||||
|
density="comfortable" max-width="none"
|
||||||
|
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||||
|
/>
|
||||||
|
<div class="text-caption fc-muted mt-1">
|
||||||
|
0 is identical, 1 is unrelated. Lower groups less. Raising this is
|
||||||
|
the fix when a drop comes out scattered across several posts —
|
||||||
|
but raise it slowly: too high merges pieces that only look alike.
|
||||||
|
</div>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="12" sm="6">
|
||||||
|
<SettingNumberField
|
||||||
|
v-model="local.discord_group_window_minutes"
|
||||||
|
label="Drop window (minutes)" :min="1" :step="5"
|
||||||
|
density="comfortable" max-width="none"
|
||||||
|
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||||
|
/>
|
||||||
|
<div class="text-caption fc-muted mt-1">
|
||||||
|
The quiet gap that ends a drop, measured between consecutive
|
||||||
|
messages — so variants trickling out over an evening stay one post.
|
||||||
|
This is the setting doing most of the work: without it, everything
|
||||||
|
an artist ever drew of one character would collapse into one post.
|
||||||
|
</div>
|
||||||
|
</v-col>
|
||||||
|
</v-row>
|
||||||
|
|
||||||
|
<!-- E3: a grouping stays open, and these decide for how long and how
|
||||||
|
loudly it announces that it grew. -->
|
||||||
|
<div class="text-caption fc-muted mt-4 mb-2">
|
||||||
|
A grouped post stays <strong>open</strong>: variants the creator adds
|
||||||
|
later join the existing post instead of starting a new one, and its
|
||||||
|
text grows with them.
|
||||||
|
</div>
|
||||||
|
<v-row>
|
||||||
|
<v-col cols="12" sm="4">
|
||||||
|
<SettingNumberField
|
||||||
|
v-model="local.discord_group_close_after_hours"
|
||||||
|
label="Stays open for (hours)" :min="1" :step="24"
|
||||||
|
density="comfortable" max-width="none"
|
||||||
|
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||||
|
/>
|
||||||
|
<div class="text-caption fc-muted mt-1">
|
||||||
|
How long after its last addition a post still accepts new variants.
|
||||||
|
Not the drop window above — that cuts one session into drops; this
|
||||||
|
decides how late a follow-up can still join. Too long and the same
|
||||||
|
character coming round again months later gets absorbed by mistake.
|
||||||
|
</div>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="12" sm="4">
|
||||||
|
<SettingNumberField
|
||||||
|
v-model="local.discord_group_resurface_min_images"
|
||||||
|
label="New images before it resurfaces" :min="1" :step="1"
|
||||||
|
density="comfortable" max-width="none"
|
||||||
|
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||||
|
/>
|
||||||
|
<div class="text-caption fc-muted mt-1">
|
||||||
|
Growth smaller than this updates the post where it sits instead of
|
||||||
|
moving it back to the top of the feed.
|
||||||
|
</div>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="12" sm="4">
|
||||||
|
<SettingNumberField
|
||||||
|
v-model="local.discord_group_resurface_cooldown_hours"
|
||||||
|
label="Resurface at most every (hours)" :min="0" :step="1"
|
||||||
|
density="comfortable" max-width="none"
|
||||||
|
:disabled="!local.discord_grouping_enabled" @change="onSave"
|
||||||
|
/>
|
||||||
|
<div class="text-caption fc-muted mt-1">
|
||||||
|
Together with the count above, this is what stops a post that gains
|
||||||
|
an image a day from living permanently at the top of the feed.
|
||||||
|
</div>
|
||||||
|
</v-col>
|
||||||
|
</v-row>
|
||||||
|
</div>
|
||||||
|
<div v-else><v-skeleton-loader type="paragraph" /></div>
|
||||||
|
</MaintenanceTile>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
// #388 E2. Every value here is operator-tunable because the quality bar is a
|
||||||
|
// judgement no test can settle: grouping too greedy merges distinct pieces,
|
||||||
|
// too shy leaves a drop scattered. The two failure modes are not symmetric —
|
||||||
|
// scattered is visible and fixable, a wrong merge is a post asserting that
|
||||||
|
// unrelated art belongs together — so the shipped default sits on the tight
|
||||||
|
// side and this card is how it gets loosened.
|
||||||
|
import { reactive, watch } from 'vue'
|
||||||
|
|
||||||
|
import { useSettingSave } from '../../composables/useSettingSave.js'
|
||||||
|
import { useMLStore } from '../../stores/ml.js'
|
||||||
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
|
import SettingNumberField from '../common/SettingNumberField.vue'
|
||||||
|
|
||||||
|
const store = useMLStore()
|
||||||
|
const { save } = useSettingSave(store.patchSettings)
|
||||||
|
const local = reactive({})
|
||||||
|
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
||||||
|
|
||||||
|
function onSave() {
|
||||||
|
save({
|
||||||
|
discord_grouping_enabled: Boolean(local.discord_grouping_enabled),
|
||||||
|
discord_group_max_distance: Number(local.discord_group_max_distance),
|
||||||
|
discord_group_window_minutes: Number(local.discord_group_window_minutes),
|
||||||
|
discord_group_close_after_hours: Number(local.discord_group_close_after_hours),
|
||||||
|
discord_group_resurface_min_images: Number(local.discord_group_resurface_min_images),
|
||||||
|
discord_group_resurface_cooldown_hours:
|
||||||
|
Number(local.discord_group_resurface_cooldown_hours),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
</script>
|
||||||
@@ -14,6 +14,8 @@
|
|||||||
<div class="fc-tile-stack">
|
<div class="fc-tile-stack">
|
||||||
<ImportFiltersForm />
|
<ImportFiltersForm />
|
||||||
<TranslationCard />
|
<TranslationCard />
|
||||||
|
<DiscordGroupingCard />
|
||||||
|
<PostAssociationsCard />
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
@@ -80,6 +82,8 @@ import DbMaintenanceCard from './DbMaintenanceCard.vue'
|
|||||||
import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
|
import VideoEmbeddingCard from './VideoEmbeddingCard.vue'
|
||||||
import CropProposersCard from './CropProposersCard.vue'
|
import CropProposersCard from './CropProposersCard.vue'
|
||||||
import HeadsCard from './HeadsCard.vue'
|
import HeadsCard from './HeadsCard.vue'
|
||||||
|
import DiscordGroupingCard from './DiscordGroupingCard.vue'
|
||||||
|
import PostAssociationsCard from './PostAssociationsCard.vue'
|
||||||
import GpuAgentCard from './GpuAgentCard.vue'
|
import GpuAgentCard from './GpuAgentCard.vue'
|
||||||
import AliasTable from './AliasTable.vue'
|
import AliasTable from './AliasTable.vue'
|
||||||
import BackupCard from './BackupCard.vue'
|
import BackupCard from './BackupCard.vue'
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
<template>
|
||||||
|
<MaintenanceTile
|
||||||
|
icon="mdi-link-variant"
|
||||||
|
title="Announcement links"
|
||||||
|
blurb="Pair a Patreon teaser with the Discord drop it announced."
|
||||||
|
>
|
||||||
|
<div class="text-caption fc-muted mb-3">
|
||||||
|
Some creators post a cropped fragment on Patreon to say the real thing has
|
||||||
|
landed in their Discord. FC proposes those pairs; nothing is linked until
|
||||||
|
you accept one. A wrong link would tell you two different pieces are the
|
||||||
|
same, so this stays a suggestion.
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<v-switch
|
||||||
|
v-model="enabled" color="accent" hide-details density="compact"
|
||||||
|
label="Look for announcement pairs"
|
||||||
|
@update:model-value="store.setEnabled"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<v-row class="mt-2">
|
||||||
|
<v-col cols="12" sm="6">
|
||||||
|
<SettingNumberField
|
||||||
|
v-model="threshold" label="Confidence needed"
|
||||||
|
:min="0" :max="1" :step="0.05"
|
||||||
|
density="comfortable" max-width="none"
|
||||||
|
:disabled="!enabled" @change="store.setThreshold(Number(threshold))"
|
||||||
|
/>
|
||||||
|
<div class="text-caption fc-muted mt-1">
|
||||||
|
A pair always needs <strong>two</strong> reasons — being close in time
|
||||||
|
and the post itself mentioning Discord. Neither is enough alone at any
|
||||||
|
setting at or above 0.55, which is what stops a busy posting day from
|
||||||
|
producing false pairs.
|
||||||
|
</div>
|
||||||
|
</v-col>
|
||||||
|
<v-col cols="12" sm="6">
|
||||||
|
<SettingNumberField
|
||||||
|
v-model="windowHours" label="How far apart (hours)"
|
||||||
|
:min="1" :step="1"
|
||||||
|
density="comfortable" max-width="none"
|
||||||
|
:disabled="!enabled" @change="store.setWindowHours(Number(windowHours))"
|
||||||
|
/>
|
||||||
|
<div class="text-caption fc-muted mt-1">
|
||||||
|
The announcement exists in order to point at the drop, so the two are
|
||||||
|
usually minutes to hours apart.
|
||||||
|
</div>
|
||||||
|
</v-col>
|
||||||
|
</v-row>
|
||||||
|
|
||||||
|
<div class="d-flex align-center mt-4 mb-2" style="gap:12px">
|
||||||
|
<strong class="text-body-2">
|
||||||
|
{{ store.proposals.length }} waiting for review
|
||||||
|
</strong>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn size="small" variant="text" :loading="store.loading" @click="store.rescan">
|
||||||
|
Scan now
|
||||||
|
</v-btn>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div v-if="!store.proposals.length" class="text-caption fc-muted">
|
||||||
|
Nothing proposed. That is the expected state most of the time — pairs only
|
||||||
|
appear when a post both lands near a drop and says it is about Discord.
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-for="p in store.proposals" :key="p.id"
|
||||||
|
class="fc-assoc"
|
||||||
|
>
|
||||||
|
<div class="fc-assoc__body">
|
||||||
|
<RouterLink :to="{ name: 'latest', query: { post_id: p.announcement_post_id } }">
|
||||||
|
post {{ p.announcement_post_id }}
|
||||||
|
</RouterLink>
|
||||||
|
<span class="fc-assoc__arrow">announced</span>
|
||||||
|
<RouterLink :to="{ name: 'latest', query: { post_id: p.payload_post_id } }">
|
||||||
|
drop {{ p.payload_post_id }}
|
||||||
|
</RouterLink>
|
||||||
|
<!-- The per-signal breakdown, not just the total: "why did it suggest
|
||||||
|
this" is the question the operator actually has, and a lone score
|
||||||
|
cannot answer it. -->
|
||||||
|
<div class="text-caption fc-muted">
|
||||||
|
{{ Math.round(p.score * 100) }}% ·
|
||||||
|
timing {{ Math.round((p.signals?.proximity ?? 0) * 100) }}% ·
|
||||||
|
says so {{ Math.round((p.signals?.declared ?? 0) * 100) }}%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<v-btn size="small" variant="tonal" @click="store.accept(p.id)">Link</v-btn>
|
||||||
|
<v-btn size="small" variant="text" @click="store.dismiss(p.id)">Dismiss</v-btn>
|
||||||
|
</div>
|
||||||
|
</MaintenanceTile>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted, ref, watch } from 'vue'
|
||||||
|
import { RouterLink } from 'vue-router'
|
||||||
|
|
||||||
|
import { usePostAssociationsStore } from '../../stores/postAssociations.js'
|
||||||
|
import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||||
|
import SettingNumberField from '../common/SettingNumberField.vue'
|
||||||
|
|
||||||
|
const store = usePostAssociationsStore()
|
||||||
|
const enabled = ref(true)
|
||||||
|
const threshold = ref(0.6)
|
||||||
|
const windowHours = ref(24)
|
||||||
|
|
||||||
|
watch(() => store.enabled, (v) => { enabled.value = v }, { immediate: true })
|
||||||
|
watch(() => store.threshold, (v) => { threshold.value = v }, { immediate: true })
|
||||||
|
watch(() => store.windowHours, (v) => { windowHours.value = v }, { immediate: true })
|
||||||
|
|
||||||
|
onMounted(async () => {
|
||||||
|
// Both swallow their own failures: a settings read that fails should not
|
||||||
|
// leave the queue unrendered, and vice versa.
|
||||||
|
await Promise.allSettled([store.loadSettings(), store.load()])
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-assoc {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
padding: 8px 0;
|
||||||
|
border-top: 1px solid rgba(var(--v-theme-on-surface), 0.12);
|
||||||
|
}
|
||||||
|
.fc-assoc__body { flex: 1; min-width: 0; }
|
||||||
|
.fc-assoc__arrow {
|
||||||
|
color: rgb(var(--v-theme-on-surface-variant));
|
||||||
|
margin: 0 6px;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
<template>
|
||||||
|
<!-- A Settings tab, not a page of its own (operator 2026-09-02): the first
|
||||||
|
cut hung this off the health dot alone, which is a target you have to
|
||||||
|
already suspect something to look for. Settings is where someone goes
|
||||||
|
to ask the instance about itself, so it lives beside Activity. -->
|
||||||
|
<div>
|
||||||
|
<div class="d-flex align-center mb-1">
|
||||||
|
<v-spacer />
|
||||||
|
<span class="fc-sys__checked">
|
||||||
|
{{ store.checkedAt ? `checked ${formatRelative(store.checkedAt)}` : 'checking…' }}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<p class="fc-sys__lede text-body-2 mb-5">
|
||||||
|
Every moving part of FabledCurator and whether it is still checking in.
|
||||||
|
Parts are learned as they appear, so anything that has run at least once
|
||||||
|
stays listed — that is what lets a stopped one be noticed rather than
|
||||||
|
simply vanishing.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<v-alert
|
||||||
|
v-if="store.lastError" type="error" variant="tonal" density="compact" class="mb-4"
|
||||||
|
>
|
||||||
|
Could not reach FabledCurator: {{ store.lastError }}
|
||||||
|
</v-alert>
|
||||||
|
|
||||||
|
<v-card v-else variant="flat" class="fc-sys__card">
|
||||||
|
<div v-if="!store.parts.length" class="pa-6 text-center fc-sys__muted">
|
||||||
|
Still gathering — this fills in on the first check.
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div
|
||||||
|
v-for="part in store.parts" :key="part.key"
|
||||||
|
class="fc-sys__row" :class="`fc-sys__row--${part.state}`"
|
||||||
|
>
|
||||||
|
<span class="fc-sys__dot" :class="`fc-sys__dot--${part.state}`" />
|
||||||
|
|
||||||
|
<div class="fc-sys__body">
|
||||||
|
<div class="fc-sys__name">
|
||||||
|
{{ part.name }}
|
||||||
|
<span class="fc-sys__kind">{{ kindLabel(part.kind) }}</span>
|
||||||
|
</div>
|
||||||
|
<!-- The sentence, not just a chip. At the moment someone is deciding
|
||||||
|
whether to go and open Portainer, "has not checked in for 6 min"
|
||||||
|
is the thing that answers them. -->
|
||||||
|
<div class="fc-sys__detail">{{ part.detail }}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="fc-sys__meta">
|
||||||
|
<div v-if="part.last_seen_at" :title="part.last_seen_at">
|
||||||
|
seen {{ formatRelative(part.last_seen_at) }}
|
||||||
|
</div>
|
||||||
|
<div v-if="part.latency_ms != null">{{ part.latency_ms }} ms</div>
|
||||||
|
<div v-if="part.queues?.length" class="fc-sys__queues">{{ part.queues.join(', ') }}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</v-card>
|
||||||
|
|
||||||
|
<p v-if="store.thresholds" class="fc-sys__foot text-caption mt-4">
|
||||||
|
A part is called stale after
|
||||||
|
{{ Math.round(store.thresholds.stale_after_seconds / 60) }} min without a
|
||||||
|
check-in and treated as stopped after
|
||||||
|
{{ Math.round(store.thresholds.down_after_seconds / 60) }} min. The window
|
||||||
|
is deliberately wide: a rolling deploy briefly runs two of a service and
|
||||||
|
then neither, and an indicator that reddened on every update would stop
|
||||||
|
being read.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { onMounted } from 'vue'
|
||||||
|
|
||||||
|
import { useSystemHealthStore } from '../../stores/systemHealth.js'
|
||||||
|
import { formatRelative } from '../../utils/date.js'
|
||||||
|
|
||||||
|
const store = useSystemHealthStore()
|
||||||
|
|
||||||
|
function kindLabel(kind) {
|
||||||
|
if (kind === 'celery') return 'background worker'
|
||||||
|
if (kind === 'agent') return 'GPU agent'
|
||||||
|
if (kind === 'datastore') return 'datastore'
|
||||||
|
return kind
|
||||||
|
}
|
||||||
|
|
||||||
|
// No timer of its own. TopNav already polls this same pinia store every 15s
|
||||||
|
// for the health dot, and it is mounted on every route this tab is reachable
|
||||||
|
// from — a second interval here would just double the request rate for a 5s
|
||||||
|
// freshness gain. v-window keeps a visited item MOUNTED (hidden, not
|
||||||
|
// destroyed), so a local timer would also have kept firing behind Maintenance.
|
||||||
|
// One refresh on open, so arriving at the tab doesn't wait out the nav's tick.
|
||||||
|
onMounted(() => { store.refresh() })
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-sys__lede, .fc-sys__muted, .fc-sys__checked, .fc-sys__foot {
|
||||||
|
color: rgb(var(--v-theme-on-surface) / 0.66);
|
||||||
|
}
|
||||||
|
.fc-sys__checked { font-size: 0.78rem; }
|
||||||
|
.fc-sys__card { background: rgb(var(--v-theme-on-surface) / 0.04); }
|
||||||
|
|
||||||
|
.fc-sys__row {
|
||||||
|
display: flex; align-items: center; gap: 12px;
|
||||||
|
padding: 12px 16px;
|
||||||
|
border-bottom: 1px solid rgb(var(--v-theme-on-surface) / 0.08);
|
||||||
|
}
|
||||||
|
.fc-sys__row:last-child { border-bottom: 0; }
|
||||||
|
|
||||||
|
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
|
||||||
|
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
|
||||||
|
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
|
||||||
|
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
|
||||||
|
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
|
||||||
|
|
||||||
|
.fc-sys__body { min-width: 0; flex: 1 1 auto; }
|
||||||
|
.fc-sys__name { font-weight: 600; }
|
||||||
|
.fc-sys__kind {
|
||||||
|
margin-left: 8px; font-weight: 400; font-size: 0.72rem; text-transform: uppercase;
|
||||||
|
letter-spacing: 0.04em; color: rgb(var(--v-theme-on-surface) / 0.5);
|
||||||
|
}
|
||||||
|
.fc-sys__detail { font-size: 0.82rem; color: rgb(var(--v-theme-on-surface) / 0.72); }
|
||||||
|
|
||||||
|
.fc-sys__meta {
|
||||||
|
text-align: right; font-size: 0.75rem; flex: 0 0 auto;
|
||||||
|
font-variant-numeric: tabular-nums; color: rgb(var(--v-theme-on-surface) / 0.6);
|
||||||
|
}
|
||||||
|
.fc-sys__queues { opacity: 0.75; }
|
||||||
|
</style>
|
||||||
@@ -32,6 +32,13 @@
|
|||||||
v-if="e.live.errors" class="fc-active__count fc-active__count--err"
|
v-if="e.live.errors" class="fc-active__count fc-active__count--err"
|
||||||
title="errors"
|
title="errors"
|
||||||
>✕ {{ e.live.errors }}</span>
|
>✕ {{ e.live.errors }}</span>
|
||||||
|
<!-- Tier-gated posts (#874): not an error — the walk is working, the
|
||||||
|
content just isn't ours. Shown only when non-zero so a healthy
|
||||||
|
run stays uncluttered. -->
|
||||||
|
<span
|
||||||
|
v-if="e.live.gated" class="fc-active__count fc-active__count--gated"
|
||||||
|
title="posts skipped — no access at your tier"
|
||||||
|
>🔒 {{ e.live.gated }}</span>
|
||||||
<span class="fc-active__count fc-active__count--posts" title="posts scanned">
|
<span class="fc-active__count fc-active__count--posts" title="posts scanned">
|
||||||
{{ e.live.posts }} posts</span>
|
{{ e.live.posts }} posts</span>
|
||||||
</span>
|
</span>
|
||||||
@@ -141,6 +148,9 @@ function elapsed (startedIso) {
|
|||||||
}
|
}
|
||||||
.fc-active__count { color: rgb(var(--v-theme-on-surface-variant)); }
|
.fc-active__count { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
.fc-active__count--err { color: rgb(var(--v-theme-error)); }
|
.fc-active__count--err { color: rgb(var(--v-theme-error)); }
|
||||||
|
/* Matches FailingSourcesCard's severity map, which already colours
|
||||||
|
tier_limited as 'info' — no-access is information, not a failure. */
|
||||||
|
.fc-active__count--gated { color: rgb(var(--v-theme-info)); }
|
||||||
.fc-active__count--posts { opacity: 0.7; }
|
.fc-active__count--posts { opacity: 0.7; }
|
||||||
@keyframes fc-active-pulse {
|
@keyframes fc-active-pulse {
|
||||||
0%, 100% { opacity: 1; transform: scale(1); }
|
0%, 100% { opacity: 1; transform: scale(1); }
|
||||||
|
|||||||
@@ -27,7 +27,10 @@ import { computed } from 'vue'
|
|||||||
import { formatRelative } from '../../utils/date.js'
|
import { formatRelative } from '../../utils/date.js'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
// { last_tick_at, next_due_at, due_now, auto_sources } | null
|
// { last_tick_at, next_due_at, due_now, auto_sources,
|
||||||
|
// failing_sources, no_access_sources, platform_cooldowns } | null
|
||||||
|
// This bar renders the scheduling half; the ingestion counts are read by the
|
||||||
|
// front door's FeedStatusRibbon (#387 B3) off the same payload.
|
||||||
status: { type: Object, default: null },
|
status: { type: Object, default: null },
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,7 @@
|
|||||||
<div class="fc-health-tip">
|
<div class="fc-health-tip">
|
||||||
<div>Last checked: {{ lastCheckedText }}</div>
|
<div>Last checked: {{ lastCheckedText }}</div>
|
||||||
<div v-if="nextCheckText">Next check: {{ nextCheckText }}</div>
|
<div v-if="nextCheckText">Next check: {{ nextCheckText }}</div>
|
||||||
|
<div v-if="noAccess" class="fc-health-tip__gated">{{ noAccessText }}</div>
|
||||||
<div v-if="(source.consecutive_failures || 0) > 0">
|
<div v-if="(source.consecutive_failures || 0) > 0">
|
||||||
Failures: {{ source.consecutive_failures }}
|
Failures: {{ source.consecutive_failures }}
|
||||||
</div>
|
</div>
|
||||||
@@ -29,14 +30,29 @@ const props = defineProps({
|
|||||||
warningThreshold: { type: Number, default: 5 },
|
warningThreshold: { type: Number, default: 5 },
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const noAccess = computed(() => props.source.error_type === 'tier_limited')
|
||||||
|
|
||||||
const level = computed(() => {
|
const level = computed(() => {
|
||||||
if (!props.source.last_checked_at) return 'unchecked'
|
if (!props.source.last_checked_at) return 'unchecked'
|
||||||
const f = props.source.consecutive_failures || 0
|
const f = props.source.consecutive_failures || 0
|
||||||
if (f === 0) return 'healthy'
|
// No-access outranks 'healthy' but is NOT a failure grade: the walk worked,
|
||||||
|
// the content simply isn't ours. Checked after failures so a source that is
|
||||||
|
// genuinely erroring still reads as erroring.
|
||||||
|
if (f === 0) return noAccess.value ? 'no-access' : 'healthy'
|
||||||
if (f < props.warningThreshold) return 'warning'
|
if (f < props.warningThreshold) return 'warning'
|
||||||
return 'critical'
|
return 'critical'
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// The count comes from the last walk's run_stats and is only joined in by the
|
||||||
|
// list endpoint, so it can legitimately be absent — say the state without it
|
||||||
|
// rather than printing a fabricated zero.
|
||||||
|
const noAccessText = computed(() => {
|
||||||
|
const n = props.source.tier_gated_count
|
||||||
|
return n
|
||||||
|
? `${n} post${n === 1 ? '' : 's'} you don't have access to`
|
||||||
|
: "Some posts are behind a tier you don't hold"
|
||||||
|
})
|
||||||
|
|
||||||
const ariaLabel = computed(() => `source health: ${level.value}`)
|
const ariaLabel = computed(() => `source health: ${level.value}`)
|
||||||
|
|
||||||
const lastCheckedText = computed(() => formatRelative(props.source.last_checked_at))
|
const lastCheckedText = computed(() => formatRelative(props.source.last_checked_at))
|
||||||
@@ -63,6 +79,9 @@ const truncatedError = computed(() => {
|
|||||||
}
|
}
|
||||||
.fc-health-dot--unchecked { background-color: rgb(var(--v-theme-on-surface-variant)); opacity: 0.5; }
|
.fc-health-dot--unchecked { background-color: rgb(var(--v-theme-on-surface-variant)); opacity: 0.5; }
|
||||||
.fc-health-dot--healthy { background-color: rgb(var(--v-theme-success, 76 175 80)); }
|
.fc-health-dot--healthy { background-color: rgb(var(--v-theme-success, 76 175 80)); }
|
||||||
|
/* Matches the 'info' severity FailingSourcesCard already assigns tier_limited —
|
||||||
|
deliberately not a warning/error hue: nothing is broken. */
|
||||||
|
.fc-health-dot--no-access { background-color: rgb(var(--v-theme-info, 33 150 243)); }
|
||||||
.fc-health-dot--warning { background-color: rgb(var(--v-theme-warning, 255 167 38)); }
|
.fc-health-dot--warning { background-color: rgb(var(--v-theme-warning, 255 167 38)); }
|
||||||
.fc-health-dot--critical { background-color: rgb(var(--v-theme-error, 244 67 54)); }
|
.fc-health-dot--critical { background-color: rgb(var(--v-theme-error, 244 67 54)); }
|
||||||
|
|
||||||
@@ -70,6 +89,9 @@ const truncatedError = computed(() => {
|
|||||||
font-size: 0.85rem;
|
font-size: 0.85rem;
|
||||||
line-height: 1.4;
|
line-height: 1.4;
|
||||||
}
|
}
|
||||||
|
.fc-health-tip__gated {
|
||||||
|
color: rgb(var(--v-theme-info, 33 150 243));
|
||||||
|
}
|
||||||
.fc-health-tip__err {
|
.fc-health-tip__err {
|
||||||
margin-top: 0.25rem;
|
margin-top: 0.25rem;
|
||||||
color: rgb(var(--v-theme-error, 244 67 54));
|
color: rgb(var(--v-theme-error, 244 67 54));
|
||||||
|
|||||||
@@ -48,6 +48,20 @@
|
|||||||
<span class="fc-source-row__err-text">{{ source.last_error }}</span>
|
<span class="fc-source-row__err-text">{{ source.last_error }}</span>
|
||||||
</v-tooltip>
|
</v-tooltip>
|
||||||
</v-chip>
|
</v-chip>
|
||||||
|
<!-- No access (#387 A3). Sits directly after the failure chip and before
|
||||||
|
the backfill states: a source we can't see is the more useful thing
|
||||||
|
to say about it than which walk phase it's in, and unlike those it
|
||||||
|
doesn't resolve on its own. Info-coloured, never error — the walk
|
||||||
|
worked, the content just isn't ours. -->
|
||||||
|
<v-chip
|
||||||
|
v-else-if="source.error_type === 'tier_limited'"
|
||||||
|
size="x-small" color="info" variant="tonal" label
|
||||||
|
prepend-icon="mdi-lock-outline"
|
||||||
|
>{{ source.tier_gated_count ? `${source.tier_gated_count} gated` : 'No access' }}
|
||||||
|
<v-tooltip activator="parent" location="top" max-width="480">
|
||||||
|
<span>{{ noAccessTip }}</span>
|
||||||
|
</v-tooltip>
|
||||||
|
</v-chip>
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'running'"
|
v-else-if="source.backfill_state === 'running'"
|
||||||
size="x-small" color="info" variant="tonal" label
|
size="x-small" color="info" variant="tonal" label
|
||||||
@@ -79,6 +93,8 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
|
import { computed } from 'vue'
|
||||||
|
|
||||||
import SourceActions from './SourceActions.vue'
|
import SourceActions from './SourceActions.vue'
|
||||||
import SourceHealthDot from './SourceHealthDot.vue'
|
import SourceHealthDot from './SourceHealthDot.vue'
|
||||||
import { formatRelative } from '../../utils/date.js'
|
import { formatRelative } from '../../utils/date.js'
|
||||||
@@ -88,6 +104,19 @@ const props = defineProps({
|
|||||||
checking: { type: Boolean, default: false },
|
checking: { type: Boolean, default: false },
|
||||||
warningThreshold: { type: Number, default: 5 },
|
warningThreshold: { type: Number, default: 5 },
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Says what to DO about it, not just what it is — the action here is the
|
||||||
|
// operator's subscription, not anything FC can retry. The count is only joined
|
||||||
|
// in by the list endpoint, so phrase it without one when it's absent rather
|
||||||
|
// than rendering a fabricated zero.
|
||||||
|
const noAccessTip = computed(() => {
|
||||||
|
const n = props.source.tier_gated_count
|
||||||
|
const what = n
|
||||||
|
? `The last check skipped ${n} post${n === 1 ? '' : 's'}`
|
||||||
|
: 'The last check skipped posts'
|
||||||
|
return `${what} this account can't view. Nothing is broken — your `
|
||||||
|
+ 'subscription tier does not grant access to them.'
|
||||||
|
})
|
||||||
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
|
const emit = defineEmits(['edit', 'remove', 'toggle', 'check', 'backfill', 'recover', 'recapture'])
|
||||||
|
|
||||||
function onToggleEnabled(value) {
|
function onToggleEnabled(value) {
|
||||||
|
|||||||
@@ -348,6 +348,12 @@ const STATUS_OPTIONS = [
|
|||||||
{ title: 'Enabled', value: 'enabled' },
|
{ title: 'Enabled', value: 'enabled' },
|
||||||
{ title: 'Disabled', value: 'disabled' },
|
{ title: 'Disabled', value: 'disabled' },
|
||||||
{ title: 'Has errors', value: 'errors' },
|
{ title: 'Has errors', value: 'errors' },
|
||||||
|
// #387 A3: no-access is deliberately its own filter and NOT folded into
|
||||||
|
// "Has errors" — nothing failed, and it is the only status here whose fix is
|
||||||
|
// the operator's subscription rather than anything FC can retry. Without a
|
||||||
|
// filter a gated source is invisible in a long list, since it correctly
|
||||||
|
// stays out of the failing rollup.
|
||||||
|
{ title: 'No access', value: 'no_access' },
|
||||||
{ title: 'Stale', value: 'stale' },
|
{ title: 'Stale', value: 'stale' },
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -362,7 +368,19 @@ const platformsStore = usePlatformsStore()
|
|||||||
const importStore = useImportStore()
|
const importStore = useImportStore()
|
||||||
|
|
||||||
const search = ref('')
|
const search = ref('')
|
||||||
const statusFilter = ref('all')
|
// URL-addressable (#387 B3) so the front-door status ribbon can link straight
|
||||||
|
// to "the sources this number is about" — a count that lands you on an
|
||||||
|
// unfiltered list makes the reader do the filtering the ribbon just did.
|
||||||
|
// Mirrors how artistFilter already reads from route.query below.
|
||||||
|
const statusFilter = computed({
|
||||||
|
get: () => route.query.status || 'all',
|
||||||
|
set: (v) => {
|
||||||
|
const q = { ...route.query }
|
||||||
|
if (!v || v === 'all') delete q.status
|
||||||
|
else q.status = v
|
||||||
|
router.replace({ query: q })
|
||||||
|
},
|
||||||
|
})
|
||||||
const needsAttention = ref(false)
|
const needsAttention = ref(false)
|
||||||
const expanded = ref([])
|
const expanded = ref([])
|
||||||
const selected = ref([])
|
const selected = ref([])
|
||||||
@@ -506,6 +524,7 @@ function groupMatchesStatus(g, status) {
|
|||||||
if (status === 'enabled') return g.sources.some((s) => s.enabled)
|
if (status === 'enabled') return g.sources.some((s) => s.enabled)
|
||||||
if (status === 'disabled') return g.sources.every((s) => !s.enabled)
|
if (status === 'disabled') return g.sources.every((s) => !s.enabled)
|
||||||
if (status === 'errors') return g.sources.some((s) => (s.consecutive_failures || 0) > 0)
|
if (status === 'errors') return g.sources.some((s) => (s.consecutive_failures || 0) > 0)
|
||||||
|
if (status === 'no_access') return g.sources.some((s) => s.error_type === 'tier_limited')
|
||||||
if (status === 'stale') return g.sources.some((s) => !s.last_checked_at)
|
if (status === 'stale') return g.sources.some((s) => !s.last_checked_at)
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-1
@@ -9,15 +9,37 @@ import SeriesView from './views/SeriesView.vue'
|
|||||||
import SeriesManageView from './views/SeriesManageView.vue'
|
import SeriesManageView from './views/SeriesManageView.vue'
|
||||||
import SeriesReaderView from './views/SeriesReaderView.vue'
|
import SeriesReaderView from './views/SeriesReaderView.vue'
|
||||||
import SubscriptionsView from './views/SubscriptionsView.vue'
|
import SubscriptionsView from './views/SubscriptionsView.vue'
|
||||||
|
import PostsView from './views/PostsView.vue'
|
||||||
|
|
||||||
// The application's front door. `/` redirects here. Changing the front door
|
// The application's front door. `/` redirects here. Changing the front door
|
||||||
// is a one-line edit (e.g. '/gallery' or '/tags').
|
// is a one-line edit (e.g. '/gallery' or '/tags').
|
||||||
export const FRONT_DOOR = '/showcase'
|
//
|
||||||
|
// Moved from '/showcase' to '/latest' (milestone #387 B1). Showcase answers
|
||||||
|
// "show me something" — a random TABLESAMPLE, lean-back, and it can never tell
|
||||||
|
// you anything is wrong. The feed answers "what arrived?", which is a question
|
||||||
|
// the operator has every day, and it is the only view where a failing source
|
||||||
|
// shows up on its own as a creator who has gone quiet. The per-artist "new
|
||||||
|
// since last visit" badges (#597) were a workaround for this view not being
|
||||||
|
// the door. Showcase is demoted to a nav entry, not removed.
|
||||||
|
export const FRONT_DOOR = '/latest'
|
||||||
|
|
||||||
const routes = [
|
const routes = [
|
||||||
// Root is a redirect only — no meta.title so it stays out of the nav.
|
// Root is a redirect only — no meta.title so it stays out of the nav.
|
||||||
{ path: '/', redirect: FRONT_DOOR },
|
{ path: '/', redirect: FRONT_DOOR },
|
||||||
|
|
||||||
|
// The front door (#387 B1): the post feed as its own surface. Reuses
|
||||||
|
// PostsView unchanged — it is already a self-contained feed (own container,
|
||||||
|
// own store, infinite scroll, deep-link anchoring), and Browse only ever
|
||||||
|
// wrapped it in a tab strip. Mounting it directly IS the difference the
|
||||||
|
// promotion was after: a door you arrive at, not a hub you navigate out of.
|
||||||
|
//
|
||||||
|
// No stickyChrome: unlike Browse/Gallery/Settings this view has no sticky
|
||||||
|
// sub-header for the nav to butt against, so the nav keeps its normal fade.
|
||||||
|
// `props` turns on the ingestion-status ribbon (#387 B3). Only here: inside
|
||||||
|
// Browse's Posts tab the same view renders without it.
|
||||||
|
{ path: '/latest', name: 'latest', component: PostsView, props: { statusRibbon: true },
|
||||||
|
meta: { title: 'Latest', navOrder: 5 } },
|
||||||
|
|
||||||
// FC-2: image backbone
|
// FC-2: image backbone
|
||||||
{ path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase', navOrder: 10 } },
|
{ path: '/showcase', name: 'showcase', component: ShowcaseView, meta: { title: 'Showcase', navOrder: 10 } },
|
||||||
{ path: '/gallery', name: 'gallery', component: GalleryView, meta: { title: 'Gallery', navOrder: 20, stickyChrome: true } },
|
{ path: '/gallery', name: 'gallery', component: GalleryView, meta: { title: 'Gallery', navOrder: 20, stickyChrome: true } },
|
||||||
@@ -45,6 +67,13 @@ const routes = [
|
|||||||
|
|
||||||
// Settings — config, pinned to the right of the nav (TopNav special-cases it).
|
// Settings — config, pinned to the right of the nav (TopNav special-cases it).
|
||||||
{ path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings', stickyChrome: true } },
|
{ path: '/settings', name: 'settings', component: SettingsView, meta: { title: 'Settings', stickyChrome: true } },
|
||||||
|
// System health is a Settings TAB, not a route of its own (operator
|
||||||
|
// 2026-09-02). It first shipped as /system reachable only from the health
|
||||||
|
// dot, which is a target you have to already suspect something to go
|
||||||
|
// looking for. Settings is where someone goes to ask the instance about
|
||||||
|
// itself. The path stays as a redirect so the dot's old link, and any
|
||||||
|
// bookmark from that build, still land somewhere real.
|
||||||
|
{ path: '/system', name: 'system', redirect: () => ({ name: 'settings', query: { tab: 'system' } }) },
|
||||||
|
|
||||||
// The old standalone paths now redirect into the Browse hub, preserving any
|
// The old standalone paths now redirect into the Browse hub, preserving any
|
||||||
// deep-link query (e.g. /posts?post_id=N → /browse?tab=posts&post_id=N). The
|
// deep-link query (e.g. /posts?post_id=N → /browse?tab=posts&post_id=N). The
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
import { defineStore } from 'pinia'
|
||||||
|
import { ref } from 'vue'
|
||||||
|
|
||||||
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
import { useAsyncAction } from '../composables/useAsyncAction.js'
|
||||||
|
import { toast } from '../utils/toast.js'
|
||||||
|
|
||||||
|
// Backs the announcement review queue (#388 E5): "this Patreon post announced
|
||||||
|
// that Discord drop". Confirm-only, deliberately — a wrongly-asserted link
|
||||||
|
// tells the operator two different pieces are one, which is worse than no link
|
||||||
|
// at all, so accept is the ONLY thing that makes a link real. Mirrors
|
||||||
|
// seriesSuggestions (FC-6.3), which is the same shape for the same reason.
|
||||||
|
export const usePostAssociationsStore = defineStore('postAssociations', () => {
|
||||||
|
const api = useApi()
|
||||||
|
const proposals = ref([])
|
||||||
|
const enabled = ref(true)
|
||||||
|
const threshold = ref(0.6)
|
||||||
|
const windowHours = ref(24)
|
||||||
|
const { loading, error, run } = useAsyncAction({ errorAs: 'message' })
|
||||||
|
|
||||||
|
async function load () {
|
||||||
|
await run(async () => {
|
||||||
|
const body = await api.get('/api/posts/associations')
|
||||||
|
proposals.value = body.items || []
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadSettings () {
|
||||||
|
const s = await api.get('/api/settings/import')
|
||||||
|
enabled.value = s.discord_link_enabled
|
||||||
|
threshold.value = s.discord_link_threshold
|
||||||
|
windowHours.value = s.discord_link_window_hours
|
||||||
|
}
|
||||||
|
|
||||||
|
async function saveSettings (patch) {
|
||||||
|
await api.patch('/api/settings/import', { body: patch })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setEnabled (v) {
|
||||||
|
enabled.value = v
|
||||||
|
await saveSettings({ discord_link_enabled: v })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setThreshold (v) {
|
||||||
|
threshold.value = v
|
||||||
|
await saveSettings({ discord_link_threshold: v })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function setWindowHours (v) {
|
||||||
|
windowHours.value = v
|
||||||
|
await saveSettings({ discord_link_window_hours: v })
|
||||||
|
}
|
||||||
|
|
||||||
|
async function accept (id) {
|
||||||
|
try {
|
||||||
|
await api.post(`/api/posts/associations/${id}/accept`, {})
|
||||||
|
proposals.value = proposals.value.filter(p => p.id !== id)
|
||||||
|
toast({ text: 'Linked', type: 'success' })
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Link failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function dismiss (id) {
|
||||||
|
try {
|
||||||
|
await api.post(`/api/posts/associations/${id}/dismiss`, {})
|
||||||
|
proposals.value = proposals.value.filter(p => p.id !== id)
|
||||||
|
} catch (e) {
|
||||||
|
toast({ text: `Dismiss failed: ${e.message}`, type: 'error' })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rescan () {
|
||||||
|
await api.post('/api/posts/associations/rescan', {})
|
||||||
|
await load()
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
proposals, enabled, threshold, windowHours, loading, error,
|
||||||
|
load, loadSettings, setEnabled, setThreshold, setWindowHours,
|
||||||
|
accept, dismiss, rescan
|
||||||
|
}
|
||||||
|
})
|
||||||
@@ -4,7 +4,12 @@ import { useApi } from '../composables/useApi.js'
|
|||||||
|
|
||||||
export const useSystemStore = defineStore('system', () => {
|
export const useSystemStore = defineStore('system', () => {
|
||||||
const api = useApi()
|
const api = useApi()
|
||||||
const healthy = ref(null) // null=unknown, true=ok, false=down
|
// NOT what the nav dot reads any more (milestone 365): that is the
|
||||||
|
// whole-stack verdict in systemHealth.js. /api/health only proves the web
|
||||||
|
// container is serving, which is why a green dot here sat happily beside a
|
||||||
|
// dead worker. refreshHealth() is still called — it is also how build/version
|
||||||
|
// info arrives — so this stays as its by-product rather than its purpose.
|
||||||
|
const healthy = ref(null)
|
||||||
// What the instance says it is. Since milestone 318 stopped publishing
|
// What the instance says it is. Since milestone 318 stopped publishing
|
||||||
// version image tags, this is the only answer to "which build is this?" —
|
// version image tags, this is the only answer to "which build is this?" —
|
||||||
// there is no registry name left to check it against.
|
// there is no registry name left to check it against.
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import { defineStore } from 'pinia'
|
||||||
|
import { computed, ref } from 'vue'
|
||||||
|
|
||||||
|
import { useApi } from '../composables/useApi.js'
|
||||||
|
|
||||||
|
// Whole-stack health: is every part of FabledCurator running (milestone 365)?
|
||||||
|
//
|
||||||
|
// Distinct from `system.js`, which polls /api/health — a no-DB liveness check
|
||||||
|
// that only proves the web container is serving. That endpoint answers "can I
|
||||||
|
// reach the API"; this one answers "is anything broken", which is the question
|
||||||
|
// a green dot beside the brand was already being read as answering.
|
||||||
|
//
|
||||||
|
// Also distinct from `systemActivity.js`, which is about what the pipeline is
|
||||||
|
// DOING — queue depths, running tasks, failures. Running and alive are
|
||||||
|
// different questions and they fail independently: a perfectly idle stack with
|
||||||
|
// a dead worker looks identical to a healthy one on the activity surfaces.
|
||||||
|
export const useSystemHealthStore = defineStore('systemHealth', () => {
|
||||||
|
const api = useApi()
|
||||||
|
|
||||||
|
const overall = ref(null) // null until the first answer: unknown ≠ ok
|
||||||
|
const parts = ref([])
|
||||||
|
const checkedAt = ref(null)
|
||||||
|
const thresholds = ref(null) // server-owned, so the UI keeps no second copy
|
||||||
|
const lastError = ref(null)
|
||||||
|
|
||||||
|
async function refresh() {
|
||||||
|
try {
|
||||||
|
const body = await api.get('/api/system/health')
|
||||||
|
overall.value = body.overall
|
||||||
|
parts.value = body.parts || []
|
||||||
|
checkedAt.value = body.checked_at
|
||||||
|
thresholds.value = body.thresholds || null
|
||||||
|
lastError.value = null
|
||||||
|
} catch (e) {
|
||||||
|
// The endpoint is built never to fail because a dependency failed, so a
|
||||||
|
// throw here means the API itself is unreachable — which is its own kind
|
||||||
|
// of unhealthy and must not be shown as "ok".
|
||||||
|
lastError.value = e.message
|
||||||
|
overall.value = 'unknown'
|
||||||
|
}
|
||||||
|
return overall.value
|
||||||
|
}
|
||||||
|
|
||||||
|
// The parts worth naming in a tooltip — everything that is not ok, worst
|
||||||
|
// first. The endpoint already sorts that way.
|
||||||
|
const problems = computed(() => parts.value.filter(p => p.state !== 'ok'))
|
||||||
|
|
||||||
|
return { overall, parts, checkedAt, thresholds, lastError, problems, refresh }
|
||||||
|
})
|
||||||
@@ -3,9 +3,14 @@
|
|||||||
<!-- In-context view: deep-linked to one post, with bidirectional infinite
|
<!-- In-context view: deep-linked to one post, with bidirectional infinite
|
||||||
scroll — newer posts load above, older posts below. -->
|
scroll — newer posts load above, older posts below. -->
|
||||||
<template v-if="postIdFilter != null">
|
<template v-if="postIdFilter != null">
|
||||||
|
<!-- Returns to whichever surface you deep-linked FROM. This used to be a
|
||||||
|
hard `{ name: 'posts' }`, which redirects into Browse — fine while
|
||||||
|
this view only ever rendered inside Browse's tab, but it now also
|
||||||
|
serves the front door (#387 B1), where it would have yanked the
|
||||||
|
operator sideways into a different view. -->
|
||||||
<v-btn
|
<v-btn
|
||||||
variant="text" size="small" prepend-icon="mdi-arrow-left"
|
variant="text" size="small" prepend-icon="mdi-arrow-left"
|
||||||
:to="{ name: 'posts' }" class="mb-2"
|
:to="allPostsTarget" class="mb-2"
|
||||||
>All posts</v-btn>
|
>All posts</v-btn>
|
||||||
|
|
||||||
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
<v-alert v-if="store.error" type="error" variant="tonal" closable class="mb-3">
|
||||||
@@ -44,6 +49,8 @@
|
|||||||
|
|
||||||
<!-- Normal feed -->
|
<!-- Normal feed -->
|
||||||
<template v-else>
|
<template v-else>
|
||||||
|
<FeedStatusRibbon v-if="statusRibbon" />
|
||||||
|
|
||||||
<PostsFilterBar
|
<PostsFilterBar
|
||||||
:artist-id="artistFilter"
|
:artist-id="artistFilter"
|
||||||
:platform="platformFilter"
|
:platform="platformFilter"
|
||||||
@@ -59,11 +66,11 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else-if="store.items.length === 0 && store.done" class="fc-posts__empty">
|
<div v-else-if="store.items.length === 0 && store.done" class="fc-posts__empty">
|
||||||
|
<!-- A filtered miss is NOT the onboarding case: the operator has posts,
|
||||||
|
they just narrowed past them. Showing a fresh-install on-ramp here
|
||||||
|
would be telling someone with a full library to go and set it up. -->
|
||||||
<p v-if="hasActiveFilter">No posts match your search or filters.</p>
|
<p v-if="hasActiveFilter">No posts match your search or filters.</p>
|
||||||
<p v-else>No posts yet. Subscribe to a source on the
|
<FeedEmptyState v-else />
|
||||||
<RouterLink to="/subscriptions">Subscriptions</RouterLink>
|
|
||||||
tab to start capturing posts.
|
|
||||||
</p>
|
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div v-else>
|
<div v-else>
|
||||||
@@ -84,6 +91,17 @@ import { useRoute, useRouter } from 'vue-router'
|
|||||||
import { usePostsStore } from '../stores/posts.js'
|
import { usePostsStore } from '../stores/posts.js'
|
||||||
import PostsFilterBar from '../components/posts/PostsFilterBar.vue'
|
import PostsFilterBar from '../components/posts/PostsFilterBar.vue'
|
||||||
import PostCard from '../components/posts/PostCard.vue'
|
import PostCard from '../components/posts/PostCard.vue'
|
||||||
|
import FeedStatusRibbon from '../components/posts/FeedStatusRibbon.vue'
|
||||||
|
import FeedEmptyState from '../components/posts/FeedEmptyState.vue'
|
||||||
|
|
||||||
|
// The ingestion-status ribbon is a FRONT-DOOR concern, not a feed concern —
|
||||||
|
// inside Browse's Posts tab you are looking FOR something, and the Subscriptions
|
||||||
|
// hub is a click away. Passed as a route prop rather than sniffed from
|
||||||
|
// route.name so the view does not have to know what it is mounted as, and so
|
||||||
|
// the router file states the intent in one place.
|
||||||
|
defineProps({
|
||||||
|
statusRibbon: { type: Boolean, default: false },
|
||||||
|
})
|
||||||
|
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
@@ -103,6 +121,13 @@ const hasActiveFilter = computed(() =>
|
|||||||
artistFilter.value != null || platformFilter.value != null || searchFilter.value != null
|
artistFilter.value != null || platformFilter.value != null || searchFilter.value != null
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Drop only `post_id` and stay where we are — keeps Browse's `tab=posts` (and
|
||||||
|
// any active artist/platform scope) intact instead of resetting the surface.
|
||||||
|
const allPostsTarget = computed(() => {
|
||||||
|
const { post_id: _drop, ...rest } = route.query
|
||||||
|
return { name: route.name, query: rest }
|
||||||
|
})
|
||||||
|
|
||||||
// --- normal feed (downward infinite scroll) ---
|
// --- normal feed (downward infinite scroll) ---
|
||||||
const sentinel = ref(null)
|
const sentinel = ref(null)
|
||||||
let observer = null
|
let observer = null
|
||||||
|
|||||||
@@ -14,6 +14,7 @@
|
|||||||
style="position: sticky; top: var(--fc-nav-h, 64px); z-index: 4;"
|
style="position: sticky; top: var(--fc-nav-h, 64px); z-index: 4;"
|
||||||
>
|
>
|
||||||
<v-tab value="overview">Overview</v-tab>
|
<v-tab value="overview">Overview</v-tab>
|
||||||
|
<v-tab value="system">System</v-tab>
|
||||||
<v-tab value="activity">Activity</v-tab>
|
<v-tab value="activity">Activity</v-tab>
|
||||||
<v-tab value="cleanup">Cleanup</v-tab>
|
<v-tab value="cleanup">Cleanup</v-tab>
|
||||||
<v-tab value="maintenance">Maintenance</v-tab>
|
<v-tab value="maintenance">Maintenance</v-tab>
|
||||||
@@ -42,6 +43,13 @@
|
|||||||
</v-alert>
|
</v-alert>
|
||||||
</v-window-item>
|
</v-window-item>
|
||||||
|
|
||||||
|
<!-- Is every part of the stack still running (milestone 365). Sits
|
||||||
|
beside Activity deliberately: Activity answers "what is the queue
|
||||||
|
doing", this answers "is anything left to do it". -->
|
||||||
|
<v-window-item value="system">
|
||||||
|
<SystemHealthTab />
|
||||||
|
</v-window-item>
|
||||||
|
|
||||||
<v-window-item value="activity">
|
<v-window-item value="activity">
|
||||||
<SystemActivityTab @open-maintenance="tab = 'maintenance'" />
|
<SystemActivityTab @open-maintenance="tab = 'maintenance'" />
|
||||||
</v-window-item>
|
</v-window-item>
|
||||||
@@ -73,18 +81,24 @@
|
|||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { onMounted, onUnmounted, ref, watch } from 'vue'
|
import { onMounted, onUnmounted, watch } from 'vue'
|
||||||
import { useSystemStore } from '../stores/system.js'
|
import { useSystemStore } from '../stores/system.js'
|
||||||
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
import SystemStatsCards from '../components/settings/SystemStatsCards.vue'
|
||||||
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
|
import SystemActivitySummary from '../components/settings/SystemActivitySummary.vue'
|
||||||
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
|
import SystemActivityTab from '../components/settings/SystemActivityTab.vue'
|
||||||
|
import SystemHealthTab from '../components/settings/SystemHealthTab.vue'
|
||||||
import GpuActivityPanel from '../components/settings/GpuActivityPanel.vue'
|
import GpuActivityPanel from '../components/settings/GpuActivityPanel.vue'
|
||||||
import DownloadsActivityPanel from '../components/settings/DownloadsActivityPanel.vue'
|
import DownloadsActivityPanel from '../components/settings/DownloadsActivityPanel.vue'
|
||||||
import MaintenancePanel from '../components/settings/MaintenancePanel.vue'
|
import MaintenancePanel from '../components/settings/MaintenancePanel.vue'
|
||||||
import CleanupView from './CleanupView.vue'
|
import CleanupView from './CleanupView.vue'
|
||||||
|
import { useTabQuery } from '../composables/useTabQuery.js'
|
||||||
import { useMLStore } from '../stores/ml.js'
|
import { useMLStore } from '../stores/ml.js'
|
||||||
|
|
||||||
const tab = ref('overview')
|
// ?tab= sync (the same composable Browse/Subscriptions use) so a tab can be
|
||||||
|
// linked TO — the health dot beside the brand points at ?tab=system, and the
|
||||||
|
// old /system path redirects there.
|
||||||
|
const VALID_TABS = ['overview', 'system', 'activity', 'cleanup', 'maintenance']
|
||||||
|
const { tab } = useTabQuery(VALID_TABS, 'overview')
|
||||||
const system = useSystemStore()
|
const system = useSystemStore()
|
||||||
const mlStore = useMLStore()
|
const mlStore = useMLStore()
|
||||||
|
|
||||||
|
|||||||
@@ -20,6 +20,35 @@ describe('ActiveDownloadsPanel', () => {
|
|||||||
w.unmount() // clear the 1s elapsed-timer interval
|
w.unmount() // clear the 1s elapsed-timer interval
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// #387 A1: the live payload gained a `gated` count. Shown only when non-zero
|
||||||
|
// so a healthy run stays uncluttered — a "🔒 0" on every download would be
|
||||||
|
// noise, and noise is what stops the number being noticed when it matters.
|
||||||
|
it('ticks the tier-gated count mid-walk when there is one', () => {
|
||||||
|
const pinia = freshPinia()
|
||||||
|
useDownloadsStore().activeEvents = [{
|
||||||
|
id: 1, status: 'running',
|
||||||
|
started_at: new Date(Date.now() - 65000).toISOString(),
|
||||||
|
platform: 'patreon', artist_name: 'Alice',
|
||||||
|
live: { downloaded: 2, skipped: 0, errors: 0, posts: 12, gated: 9 },
|
||||||
|
}]
|
||||||
|
const w = mountComponent(ActiveDownloadsPanel, { pinia })
|
||||||
|
expect(w.text()).toContain('9')
|
||||||
|
w.unmount()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('omits the gated count when nothing was gated', () => {
|
||||||
|
const pinia = freshPinia()
|
||||||
|
useDownloadsStore().activeEvents = [{
|
||||||
|
id: 1, status: 'running',
|
||||||
|
started_at: new Date(Date.now() - 65000).toISOString(),
|
||||||
|
platform: 'patreon', artist_name: 'Alice',
|
||||||
|
live: { downloaded: 2, skipped: 0, errors: 0, posts: 12, gated: 0 },
|
||||||
|
}]
|
||||||
|
const w = mountComponent(ActiveDownloadsPanel, { pinia })
|
||||||
|
expect(w.find('.fc-active__count--gated').exists()).toBe(false)
|
||||||
|
w.unmount()
|
||||||
|
})
|
||||||
|
|
||||||
it('renders nothing when there is no active work', () => {
|
it('renders nothing when there is no active work', () => {
|
||||||
const pinia = freshPinia()
|
const pinia = freshPinia()
|
||||||
useDownloadsStore().activeEvents = []
|
useDownloadsStore().activeEvents = []
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
|
|
||||||
|
import FeedEmptyState from '../../src/components/posts/FeedEmptyState.vue'
|
||||||
|
import { useSourcesStore } from '../../src/stores/sources.js'
|
||||||
|
import { mountWithStore } from '../support/mountComponent.js'
|
||||||
|
|
||||||
|
// #387 B4. This is the first screen a fresh install shows anyone, so the thing
|
||||||
|
// worth pinning is that it tells the two empties apart. Telling an operator to
|
||||||
|
// "add a source" when they already have three and are mid-backfill is worse
|
||||||
|
// than saying nothing — it reads as the app not knowing its own state.
|
||||||
|
|
||||||
|
const mountWith = (status) => mountWithStore(FeedEmptyState, () => {
|
||||||
|
useSourcesStore().scheduleStatus = status
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('FeedEmptyState', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
globalThis.fetch = vi.fn(async () => { throw new Error('offline') })
|
||||||
|
})
|
||||||
|
afterEach(() => vi.restoreAllMocks())
|
||||||
|
|
||||||
|
it('a fresh install gets the on-ramp, credential first', () => {
|
||||||
|
const w = mountWith({ total_sources: 0 })
|
||||||
|
expect(w.text()).toContain('Add a credential')
|
||||||
|
expect(w.text()).toContain('Add a source')
|
||||||
|
expect(w.text()).not.toContain("See what's running")
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an install that is already fetching is told so, not told to set up', () => {
|
||||||
|
const w = mountWith({ total_sources: 3 })
|
||||||
|
expect(w.text()).toContain('3 sources')
|
||||||
|
expect(w.text()).toContain("See what's running")
|
||||||
|
expect(w.text()).not.toContain('Add a credential')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('singularises a lone source', () => {
|
||||||
|
const w = mountWith({ total_sources: 1 })
|
||||||
|
expect(w.text()).toContain('1 source')
|
||||||
|
expect(w.text()).not.toContain('1 sources')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('falls back to the on-ramp when the status call never answered', () => {
|
||||||
|
// Safe direction: the on-ramp is merely redundant to an established
|
||||||
|
// operator, whereas "see what's running" shown to someone with nothing
|
||||||
|
// configured is a dead end.
|
||||||
|
const w = mountWith(null)
|
||||||
|
expect(w.text()).toContain('Add a credential')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows the brand mark, sized to be readable', () => {
|
||||||
|
// logo.svg stops reading below ~48px — that is why the 22px nav slot has a
|
||||||
|
// different mark. If this ever shrinks to a glyph, it is the wrong asset.
|
||||||
|
const img = mountWith({ total_sources: 0 }).find('img.fc-empty__mark')
|
||||||
|
expect(img.exists()).toBe(true)
|
||||||
|
expect(img.attributes('src')).toBe('/logo.svg')
|
||||||
|
expect(Number(img.attributes('width'))).toBeGreaterThanOrEqual(48)
|
||||||
|
// Decorative: the surrounding prose already carries the meaning.
|
||||||
|
expect(img.attributes('alt')).toBe('')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||||
|
|
||||||
|
import FeedStatusRibbon from '../../src/components/posts/FeedStatusRibbon.vue'
|
||||||
|
import { useSourcesStore } from '../../src/stores/sources.js'
|
||||||
|
import { mountWithStore } from '../support/mountComponent.js'
|
||||||
|
|
||||||
|
// #387 B3. This ribbon is the ONLY place phase A's work reaches someone who
|
||||||
|
// wasn't already looking for it, so what it does and does not say is the whole
|
||||||
|
// feature. These pin: it stays silent when it has nothing true to report, it
|
||||||
|
// keeps "failing" and "no access" as separate claims, and it never renders a
|
||||||
|
// zero — a "0 failing" on the front door is noise that trains you to ignore
|
||||||
|
// the line, which is exactly what would hide the real number later.
|
||||||
|
|
||||||
|
const mountWith = (status) => mountWithStore(FeedStatusRibbon, () => {
|
||||||
|
useSourcesStore().scheduleStatus = status
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('FeedStatusRibbon', () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
// The component fetches on mount and swallows failures by design; stub it
|
||||||
|
// so the assertions are about the seeded state, not a race with the call.
|
||||||
|
globalThis.fetch = vi.fn(async () => { throw new Error('offline') })
|
||||||
|
})
|
||||||
|
afterEach(() => vi.restoreAllMocks())
|
||||||
|
|
||||||
|
it('renders nothing before it has a status', () => {
|
||||||
|
const w = mountWith(null)
|
||||||
|
expect(w.find('.fc-ribbon').exists()).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a healthy instance is one quiet line, with no zeroes', () => {
|
||||||
|
const w = mountWith({
|
||||||
|
last_tick_at: new Date().toISOString(), failing_sources: 0, no_access_sources: 0,
|
||||||
|
})
|
||||||
|
expect(w.find('.fc-ribbon').exists()).toBe(true)
|
||||||
|
expect(w.text()).toContain('Checked')
|
||||||
|
// Assert on the claims, not on the digit — a just-now timestamp can render
|
||||||
|
// its own "0 minutes ago" and a substring check would catch that instead.
|
||||||
|
expect(w.text()).not.toContain('failing')
|
||||||
|
expect(w.text()).not.toContain("can't see")
|
||||||
|
expect(w.find('.fc-ribbon__item--err').exists()).toBe(false)
|
||||||
|
expect(w.find('.fc-ribbon__item--gated').exists()).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports failing and no-access as two separate claims', () => {
|
||||||
|
const w = mountWith({
|
||||||
|
last_tick_at: new Date().toISOString(), failing_sources: 2, no_access_sources: 5,
|
||||||
|
})
|
||||||
|
expect(w.find('.fc-ribbon__item--err').text()).toContain('2 sources are failing')
|
||||||
|
expect(w.find('.fc-ribbon__item--gated').text()).toContain("5 you can't see")
|
||||||
|
})
|
||||||
|
|
||||||
|
it('no-access alone does not light the failing item', () => {
|
||||||
|
// The whole point of phase A: a paywalled creator is not a broken one.
|
||||||
|
const w = mountWith({
|
||||||
|
last_tick_at: new Date().toISOString(), failing_sources: 0, no_access_sources: 3,
|
||||||
|
})
|
||||||
|
expect(w.find('.fc-ribbon__item--err').exists()).toBe(false)
|
||||||
|
expect(w.find('.fc-ribbon__item--gated').exists()).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('says "never" rather than a blank when nothing has ever run', () => {
|
||||||
|
const w = mountWith({ last_tick_at: null, failing_sources: 0, no_access_sources: 0 })
|
||||||
|
expect(w.text()).toContain('never')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('singularises one failing source', () => {
|
||||||
|
const w = mountWith({
|
||||||
|
last_tick_at: new Date().toISOString(), failing_sources: 1, no_access_sources: 0,
|
||||||
|
})
|
||||||
|
expect(w.find('.fc-ribbon__item--err').text()).toContain('1 source is failing')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -40,6 +40,124 @@ describe('PostCard', () => {
|
|||||||
expect(full.text()).not.toContain('Show more')
|
expect(full.text()).not.toContain('Show more')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// #388 E2. The honesty marker is the only thing standing between "FC
|
||||||
|
// assembled this" and the card reading as something the artist authored, so
|
||||||
|
// these pin BOTH directions: it appears when the flag is set, and — the one
|
||||||
|
// that actually matters — it is absent on every ordinary post.
|
||||||
|
describe('synthetic posts', () => {
|
||||||
|
const SYNTH = {
|
||||||
|
...BASE,
|
||||||
|
post_title: null,
|
||||||
|
synthesized_by: 'discord_drop',
|
||||||
|
synthesis: { message_count: 4, member_post_ids: [1, 2, 3, 4] },
|
||||||
|
}
|
||||||
|
|
||||||
|
it('says FC grouped it, and what from', () => {
|
||||||
|
const w = mountComponent(PostCard, { props: { post: SYNTH }, pinia: freshPinia() })
|
||||||
|
expect(w.find('.fc-post-card__synthetic').exists()).toBe(true)
|
||||||
|
expect(w.text()).toContain('grouped by FabledCurator')
|
||||||
|
expect(w.text()).toContain('Grouped from 4 Discord messages')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('never marks an ordinary post', () => {
|
||||||
|
const w = mountComponent(PostCard, { props: { post: BASE }, pinia: freshPinia() })
|
||||||
|
expect(w.find('.fc-post-card__synthetic').exists()).toBe(false)
|
||||||
|
expect(w.text()).not.toContain('grouped by FabledCurator')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('does not leak the internal drop key as a title', () => {
|
||||||
|
// post_title is NULL on purpose (inventing one would put words in a
|
||||||
|
// creator's mouth), so the untitled fallback must not print
|
||||||
|
// `fc-drop:<message id>` the way it does for a real untitled post.
|
||||||
|
const w = mountComponent(PostCard, {
|
||||||
|
props: { post: { ...SYNTH, external_post_id: 'fc-drop:99887766' } },
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(w.text()).not.toContain('fc-drop:')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('degrades to unmarked when the field is absent entirely', () => {
|
||||||
|
// A post dict composed before the field existed must read as "not
|
||||||
|
// synthetic" rather than throw on `synthesis.message_count`.
|
||||||
|
const { synthesized_by: _drop, synthesis: _also, ...legacy } = SYNTH
|
||||||
|
const w = mountComponent(PostCard, { props: { post: legacy }, pinia: freshPinia() })
|
||||||
|
expect(w.find('.fc-post-card__synthetic').exists()).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('reports growth separately from the post date', () => {
|
||||||
|
// The drop's own date is its identity; growth is news about it. "From
|
||||||
|
// Tuesday, gained images this morning" is the trickling-in signal, and
|
||||||
|
// collapsing the two would erase it.
|
||||||
|
const w = mountComponent(PostCard, {
|
||||||
|
props: {
|
||||||
|
post: { ...SYNTH, last_grew_at: new Date(Date.now() - 3600e3).toISOString() },
|
||||||
|
},
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(w.text()).toContain('updated 1h ago')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('says nothing about growth on a grouping that has not grown', () => {
|
||||||
|
// An "updated" label that is always there teaches you to ignore it.
|
||||||
|
const w = mountComponent(PostCard, {
|
||||||
|
props: { post: { ...SYNTH, last_grew_at: null } },
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(w.text()).not.toContain('updated')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('never claims an ordinary post grew, even if the field leaks in', () => {
|
||||||
|
const w = mountComponent(PostCard, {
|
||||||
|
props: { post: { ...BASE, last_grew_at: new Date().toISOString() } },
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(w.text()).not.toContain('updated')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows an accepted announcement link from either end', () => {
|
||||||
|
const teaser = mountComponent(PostCard, {
|
||||||
|
props: {
|
||||||
|
post: {
|
||||||
|
...BASE,
|
||||||
|
associations: [{ id: 7, role: 'announces', post_id: 42 }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(teaser.text()).toContain('The full set is in Discord')
|
||||||
|
|
||||||
|
const drop = mountComponent(PostCard, {
|
||||||
|
props: {
|
||||||
|
post: {
|
||||||
|
...SYNTH,
|
||||||
|
associations: [{ id: 7, role: 'announced_by', post_id: 1 }],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(drop.text()).toContain('Announced on Patreon')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows no link when there is no accepted association', () => {
|
||||||
|
// Pending proposals never reach the payload, so an empty list here is
|
||||||
|
// the normal case and must render as silence, not an empty row.
|
||||||
|
const w = mountComponent(PostCard, {
|
||||||
|
props: { post: { ...BASE, associations: [] } },
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(w.find('.fc-post-card__assoc').exists()).toBe(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('singularises a one-message drop', () => {
|
||||||
|
const w = mountComponent(PostCard, {
|
||||||
|
props: { post: { ...SYNTH, synthesis: { message_count: 1 } } },
|
||||||
|
pinia: freshPinia(),
|
||||||
|
})
|
||||||
|
expect(w.text()).toContain('Grouped from 1 Discord message')
|
||||||
|
expect(w.text()).not.toContain('1 Discord messages')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
const thumbs = (n) =>
|
const thumbs = (n) =>
|
||||||
Array.from({ length: n }, (_, i) => ({ image_id: 100 + i, thumbnail_url: `/t${i}` }))
|
Array.from({ length: n }, (_, i) => ({ image_id: 100 + i, thumbnail_url: `/t${i}` }))
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
// @vitest-environment happy-dom
|
||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
|
||||||
|
import SourceHealthDot from '../../src/components/subscriptions/SourceHealthDot.vue'
|
||||||
|
import { VTooltipStub, mountComponent } from '../support/mountComponent.js'
|
||||||
|
|
||||||
|
// Milestone #387 A3. The dot is the only always-visible signal per source, so
|
||||||
|
// the grade it picks IS the claim FC makes about that subscription. These pin
|
||||||
|
// that no-access is graded as its own thing — not as healthy (which hides it)
|
||||||
|
// and not as a failure (which would send the operator hunting for a break that
|
||||||
|
// isn't there).
|
||||||
|
|
||||||
|
const checked = { last_checked_at: '2026-09-09T12:00:00+00:00' }
|
||||||
|
|
||||||
|
function dotClass (w) {
|
||||||
|
return w.find('.fc-health-dot').classes().join(' ')
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('SourceHealthDot', () => {
|
||||||
|
it('grades a tier-gated source as no-access, not healthy', () => {
|
||||||
|
const w = mountComponent(SourceHealthDot, {
|
||||||
|
stubs: { VTooltip: VTooltipStub },
|
||||||
|
props: {
|
||||||
|
source: { ...checked, consecutive_failures: 0, error_type: 'tier_limited' },
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(dotClass(w)).toContain('fc-health-dot--no-access')
|
||||||
|
expect(dotClass(w)).not.toContain('fc-health-dot--healthy')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows the gated count when the list endpoint supplied one', () => {
|
||||||
|
const w = mountComponent(SourceHealthDot, {
|
||||||
|
stubs: { VTooltip: VTooltipStub },
|
||||||
|
props: {
|
||||||
|
source: {
|
||||||
|
...checked, consecutive_failures: 0,
|
||||||
|
error_type: 'tier_limited', tier_gated_count: 47,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(w.text()).toContain('47 posts')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('states the condition without a number when no count was joined in', () => {
|
||||||
|
const w = mountComponent(SourceHealthDot, {
|
||||||
|
stubs: { VTooltip: VTooltipStub },
|
||||||
|
props: {
|
||||||
|
source: {
|
||||||
|
...checked, consecutive_failures: 0,
|
||||||
|
error_type: 'tier_limited', tier_gated_count: null,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
// Absent is not zero: never render "0 posts you don't have access to".
|
||||||
|
expect(w.text()).not.toContain('0 post')
|
||||||
|
expect(w.text()).toContain("tier you don't hold")
|
||||||
|
})
|
||||||
|
|
||||||
|
it('a genuinely failing source still grades as a failure, gated or not', () => {
|
||||||
|
const w = mountComponent(SourceHealthDot, {
|
||||||
|
stubs: { VTooltip: VTooltipStub },
|
||||||
|
props: {
|
||||||
|
source: { ...checked, consecutive_failures: 9, error_type: 'tier_limited' },
|
||||||
|
warningThreshold: 5,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
expect(dotClass(w)).toContain('fc-health-dot--critical')
|
||||||
|
expect(dotClass(w)).not.toContain('fc-health-dot--no-access')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('an unchecked source is still unchecked', () => {
|
||||||
|
const w = mountComponent(SourceHealthDot, {
|
||||||
|
stubs: { VTooltip: VTooltipStub },
|
||||||
|
props: { source: { last_checked_at: null, consecutive_failures: 0 } },
|
||||||
|
})
|
||||||
|
expect(dotClass(w)).toContain('fc-health-dot--unchecked')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -2,8 +2,29 @@ import { describe, it, expect } from 'vitest'
|
|||||||
import router, { FRONT_DOOR } from '../src/router.js'
|
import router, { FRONT_DOOR } from '../src/router.js'
|
||||||
|
|
||||||
describe('router', () => {
|
describe('router', () => {
|
||||||
it('FRONT_DOOR defaults to /showcase', () => {
|
it('FRONT_DOOR is the post feed', () => {
|
||||||
expect(FRONT_DOOR).toBe('/showcase')
|
// Moved from /showcase in #387 B1. Asserted on the constant rather than on
|
||||||
|
// a navigation so the intent is pinned: the door is the feed, and moving it
|
||||||
|
// again should be a deliberate edit to a failing test, not a quiet change.
|
||||||
|
expect(FRONT_DOOR).toBe('/latest')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('/latest is the feed, mounted outside the Browse tab strip', () => {
|
||||||
|
const r = router.resolve('/latest')
|
||||||
|
expect(r.name).toBe('latest')
|
||||||
|
// A nav entry in its own right (meta.title is what TopNav lists on), and
|
||||||
|
// first in the order.
|
||||||
|
expect(r.meta.title).toBe('Latest')
|
||||||
|
expect(r.meta.navOrder).toBe(5)
|
||||||
|
// Deliberately NOT stickyChrome — it has no sticky sub-header for the nav
|
||||||
|
// to butt against, unlike Browse/Gallery/Settings.
|
||||||
|
expect(r.meta.stickyChrome).toBeUndefined()
|
||||||
|
})
|
||||||
|
|
||||||
|
it('showcase is demoted, not removed — still reachable and still in the nav', () => {
|
||||||
|
const r = router.resolve('/showcase')
|
||||||
|
expect(r.name).toBe('showcase')
|
||||||
|
expect(r.meta.title).toBe('Showcase')
|
||||||
})
|
})
|
||||||
|
|
||||||
it('/ redirects to FRONT_DOOR', async () => {
|
it('/ redirects to FRONT_DOOR', async () => {
|
||||||
@@ -40,6 +61,14 @@ describe('router', () => {
|
|||||||
expect(router.currentRoute.value.query.post_id).toBe('7')
|
expect(router.currentRoute.value.query.post_id).toBe('7')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('/system redirects into the Settings System tab', async () => {
|
||||||
|
// It shipped as a standalone page for one build; the health dot and any
|
||||||
|
// bookmark from it must still land on the surface, which is now a tab.
|
||||||
|
await router.push('/system')
|
||||||
|
expect(router.currentRoute.value.name).toBe('settings')
|
||||||
|
expect(router.currentRoute.value.query.tab).toBe('system')
|
||||||
|
})
|
||||||
|
|
||||||
it('series-read is an immersive route', () => {
|
it('series-read is an immersive route', () => {
|
||||||
const r = router.resolve('/series/5/read')
|
const r = router.resolve('/series/5/read')
|
||||||
expect(r.name).toBe('series-read')
|
expect(r.name).toBe('series-read')
|
||||||
|
|||||||
@@ -13,12 +13,35 @@ export function freshPinia () {
|
|||||||
return pinia
|
return pinia
|
||||||
}
|
}
|
||||||
|
|
||||||
export function mountComponent (Component, { props = {}, pinia } = {}) {
|
// `stubs` is merged over the defaults. Needed whenever the component under
|
||||||
|
// test puts content in a NAMED slot of a Vuetify component: leaving those
|
||||||
|
// unresolved renders default-slot children only, so a named slot (v-tooltip's
|
||||||
|
// `#activator`, say) silently renders nothing and assertions find an empty
|
||||||
|
// wrapper rather than failing loudly.
|
||||||
|
export function mountComponent (Component, { props = {}, pinia, stubs = {} } = {}) {
|
||||||
return mount(Component, {
|
return mount(Component, {
|
||||||
props,
|
props,
|
||||||
global: {
|
global: {
|
||||||
plugins: pinia ? [pinia] : [],
|
plugins: pinia ? [pinia] : [],
|
||||||
stubs: { RouterLink: { template: '<a><slot /></a>' } },
|
stubs: { RouterLink: { template: '<a><slot /></a>' }, ...stubs },
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Renders both halves of a v-tooltip: the activator (the thing the operator
|
||||||
|
// actually sees) and the tip body. `props` is passed as an empty object so the
|
||||||
|
// activator's `v-bind="tipProps"` binds cleanly.
|
||||||
|
export const VTooltipStub = {
|
||||||
|
name: 'VTooltip',
|
||||||
|
template: '<div><slot name="activator" :props="{}" /><slot /></div>',
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mount with a fresh pinia and the store already seeded, for components that
|
||||||
|
// read store state during render. Without it, the seeding has to be inlined
|
||||||
|
// between createPinia and mount in every spec — the same copy-paste that issue
|
||||||
|
// #3109 tracks for the backend row factories.
|
||||||
|
export function mountWithStore (Component, seed, opts = {}) {
|
||||||
|
const pinia = freshPinia()
|
||||||
|
seed()
|
||||||
|
return mountComponent(Component, { ...opts, pinia })
|
||||||
|
}
|
||||||
|
|||||||
+22
-1
@@ -90,7 +90,7 @@ DERIVER='scripts/artifacts.sh'
|
|||||||
|
|
||||||
|
|
||||||
usage() {
|
usage() {
|
||||||
echo "usage: artifacts.sh {paths|revision|version} {web|ml|agent|extension}" >&2
|
echo "usage: artifacts.sh {paths|revision|version|epoch} {web|ml|agent|extension}" >&2
|
||||||
exit 2
|
exit 2
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -149,6 +149,26 @@ cmd_revision() {
|
|||||||
echo "$(newest "$1")" | cut -d' ' -f2 | cut -c1-12
|
echo "$(newest "$1")" | cut -d' ' -f2 | cut -c1-12
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The BUILD CLOCK: the same commit's unix timestamp, for SOURCE_DATE_EPOCH.
|
||||||
|
#
|
||||||
|
# buildkit stamps the image config's `created` field and every history entry
|
||||||
|
# with the wall clock of the build unless this is set, so two builds of
|
||||||
|
# identical source produce different config blobs and therefore different
|
||||||
|
# manifest digests. That is #3265: the weekly refresh republished all three
|
||||||
|
# `:latest` tags on 2026-08-30 with every content step CACHED and the bases
|
||||||
|
# resolved to unchanged digests — nothing was different, and the digest moved
|
||||||
|
# anyway. A digest that changes on a calendar cannot also mean "the content
|
||||||
|
# changed", which is the only thing anyone wants it for.
|
||||||
|
#
|
||||||
|
# It is the same commit `revision` and `version` name — deliberately, and this
|
||||||
|
# is the point of routing it through `newest()` rather than taking git's word
|
||||||
|
# separately. Three values derived from three lookups can disagree; three
|
||||||
|
# views of one lookup cannot. Note #3127 §2 is the record of what a second
|
||||||
|
# clock costs.
|
||||||
|
cmd_epoch() {
|
||||||
|
echo "$(newest "$1")" | cut -d' ' -f1
|
||||||
|
}
|
||||||
|
|
||||||
# The VERSION: `YYYY.MM.DD.HHMM`, zero-padded, UTC. One shape across the whole
|
# The VERSION: `YYYY.MM.DD.HHMM`, zero-padded, UTC. One shape across the whole
|
||||||
# family (note #3127 §1, rule 148) — the number an instance reports about
|
# family (note #3127 §1, rule 148) — the number an instance reports about
|
||||||
# itself, and, with a `v` in front, the release tag naming the same build.
|
# itself, and, with a `v` in front, the release tag naming the same build.
|
||||||
@@ -197,5 +217,6 @@ case "$1" in
|
|||||||
paths) cmd_paths "$2" ;;
|
paths) cmd_paths "$2" ;;
|
||||||
revision) cmd_revision "$2" ;;
|
revision) cmd_revision "$2" ;;
|
||||||
version) cmd_version "$2" ;;
|
version) cmd_version "$2" ;;
|
||||||
|
epoch) cmd_epoch "$2" ;;
|
||||||
*) usage ;;
|
*) usage ;;
|
||||||
esac
|
esac
|
||||||
|
|||||||
+112
-8
@@ -33,6 +33,32 @@ history. Ancestry is immune to the shape change, and it is also the more honest
|
|||||||
question: "what is in this that was not in the last one" IS a reachability
|
question: "what is in this that was not in the last one" IS a reachability
|
||||||
question.
|
question.
|
||||||
|
|
||||||
|
Ancestry alone is not enough, though, and milestone 328 is where that showed.
|
||||||
|
The 28 `v26.*` tags are still in the repo — the operator kept them as history
|
||||||
|
when their releases were deleted — so `--match v*` walks straight back to
|
||||||
|
`v26.06.04.0` and reports 533 commits. That span is not a changelog: nobody has
|
||||||
|
run `v26.06.04.0`, its release page no longer exists to compare against, and
|
||||||
|
the 200 lines that survive truncation are precisely the internal build-out that
|
||||||
|
milestone 328 exists to stop shipping. So the match is `v[0-9][0-9][0-9][0-9].*`
|
||||||
|
— rule 148's four-digit-year shape — which is exactly the set of tags that name
|
||||||
|
a release a reader could have been running. A pre-convention tag is history,
|
||||||
|
not a predecessor.
|
||||||
|
|
||||||
|
## The first release has no changelog, and should not pretend to
|
||||||
|
|
||||||
|
Once the match is narrowed, the first rule-148 tag reaches no predecessor at
|
||||||
|
all, and the old fallback — diff against the whole history — is worse than the
|
||||||
|
problem it replaced. The honest content for a release nobody has a previous
|
||||||
|
version of is what the thing IS.
|
||||||
|
|
||||||
|
So a release with no reachable predecessor renders the product overview instead
|
||||||
|
of a commit list. It is read out of README.md between `<!-- overview:start -->`
|
||||||
|
and `<!-- overview:end -->` rather than written here, for the same reason the
|
||||||
|
changelog is derived: two hand-maintained descriptions of one product drift,
|
||||||
|
and nothing ever catches it. The release page and the repo front page are one
|
||||||
|
source. Every later release goes back to being a changelog, which is what §5 of
|
||||||
|
note #3127 says a release is for.
|
||||||
|
|
||||||
## Re-runs update, they do not fall through
|
## Re-runs update, they do not fall through
|
||||||
|
|
||||||
Note #3127 §6.7: a publisher that POSTs and recovers the id from a `409` never
|
Note #3127 §6.7: a publisher that POSTs and recovers the id from a `409` never
|
||||||
@@ -91,18 +117,45 @@ def git_ok(*args: str) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def previous_tag(ref: str, tag: str | None) -> str | None:
|
def previous_tag(ref: str, tag: str | None) -> str | None:
|
||||||
"""The most recent `v*` tag reachable from `ref`, excluding `tag` itself.
|
"""The most recent rule-148 tag reachable from `ref`, excluding `tag` itself.
|
||||||
|
|
||||||
`--exclude` rather than `<ref>^` so this is the same call whether or not
|
`--exclude` rather than `<ref>^` so this is the same call whether or not
|
||||||
`ref` is the tag being released — and so it does not blow up on a root
|
`ref` is the tag being released — and so it does not blow up on a root
|
||||||
commit that has no parent to walk to.
|
commit that has no parent to walk to.
|
||||||
|
|
||||||
|
The glob deliberately does NOT match the old `v26.*` tags. They are kept as
|
||||||
|
history and their releases are gone, so naming one as the predecessor emits
|
||||||
|
a span nobody can look up. See the module docstring.
|
||||||
"""
|
"""
|
||||||
args = ["describe", "--tags", "--abbrev=0", "--match", "v*"]
|
args = ["describe", "--tags", "--abbrev=0", "--match", "v[0-9][0-9][0-9][0-9].*"]
|
||||||
if tag:
|
if tag:
|
||||||
args += ["--exclude", tag]
|
args += ["--exclude", tag]
|
||||||
return git_ok(*args, ref)
|
return git_ok(*args, ref)
|
||||||
|
|
||||||
|
|
||||||
|
def product_overview() -> str | None:
|
||||||
|
"""The product description, lifted verbatim from README.md.
|
||||||
|
|
||||||
|
Returns None if the markers are absent or empty — a missing overview is
|
||||||
|
reported as a note and the release still publishes, on the same reasoning
|
||||||
|
as cross_checks(): the release is the useful object even when one part of
|
||||||
|
the derivation could not run.
|
||||||
|
"""
|
||||||
|
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
|
try:
|
||||||
|
with open(os.path.join(root, "README.md"), encoding="utf-8") as fh:
|
||||||
|
readme = fh.read()
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
match = re.search(
|
||||||
|
r"<!--\s*overview:start\s*-->(.*?)<!--\s*overview:end\s*-->",
|
||||||
|
readme, re.S,
|
||||||
|
)
|
||||||
|
if not match:
|
||||||
|
return None
|
||||||
|
return match.group(1).strip() or None
|
||||||
|
|
||||||
|
|
||||||
def commits(previous: str | None, ref: str) -> list[str]:
|
def commits(previous: str | None, ref: str) -> list[str]:
|
||||||
"""The subjects between the previous release and this one.
|
"""The subjects between the previous release and this one.
|
||||||
|
|
||||||
@@ -125,7 +178,10 @@ def truncate(log: list[str]) -> tuple[list[str], str | None]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def render(tag: str, sha: str, previous: str | None, log: list[str], notes: list[str]) -> str:
|
def render(
|
||||||
|
tag: str, sha: str, previous: str | None, log: list[str], notes: list[str],
|
||||||
|
overview: str | None,
|
||||||
|
) -> str:
|
||||||
short = sha[:7]
|
short = sha[:7]
|
||||||
parts = []
|
parts = []
|
||||||
|
|
||||||
@@ -135,6 +191,25 @@ def render(tag: str, sha: str, previous: str | None, log: list[str], notes: list
|
|||||||
# is the failure this whole milestone is about.
|
# is the failure this whole milestone is about.
|
||||||
parts.append("\n".join(f"> **Note:** {n}" for n in notes))
|
parts.append("\n".join(f"> **Note:** {n}" for n in notes))
|
||||||
|
|
||||||
|
# No predecessor means nobody reading this has run an earlier one, so the
|
||||||
|
# release describes the product rather than a diff. The overview is
|
||||||
|
# README.md's own words — see the module docstring on why it is not
|
||||||
|
# written here.
|
||||||
|
if previous is None and overview:
|
||||||
|
parts.append(overview)
|
||||||
|
parts.append(
|
||||||
|
"## Installing\n\n"
|
||||||
|
"```\ncurl -O https://git.fabledsword.com/bvandeusen/FabledCurator/raw/"
|
||||||
|
f"tag/{tag}/docker-compose.yml\ncurl -O https://git.fabledsword.com/"
|
||||||
|
f"bvandeusen/FabledCurator/raw/tag/{tag}/.env.example\n"
|
||||||
|
"mv .env.example .env # then set SECRET_KEY, DB_PASSWORD\n"
|
||||||
|
"docker compose -f docker-compose.yml up -d\n```\n\n"
|
||||||
|
"**Read \"Before you expose it\" in the README first.** FabledCurator "
|
||||||
|
"has no login, and it stores live platform session cookies for "
|
||||||
|
"accounts that usually have a payment method attached. Bind it to a "
|
||||||
|
"network you trust."
|
||||||
|
)
|
||||||
|
|
||||||
parts.append(
|
parts.append(
|
||||||
f"Built from `{short}`. The rollback unit is the immutable `:c-` tag "
|
f"Built from `{short}`. The rollback unit is the immutable `:c-` tag "
|
||||||
f"(rule 145) — these three move together:\n\n```\n"
|
f"(rule 145) — these three move together:\n\n```\n"
|
||||||
@@ -142,7 +217,19 @@ def render(tag: str, sha: str, previous: str | None, log: list[str], notes: list
|
|||||||
+ "\n```"
|
+ "\n```"
|
||||||
)
|
)
|
||||||
|
|
||||||
heading = f"## Changes since {previous}" if previous else "## Changes"
|
if previous is None:
|
||||||
|
# Deliberately NOT a commit list. The alternative is the whole history
|
||||||
|
# truncated to MAX_COMMITS, which is 200 lines of internal build-out
|
||||||
|
# presented to someone who has never seen this project.
|
||||||
|
parts.append(
|
||||||
|
"---\n\n_First release under rule 148's `vYYYY.MM.DD.HHMM` shape, so "
|
||||||
|
"there is no predecessor to diff against and no changelog to derive. "
|
||||||
|
"The description above is README.md's, quoted at publish time. Later "
|
||||||
|
"releases carry the commits since the previous one._"
|
||||||
|
)
|
||||||
|
return "\n\n".join(parts)
|
||||||
|
|
||||||
|
heading = f"## Changes since {previous}"
|
||||||
if log:
|
if log:
|
||||||
parts.append(heading + "\n\n" + "\n".join(f"- {line}" for line in log))
|
parts.append(heading + "\n\n" + "\n".join(f"- {line}" for line in log))
|
||||||
else:
|
else:
|
||||||
@@ -152,10 +239,9 @@ def render(tag: str, sha: str, previous: str | None, log: list[str], notes: list
|
|||||||
"names the same source under a new name._"
|
"names the same source under a new name._"
|
||||||
)
|
)
|
||||||
|
|
||||||
span = f"{previous}..{tag}" if previous else tag
|
|
||||||
parts.append(
|
parts.append(
|
||||||
f"---\n\n_Derived at publish time from `git log --no-merges {span}`. "
|
f"---\n\n_Derived at publish time from "
|
||||||
f"Nothing here is hand-maintained._"
|
f"`git log --no-merges {previous}..{tag}`. Nothing here is hand-maintained._"
|
||||||
)
|
)
|
||||||
return "\n\n".join(parts)
|
return "\n\n".join(parts)
|
||||||
|
|
||||||
@@ -289,13 +375,31 @@ def main() -> None:
|
|||||||
for note in notes:
|
for note in notes:
|
||||||
print(f"release: NOTE {note}")
|
print(f"release: NOTE {note}")
|
||||||
|
|
||||||
|
# A first release renders the overview instead of a changelog, so the
|
||||||
|
# commit walk is skipped entirely rather than computed and discarded —
|
||||||
|
# `commits(None, ref)` is the whole history and there is no reason to ask
|
||||||
|
# for it.
|
||||||
|
overview = None
|
||||||
|
log: list[str] = []
|
||||||
|
if previous is None:
|
||||||
|
overview = product_overview()
|
||||||
|
if overview is None:
|
||||||
|
note = (
|
||||||
|
"No `<!-- overview:start -->` block found in README.md, so this "
|
||||||
|
"first release has no product description. Published anyway; add "
|
||||||
|
"the markers and re-run the workflow to fill it in."
|
||||||
|
)
|
||||||
|
print(f"release: NOTE {note}")
|
||||||
|
notes.append(note)
|
||||||
|
print("release: no rule-148 predecessor — rendering the product overview")
|
||||||
|
else:
|
||||||
log = commits(previous, ref)
|
log = commits(previous, ref)
|
||||||
print(f"release: {len(log)} non-merge commits in the span")
|
print(f"release: {len(log)} non-merge commits in the span")
|
||||||
log, overflow = truncate(log)
|
log, overflow = truncate(log)
|
||||||
if overflow:
|
if overflow:
|
||||||
print(f"release: NOTE {overflow}")
|
print(f"release: NOTE {overflow}")
|
||||||
notes.append(overflow)
|
notes.append(overflow)
|
||||||
body = render(tag or ref, sha, previous, log, notes)
|
body = render(tag or ref, sha, previous, log, notes, overview)
|
||||||
|
|
||||||
if args.dry_run or not tag:
|
if args.dry_run or not tag:
|
||||||
print("--- body ---")
|
print("--- body ---")
|
||||||
|
|||||||
@@ -0,0 +1,155 @@
|
|||||||
|
"""Prove a freshly built image can still do the things its OS packages provide.
|
||||||
|
|
||||||
|
Run INSIDE the image, not against the source tree. That distinction is the
|
||||||
|
entire reason this file exists.
|
||||||
|
|
||||||
|
`ci.yml`'s lanes run on `ci-python:3.14` and install `requirements.txt`. A base
|
||||||
|
refresh changes neither, so all five lanes stay green through a base bump that
|
||||||
|
breaks the product. What a refresh actually re-resolves is this, from the
|
||||||
|
Dockerfile:
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ffmpeg unar libpq5 postgresql-client zstd megatools \
|
||||||
|
libjpeg62-turbo libwebp7 libpng16-16 ca-certificates
|
||||||
|
|
||||||
|
Unpinned, every build. Nothing else in this repo looks at it.
|
||||||
|
|
||||||
|
So the checks below run the APPLICATION'S OWN code — `Thumbnailer`, which needs
|
||||||
|
no database and no app context — against whatever Pillow and ffmpeg have
|
||||||
|
become. `ffmpeg -version` exiting 0 would pass while a codec removal or an
|
||||||
|
soname bump broke every thumbnail in the library; producing a thumbnail would
|
||||||
|
not.
|
||||||
|
|
||||||
|
Every failure names the package it implicates. This fires on a Sunday,
|
||||||
|
unattended, about a change nobody made deliberately — "assertion failed" a week
|
||||||
|
later teaches nobody anything.
|
||||||
|
|
||||||
|
Usage: docker run --rm -i <image> shell -c 'python3 -' < scripts/smoke_image.py
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
try:
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
from backend.app.services.thumbnailer import Thumbnailer
|
||||||
|
except Exception as exc: # noqa: BLE001 — a smoke test reports, it never raises
|
||||||
|
print(f"smoke: FAILED — could not import the thumbnail path at all: {exc}")
|
||||||
|
print(" Implicates Pillow or its shared libraries (libjpeg62-turbo,")
|
||||||
|
print(" libpng16-16, libwebp7), or the python base image itself.")
|
||||||
|
raise SystemExit(1) from exc
|
||||||
|
|
||||||
|
|
||||||
|
# Binary → what stops working without it. Listed individually because
|
||||||
|
# `--no-install-recommends` means any one of them can vanish on its own when a
|
||||||
|
# dependency chain higher up changes.
|
||||||
|
REQUIRED_BINARIES = {
|
||||||
|
"ffmpeg": "video thumbnails and transcoding (Dockerfile: ffmpeg)",
|
||||||
|
"unar": "archive import — cbz/zip/rar members (Dockerfile: unar)",
|
||||||
|
"pg_dump": "database backup (Dockerfile: postgresql-client)",
|
||||||
|
"zstd": "backup compression, pg_dump | tar --zstd (Dockerfile: zstd)",
|
||||||
|
"megatools": "mega.nz public-link downloads, #830 (Dockerfile: megatools)",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def check_jpeg(thumbs: Thumbnailer, src: Path) -> None:
|
||||||
|
path = src / "flat.jpg"
|
||||||
|
Image.new("RGB", (900, 400), (30, 90, 160)).save(path, "JPEG")
|
||||||
|
result = thumbs.generate_image_thumbnail(path, "a" * 64)
|
||||||
|
assert result.mime == "image/jpeg", f"mime was {result.mime}"
|
||||||
|
assert result.path.stat().st_size > 0, "no bytes written"
|
||||||
|
# Re-open it. A file that writes but cannot be read back is the shape a
|
||||||
|
# half-broken codec produces, and size alone would not catch it.
|
||||||
|
with Image.open(result.path) as im:
|
||||||
|
im.load()
|
||||||
|
|
||||||
|
|
||||||
|
def check_png_alpha(thumbs: Thumbnailer, src: Path) -> None:
|
||||||
|
path = src / "alpha.png"
|
||||||
|
Image.new("RGBA", (400, 900), (200, 40, 40, 128)).save(path, "PNG")
|
||||||
|
result = thumbs.generate_image_thumbnail(path, "b" * 64)
|
||||||
|
assert result.mime == "image/png", f"mime was {result.mime}"
|
||||||
|
with Image.open(result.path) as im:
|
||||||
|
im.load()
|
||||||
|
assert im.mode in ("RGBA", "LA", "P"), f"alpha lost, mode={im.mode}"
|
||||||
|
|
||||||
|
|
||||||
|
def check_webp(thumbs: Thumbnailer, src: Path) -> None:
|
||||||
|
path = src / "sample.webp"
|
||||||
|
Image.new("RGB", (500, 500), (10, 140, 70)).save(path, "WEBP")
|
||||||
|
result = thumbs.generate_image_thumbnail(path, "c" * 64)
|
||||||
|
assert result.path.stat().st_size > 0, "no bytes written"
|
||||||
|
|
||||||
|
|
||||||
|
def check_video(thumbs: Thumbnailer, src: Path) -> None:
|
||||||
|
# Synthesised rather than committed as a fixture: a checked-in video is a
|
||||||
|
# binary blob nobody can review, and lavfi ships with every ffmpeg build.
|
||||||
|
#
|
||||||
|
# 3 seconds, not 2. The seek lands at max(1.0, duration * 0.05) = 1.0s, and
|
||||||
|
# a clip barely longer than its own seek is how #1231 produced zero frames.
|
||||||
|
# This check exists to exercise ffmpeg, not to re-litigate that edge.
|
||||||
|
clip = src / "clip.mp4"
|
||||||
|
subprocess.run(
|
||||||
|
["ffmpeg", "-nostdin", "-f", "lavfi", "-i", "testsrc=size=640x360:rate=10",
|
||||||
|
"-t", "3", "-pix_fmt", "yuv420p", "-y", str(clip)],
|
||||||
|
check=True, capture_output=True, timeout=120,
|
||||||
|
)
|
||||||
|
result = thumbs.generate_video_thumbnail(clip, "d" * 64, duration_seconds=3.0)
|
||||||
|
assert result.path.stat().st_size > 0, "no bytes written"
|
||||||
|
with Image.open(result.path) as im:
|
||||||
|
im.load()
|
||||||
|
|
||||||
|
|
||||||
|
CHECKS = (
|
||||||
|
("JPEG thumbnail", "libjpeg62-turbo / Pillow", check_jpeg),
|
||||||
|
("PNG thumbnail (alpha)", "libpng16-16 / Pillow", check_png_alpha),
|
||||||
|
("WebP decode", "libwebp7 / Pillow", check_webp),
|
||||||
|
("video thumbnail", "ffmpeg", check_video),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
failures: list[str] = []
|
||||||
|
|
||||||
|
print("smoke: binaries the apt layer provides")
|
||||||
|
for binary, purpose in REQUIRED_BINARIES.items():
|
||||||
|
if shutil.which(binary) is None:
|
||||||
|
print(f" FAIL {binary}: not on PATH")
|
||||||
|
failures.append(f"{binary} — {purpose}")
|
||||||
|
else:
|
||||||
|
print(f" ok {binary}")
|
||||||
|
|
||||||
|
print("smoke: the application's own thumbnail path, against this image's libraries")
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
src = root / "src"
|
||||||
|
src.mkdir()
|
||||||
|
thumbs = Thumbnailer(root)
|
||||||
|
for name, implicates, fn in CHECKS:
|
||||||
|
try:
|
||||||
|
fn(thumbs, src)
|
||||||
|
print(f" ok {name}")
|
||||||
|
except Exception as exc: # noqa: BLE001 — report every check, then fail once
|
||||||
|
print(f" FAIL {name}: {exc}")
|
||||||
|
failures.append(f"{name} — {implicates}")
|
||||||
|
|
||||||
|
if failures:
|
||||||
|
print(f"\nsmoke: FAILED — {len(failures)} check(s)")
|
||||||
|
for failure in failures:
|
||||||
|
print(f" - {failure}")
|
||||||
|
print("\nThis image was built against freshly resolved base layers. The")
|
||||||
|
print("named packages are where to look: compare this build's apt versions")
|
||||||
|
print("against the previous :latest before assuming the app changed.")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print("\nsmoke: all checks passed")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user