Compare commits
255
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
725bf15f88 | ||
|
|
bc4eba636d | ||
|
|
dbc4e8b0c6 | ||
|
|
b14818303c | ||
|
|
08418d54a3 | ||
|
|
b1bd2531ad | ||
|
|
389afe2f7b | ||
|
|
b979062dd7 | ||
|
|
573228b9da | ||
|
|
d044e93bdb | ||
|
|
ed2b1adc2e | ||
|
|
5e1996e77f | ||
|
|
98b56330d0 | ||
|
|
6959e1220c | ||
|
|
2529b516e6 | ||
|
|
8f1ac0c96a | ||
|
|
5fd171a544 | ||
|
|
62583791d8 | ||
|
|
0a5bbe81dc | ||
|
|
6663e06aa6 | ||
|
|
63e0a423d7 | ||
|
|
499720d87e | ||
|
|
5e72076298 | ||
|
|
e21c9fdd34 | ||
|
|
6b3ec98fa8 | ||
|
|
bd24f4e876 | ||
|
|
1a941e900b | ||
|
|
0e15c44c51 | ||
|
|
6d98dfc0ec | ||
|
|
6c76f08b69 | ||
|
|
7b1019ba82 | ||
|
|
0d204e6837 | ||
|
|
8300029741 | ||
|
|
b5b437ca80 | ||
|
|
ce0dac3524 | ||
|
|
9b5ec86222 | ||
|
|
89c83ee5de | ||
|
|
d3192f1843 | ||
|
|
5a5694f200 | ||
|
|
bb1a938cc0 | ||
|
|
fc0293029d | ||
|
|
7d1c701b67 | ||
|
|
b638382cd5 | ||
|
|
17903068b4 | ||
|
|
31d400ab0a | ||
|
|
d45ce5e426 | ||
|
|
77bfc32b02 | ||
|
|
d13efe1b69 | ||
|
|
ca6d26d236 | ||
|
|
5366047d55 | ||
|
|
2923257529 | ||
|
|
934731e9ef | ||
|
|
4c67c116b2 | ||
|
|
f8b667604f | ||
|
|
11572f469a | ||
|
|
ba3fd2a118 | ||
|
|
9bf095179a | ||
|
|
cefc064e0f | ||
|
|
06757daf80 | ||
|
|
a723ef436b | ||
|
|
d68f39ca50 | ||
|
|
a66a695977 | ||
|
|
5289fa3879 | ||
|
|
ebac34e17b | ||
|
|
4c3ba20198 | ||
|
|
d2acec61ae | ||
|
|
15ae5ef4fa | ||
|
|
e3ceefc820 | ||
|
|
e52090a4cf | ||
|
|
bfba8045e4 | ||
|
|
a708357436 | ||
|
|
d9354ac1e1 | ||
|
|
1d48770793 | ||
|
|
489e6aaaee | ||
|
|
ed20df905b | ||
|
|
5ba9871ef0 | ||
|
|
2a820d0848 | ||
|
|
2f9aa3d86c | ||
|
|
560a5000a2 | ||
|
|
7ddad231f8 | ||
|
|
a78f7eaace | ||
|
|
71337b0ba4 | ||
|
|
3996205f3b | ||
|
|
9f9db01456 | ||
|
|
216b7fc743 | ||
|
|
fbb76e6f36 | ||
|
|
5ef1478ade | ||
|
|
88ff4147e1 | ||
|
|
1ea02ad44c | ||
|
|
937cfb65b4 | ||
|
|
baac851220 | ||
|
|
a28c33281a | ||
|
|
40be0a9323 | ||
|
|
2c6bf26bfc | ||
|
|
3c1e76bd44 | ||
|
|
831c5b1c10 | ||
|
|
76b6af4903 | ||
|
|
92494ec4ed | ||
|
|
bdfc17477c | ||
|
|
6cd3153bf4 | ||
|
|
5f2853168a | ||
|
|
9a979ee808 | ||
|
|
3138f912fd | ||
|
|
9df874e396 | ||
|
|
6915a7590a | ||
|
|
5e8c28236a | ||
|
|
7e3c0f0b74 | ||
|
|
5d0c7ba706 | ||
|
|
18300e1f8a | ||
|
|
d52ac0a0e2 | ||
|
|
401fe8213e | ||
|
|
e8774d7953 | ||
|
|
bc0f00c51b | ||
|
|
1bef68aa29 | ||
|
|
1a4bc2f981 | ||
|
|
862ace69d6 | ||
|
|
abf88b1a15 | ||
|
|
c05dcafbea | ||
|
|
55e8632dab | ||
|
|
825e6b90bf | ||
|
|
fb012c557c | ||
|
|
66593ab895 | ||
|
|
b266a54ad3 | ||
|
|
ad803b646f | ||
|
|
1f5da3d283 | ||
|
|
93034f580d | ||
|
|
9b9b12f410 | ||
|
|
376d310693 | ||
|
|
bc69495a16 | ||
|
|
478f898e72 | ||
|
|
38a5e7f332 | ||
|
|
57fe15c267 | ||
|
|
eb3231ef10 | ||
|
|
e9af459c0d | ||
|
|
6f02806aec | ||
|
|
a1d19bd96a | ||
|
|
26827ff38f | ||
|
|
26dcfaf6c2 | ||
|
|
9b1b0369cc | ||
|
|
18123fb9cb | ||
|
|
2e806f202f | ||
|
|
18d5c05639 | ||
|
|
11ddfc3876 | ||
|
|
2b8ce86622 | ||
|
|
49bee77cdc | ||
|
|
c209e3b37e | ||
|
|
cffdd93418 | ||
|
|
fd84be40dd | ||
|
|
79f510d7f8 | ||
|
|
59181069da | ||
|
|
428ecd8642 | ||
|
|
ed1e04b831 | ||
|
|
f5156bd847 | ||
|
|
dfc3922d24 | ||
|
|
3eb08e926b | ||
|
|
9e81ced359 | ||
|
|
11e9f5af60 | ||
|
|
909fa37b15 | ||
|
|
dfab8f65ff | ||
|
|
618f7cdc36 | ||
|
|
028ea33a7c | ||
|
|
444c1fb075 | ||
|
|
26c68b0a75 | ||
|
|
e75427b19a | ||
|
|
5447fab987 | ||
|
|
bad37e07b2 | ||
|
|
2bfc9936a1 | ||
|
|
4c6406ee18 | ||
|
|
bb47e80b3e | ||
|
|
dc1083b5e0 | ||
|
|
e46893fefd | ||
|
|
0666e15211 | ||
|
|
747390631d | ||
|
|
e0d2a20588 | ||
|
|
1d84f67418 | ||
|
|
91265df3d6 | ||
|
|
11acdb0322 | ||
|
|
2eb9fd5dd0 | ||
|
|
01e5ce1410 | ||
|
|
3bb94674cf | ||
|
|
a75c602175 | ||
|
|
ef8f4f7193 | ||
|
|
ec3d27b219 | ||
|
|
03bd3b2eda | ||
|
|
7395e77d75 | ||
|
|
575d817919 | ||
|
|
2a8f7cd8b6 | ||
|
|
83f8af8090 | ||
|
|
9a2617c1a2 | ||
|
|
81688815a0 | ||
|
|
773128c3bf | ||
|
|
ce7b154ae9 | ||
|
|
9430a9d9c3 | ||
|
|
23aee56ce3 | ||
|
|
711abea567 | ||
|
|
844bb86802 | ||
|
|
a8f6a464aa | ||
|
|
ab9922ad2e | ||
|
|
0533807669 | ||
|
|
279dff3fb6 | ||
|
|
37e66cddc4 | ||
|
|
9cf6b2d363 | ||
|
|
6ef0fed41f | ||
|
|
89b48f8f35 | ||
|
|
d60e0b9494 | ||
|
|
9c27a2d3c7 | ||
|
|
93e37681b7 | ||
|
|
64ca858574 | ||
|
|
9d0c0b7da8 | ||
|
|
8e4d252ae4 | ||
|
|
fdd3e01f56 | ||
|
|
c82fb308b6 | ||
|
|
8cf8d2ca4d | ||
|
|
b1d58bc3b8 | ||
|
|
65386f02a0 | ||
|
|
667b05f14e | ||
|
|
856e9104b4 | ||
|
|
0397642b21 | ||
|
|
237575447d | ||
|
|
ed358757dc | ||
|
|
d181f4afb8 | ||
|
|
2886fa4997 | ||
|
|
f256f587ee | ||
|
|
384d8d5e50 | ||
|
|
319e8c1d18 | ||
|
|
9075d8eadd | ||
|
|
88e53e5b86 | ||
|
|
37e8b796a1 | ||
|
|
4e82208926 | ||
|
|
52fff00353 | ||
|
|
c14338cbce | ||
|
|
8c36dd28b0 | ||
|
|
88cfb3dd02 | ||
|
|
5d4f223b71 | ||
|
|
05090c6e85 | ||
|
|
3a577d5ade | ||
|
|
f4fe02e346 | ||
|
|
e766197d99 | ||
|
|
3872e1dda9 | ||
|
|
9814f3dbaf | ||
|
|
b214460fdb | ||
|
|
ac55d0e8d8 | ||
|
|
89a89e0ded | ||
|
|
4e9aac2c05 | ||
|
|
2879ac6f2b | ||
|
|
b8dce6c483 | ||
|
|
d1c0b82a22 | ||
|
|
5526b8dc78 | ||
|
|
16eb7075c4 | ||
|
|
885dcf64f3 | ||
|
|
f2f6b6d25e | ||
|
|
0822240fde | ||
|
|
27f7f3fd01 | ||
|
|
c5bf564f53 | ||
|
|
602c7d275d |
@@ -0,0 +1,370 @@
|
||||
|
||||
# TEMPORARY — milestone 328 steps 1-2. Delete once the baseline is stamped.
|
||||
#
|
||||
# Squashing 87 alembic revisions into one baseline has exactly one dangerous
|
||||
# failure: the generated baseline does not reproduce the schema the chain
|
||||
# produced, `alembic stamp` writes a version string anyway (it validates
|
||||
# NOTHING), and the divergence surfaces on the next real migration against the
|
||||
# operator's live data.
|
||||
#
|
||||
# So this workflow does the comparison in CI, where a pgvector Postgres already
|
||||
# gets built from the chain on every integration run, and nothing is at risk.
|
||||
# It answers one question: does `upgrade head` on the collapsed chain produce a
|
||||
# byte-identical schema to `upgrade head` on the 87-revision chain?
|
||||
#
|
||||
# The chain is read from git rather than from the working tree, so this keeps
|
||||
# working AFTER the old revisions are deleted — `chain_ref` names a commit that
|
||||
# still has them. That is what makes this the proof for step 1 and the
|
||||
# pre-flight for step 2, rather than a one-shot script.
|
||||
#
|
||||
# While the chain is still present it also autogenerates a candidate baseline
|
||||
# from the models and prints it. That is a starting point, NOT the answer:
|
||||
# autogenerate reads SQLAlchemy metadata, and three things here do not live
|
||||
# there —
|
||||
# * CREATE EXTENSION vector (0001)
|
||||
# * CREATE EXTENSION tsm_system_rows (0004)
|
||||
# * the HNSW index on image_record.siglip_embedding, which is raw SQL
|
||||
# because alembic's create_index cannot express `USING hnsw (...)` (0036)
|
||||
# plus any CHECK constraint or server_default that a migration added without
|
||||
# the model declaring it. Those must be hand-added, and the diff below is what
|
||||
# proves none were missed.
|
||||
name: Alembic baseline
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
chain_ref:
|
||||
description: 'Commit/tag carrying the full chain; blank = this ref (use a pinned commit only AFTER the collapse)'
|
||||
type: string
|
||||
default: ''
|
||||
mode:
|
||||
description: 'chain = compare against this tree''s migrations; models = compare against a schema built from the MODELS'
|
||||
type: string
|
||||
default: 'chain'
|
||||
|
||||
jobs:
|
||||
compare:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
env:
|
||||
DB_USER: fabledcurator
|
||||
DB_PASSWORD: ci_integration
|
||||
DB_PORT: "5432"
|
||||
DB_NAME: fabledcurator_test
|
||||
SECRET_KEY: ci_integration_placeholder
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: fabledcurator
|
||||
POSTGRES_PASSWORD: ci_integration
|
||||
POSTGRES_DB: fabledcurator_test
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U fabledcurator"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Full history is the point: `chain_ref` is read out of git, so a
|
||||
# shallow clone would not have the revisions to compare against.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Resolve the Postgres service and install deps
|
||||
run: |
|
||||
set -eux
|
||||
# Same service-IP dance as ci.yml's integration job; see the long
|
||||
# comment there for why the job name must stay separator-free.
|
||||
PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
test -n "$PG"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
test -n "$PG_IP"
|
||||
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
||||
echo "DB_HOST=$PG_IP" >> "$GITHUB_ENV"
|
||||
for i in $(seq 1 60); do
|
||||
(echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
uv pip install --system -r requirements.txt
|
||||
else
|
||||
pip install -r requirements.txt
|
||||
fi
|
||||
|
||||
# DB 1: the 87-revision chain, read out of git at `chain_ref`.
|
||||
#
|
||||
# A git worktree rather than a checkout, so the current tree — which is
|
||||
# what we are testing — is left completely alone.
|
||||
- name: Build the schema the OLD chain produces
|
||||
env:
|
||||
CHAIN_REF: ${{ github.event.inputs.chain_ref }}
|
||||
THIS_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
set -eux
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_chain
|
||||
# Blank means "the chain in this ref", which is what you want while
|
||||
# the chain is still intact — comparing the models against a PINNED
|
||||
# older commit reports every migration written since as a difference.
|
||||
# Pin it only after the collapse, when the tree no longer has them.
|
||||
git worktree add /tmp/chain "${CHAIN_REF:-$THIS_SHA}"
|
||||
ls /tmp/chain/alembic/versions/*.py | wc -l
|
||||
cd /tmp/chain
|
||||
DB_NAME=fc_chain alembic upgrade head
|
||||
cd -
|
||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||
--no-owner --no-privileges -d fc_chain > chain.sql
|
||||
wc -l chain.sql
|
||||
# Emit the dump itself, checksummed, for local analysis. Reconciling
|
||||
# the models against the deployed schema (#3275) needs the ACTUAL
|
||||
# schema, not an inference from a diff — parsing table context out of
|
||||
# unified-diff hunks drops every table whose CREATE TABLE line falls
|
||||
# outside a hunk, which silently under-reports.
|
||||
#
|
||||
# base64 + sha256 for the same reason as the candidate: a plain cat
|
||||
# of a file this size was truncated mid-line by the runner with the
|
||||
# step still green (run 4964).
|
||||
set +x
|
||||
B64=$(base64 -w 120 chain.sql)
|
||||
echo "===== BEGIN CHAIN SCHEMA (base64) ====="
|
||||
echo "$B64"
|
||||
echo "===== END CHAIN SCHEMA ====="
|
||||
echo "chain-sha256: $(sha256sum chain.sql | cut -d' ' -f1)"
|
||||
echo "chain-bytes: $(wc -c < chain.sql)"
|
||||
set -x
|
||||
|
||||
# A candidate baseline, autogenerated from the models against an EMPTY
|
||||
# database so every table shows up as a create. Printed for a human to
|
||||
# finish — it will be missing the three raw-SQL items named at the top.
|
||||
#
|
||||
# Gated on the TREE, not on a workflow input. A `type: boolean` input
|
||||
# read back as `github.event.inputs.generate == 'true'` silently
|
||||
# evaluated false on this runner (run 4960 skipped this step entirely
|
||||
# with no diagnostic) — the same `github.event.inputs` typing quirk
|
||||
# build.yml already works around. The file count is the real question
|
||||
# anyway: there is nothing to generate once the chain is collapsed.
|
||||
- name: Autogenerate a candidate baseline
|
||||
run: |
|
||||
set -eux
|
||||
if [ "$(ls alembic/versions/*.py | wc -l)" -le 1 ]; then
|
||||
echo "already collapsed — nothing to generate"
|
||||
exit 0
|
||||
fi
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_gen
|
||||
# Hide the existing revisions so alembic sees an empty history and
|
||||
# emits the whole schema rather than a delta.
|
||||
mkdir -p /tmp/versions_held
|
||||
mv alembic/versions/*.py /tmp/versions_held/ 2>/dev/null || true
|
||||
DB_NAME=fc_gen alembic revision --autogenerate -m "baseline" || true
|
||||
# Printed rather than uploaded: ci-requirements.md records that this
|
||||
# runner cannot do actions/upload-artifact@v4+, and the repo dropped
|
||||
# the action entirely in 2026-05, so the job log is the retrieval
|
||||
# channel actually proven here.
|
||||
#
|
||||
# base64, not the raw file. A plain `cat` of the ~33KB candidate was
|
||||
# TRUNCATED MID-LINE by the runner on run 4964 — it stopped inside
|
||||
# `sa.Column('mime', sa.String(length=128)` and carried straight on
|
||||
# to the next traced command, with the step still green. A silent
|
||||
# cut in the middle of a schema definition is the worst possible
|
||||
# failure here, because the truncated text still looks like a
|
||||
# plausible file.
|
||||
#
|
||||
# base64 at a fixed narrow width gives many short lines instead of
|
||||
# few long ones, and — the actual point — a checksum and a line
|
||||
# count that make truncation DETECTABLE rather than invisible.
|
||||
set +x
|
||||
F=$(ls alembic/versions/*.py | head -1)
|
||||
B64=$(base64 -w 120 "$F")
|
||||
echo "===== BEGIN CANDIDATE BASELINE (base64) ====="
|
||||
echo "$B64"
|
||||
echo "===== END CANDIDATE BASELINE ====="
|
||||
echo "candidate-sha256: $(sha256sum "$F" | cut -d' ' -f1)"
|
||||
echo "candidate-bytes: $(wc -c < "$F")"
|
||||
echo "candidate-b64-lines: $(echo "$B64" | wc -l)"
|
||||
set -x
|
||||
mkdir -p /tmp/candidate
|
||||
cp alembic/versions/*.py /tmp/candidate/
|
||||
# Put the tree back exactly as it was; this job never mutates state.
|
||||
rm -f alembic/versions/*.py
|
||||
mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true
|
||||
|
||||
# DB 2: what the CURRENT tree produces.
|
||||
#
|
||||
# `mode: models` applies the candidate autogenerated from the MODELS
|
||||
# instead, which is what answers "do the models describe the schema?" —
|
||||
# the question #3275 exists because nobody had ever asked it. Under that
|
||||
# mode a clean diff means autogenerate is trustworthy again.
|
||||
#
|
||||
# The two extensions are created by hand first. They are database
|
||||
# objects, not table metadata, so no model can carry them and their
|
||||
# absence is not a model defect — it is simply outside what this
|
||||
# comparison is asking about.
|
||||
- name: Build the schema the CURRENT tree produces
|
||||
env:
|
||||
MODE: ${{ github.event.inputs.mode }}
|
||||
run: |
|
||||
set -eux
|
||||
docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base
|
||||
if [ "${MODE:-chain}" = "models" ]; then
|
||||
docker exec "$PG_CONTAINER" psql -U fabledcurator -d fc_base \
|
||||
-c "CREATE EXTENSION IF NOT EXISTS vector" \
|
||||
-c "CREATE EXTENSION IF NOT EXISTS tsm_system_rows"
|
||||
mkdir -p /tmp/held
|
||||
mv alembic/versions/*.py /tmp/held/
|
||||
cp /tmp/candidate/*.py alembic/versions/
|
||||
# Autogenerate EMITS pgvector.sqlalchemy.vector.VECTOR(...) without
|
||||
# importing it, so the file it writes cannot run:
|
||||
# NameError: name 'pgvector' is not defined
|
||||
# Observed on run 4988, which is the proof rather than the theory.
|
||||
# This is a defect in the GENERATOR, not in the models, so it is
|
||||
# repaired here rather than counted as a schema difference — the
|
||||
# comparison is about whether the models describe the schema.
|
||||
sed -i '0,/^import sqlalchemy as sa$/s//import sqlalchemy as sa\nimport pgvector.sqlalchemy.vector/' alembic/versions/*.py
|
||||
grep -n 'import pgvector' alembic/versions/*.py
|
||||
# Second generator defect, same class as the missing import.
|
||||
#
|
||||
# base.py's naming convention includes %(constraint_name)s for ck,
|
||||
# which — unlike uq/fk/ix — means the convention is applied even to
|
||||
# a CheckConstraint that HAS a name. So a model declaring
|
||||
# name="singleton" correctly becomes ck_ml_settings_singleton in
|
||||
# the metadata. Autogenerate then writes that RENDERED name into
|
||||
# the migration, and running the migration applies the convention a
|
||||
# SECOND time: ck_ml_settings_ck_ml_settings_singleton.
|
||||
#
|
||||
# That is round-tripping damage done by the generator, not a claim
|
||||
# the models make, so it is repaired here rather than counted as a
|
||||
# schema difference. Undone by removing the ck_<table>_ prefix the
|
||||
# convention will re-add — the exact inverse, and it only fires on
|
||||
# a name that actually carries its own table's prefix.
|
||||
python3 - alembic/versions/*.py <<'PYEOF'
|
||||
import re, sys
|
||||
|
||||
table = None
|
||||
for path in sys.argv[1:]:
|
||||
out = []
|
||||
for line in open(path):
|
||||
m = re.search(r"op\.create_table\(\s*[\"']([A-Za-z0-9_]+)[\"']", line)
|
||||
if m:
|
||||
table = m.group(1)
|
||||
if table and "CheckConstraint" in line:
|
||||
prefix = f"ck_{table}_"
|
||||
line = re.sub(
|
||||
r"(name=[\"'])" + re.escape(prefix),
|
||||
r"\1",
|
||||
line,
|
||||
)
|
||||
out.append(line)
|
||||
open(path, "w").writelines(out)
|
||||
PYEOF
|
||||
grep -n 'CheckConstraint' alembic/versions/*.py || true
|
||||
ls alembic/versions/*.py
|
||||
DB_NAME=fc_base alembic upgrade head
|
||||
rm -f alembic/versions/*.py
|
||||
mv /tmp/held/*.py alembic/versions/
|
||||
else
|
||||
ls alembic/versions/*.py | wc -l
|
||||
DB_NAME=fc_base alembic upgrade head
|
||||
fi
|
||||
docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \
|
||||
--no-owner --no-privileges -d fc_base > baseline.sql
|
||||
wc -l baseline.sql
|
||||
|
||||
# The verdict.
|
||||
#
|
||||
# pg_dump orders dumpable objects by name within type, not by creation
|
||||
# order, so two schemas built by different routes are directly
|
||||
# comparable. Normalisation is deliberately minimal, because a filter
|
||||
# that hides a real difference is the one way this check passes when it
|
||||
# should fail — blank lines, SQL comments, trailing whitespace, and:
|
||||
#
|
||||
# \restrict / \unrestrict — a per-invocation RANDOM NONCE that newer
|
||||
# pg_dump emits to fence the dump against injection during restore. It
|
||||
# differs on every run by construction, so it is noise by definition,
|
||||
# not a schema difference. Measured on run 4960, the control: two dumps
|
||||
# of the SAME schema came back 1123 lines each and differed on exactly
|
||||
# these two lines and nothing else. That control is what licenses this
|
||||
# filter — it was observed to be the only false positive, rather than
|
||||
# assumed to be one.
|
||||
# Column ORDER inside a CREATE TABLE is compared separately from column
|
||||
# CONTENT, and only content is fatal.
|
||||
#
|
||||
# A table built by 87 migrations has its columns in ADD COLUMN order; the
|
||||
# same table built in one shot has them in declaration order. That is a
|
||||
# real and permanent difference which no baseline can erase — the
|
||||
# operator's existing database keeps chain order forever, a fresh install
|
||||
# gets model order — so a check that fails on it would never pass and
|
||||
# would teach nothing. FC reaches every column through the ORM by name,
|
||||
# and `SELECT *` ordering is not depended on anywhere.
|
||||
#
|
||||
# So the second pass SORTS the column lines within each CREATE TABLE
|
||||
# rather than DELETING them. That distinction is the whole point: sorting
|
||||
# cannot hide a column that exists on one side only, or one whose type,
|
||||
# nullability or default differs — those still land in the diff. A filter
|
||||
# could have hidden all three.
|
||||
#
|
||||
# Both diffs are reported. The ordered one is informational; the
|
||||
# order-insensitive one is the verdict.
|
||||
- name: Diff
|
||||
run: |
|
||||
set -eu
|
||||
norm() {
|
||||
grep -vE '^\s*(--|$)' "$1" \
|
||||
| grep -vE '^\\(un)?restrict ' \
|
||||
| sed 's/[[:space:]]*$//'
|
||||
}
|
||||
norm chain.sql > a.txt
|
||||
norm baseline.sql > b.txt
|
||||
echo "normalised: chain=$(wc -l < a.txt) lines, current=$(wc -l < b.txt) lines"
|
||||
|
||||
sort_table_columns() {
|
||||
python3 - "$1" <<'PYEOF'
|
||||
import re, sys
|
||||
|
||||
lines = open(sys.argv[1]).read().splitlines()
|
||||
out, block = [], None
|
||||
for line in lines:
|
||||
if block is not None:
|
||||
# ');' on its own closes the CREATE TABLE body.
|
||||
if line.strip() == ");":
|
||||
out.extend(sorted(block))
|
||||
out.append(line)
|
||||
block = None
|
||||
else:
|
||||
# Drop the list comma before sorting. Only the LAST
|
||||
# column lacks one, so keeping it would make every
|
||||
# reordering look like a content change as well — the
|
||||
# comma is punctuation, and carries no schema meaning.
|
||||
block.append(line.rstrip().rstrip(","))
|
||||
continue
|
||||
out.append(line)
|
||||
if re.match(r"CREATE TABLE .*\($", line):
|
||||
block = []
|
||||
if block is not None: # unterminated body: emit it rather than drop it
|
||||
out.extend(block)
|
||||
print("\n".join(out))
|
||||
PYEOF
|
||||
}
|
||||
sort_table_columns a.txt > a.sorted.txt
|
||||
sort_table_columns b.txt > b.sorted.txt
|
||||
test "$(wc -l < a.sorted.txt)" = "$(wc -l < a.txt)"
|
||||
test "$(wc -l < b.sorted.txt)" = "$(wc -l < b.txt)"
|
||||
|
||||
if diff -u a.txt b.txt > schema.diff; then
|
||||
echo "ORDERED DIFF: identical, column order included."
|
||||
else
|
||||
echo "ORDERED DIFF: $(grep -cE '^[+-]' schema.diff) changed lines (informational):"
|
||||
cat schema.diff
|
||||
fi
|
||||
echo
|
||||
echo "================================================================"
|
||||
echo
|
||||
if diff -u a.sorted.txt b.sorted.txt > sorted.diff; then
|
||||
echo "SCHEMAS MATCH — every difference above is column ORDER alone."
|
||||
else
|
||||
echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' sorted.diff) changed lines that are NOT ordering:"
|
||||
cat sorted.diff
|
||||
echo
|
||||
echo "The baseline is wrong, not the database. Do not stamp."
|
||||
exit 1
|
||||
fi
|
||||
@@ -25,6 +25,56 @@ on:
|
||||
# Releases still happen (rule 148, on explicit request per rule 2). They
|
||||
# produce a changelog, not an image.
|
||||
|
||||
# The escape hatch for the one thing skip-if-exists makes untestable: a
|
||||
# build that WOULD be skipped. `agent/` has not changed since 2026-07-17, so
|
||||
# every push since has correctly declined to build it — which also means the
|
||||
# agent build path has not run in six weeks and cannot be exercised on
|
||||
# demand. #3190 lives on exactly that path.
|
||||
#
|
||||
# Editing build.yml does not force one either, and that is deliberate: the
|
||||
# workflow is not shipped bytes, so it is in no artifact's path set. Putting
|
||||
# it in one would re-version every artifact for a comment change.
|
||||
#
|
||||
# ONE input, not one per artifact. Forcing all three is cheap once the
|
||||
# registry cache is warm (#3114), and three booleans is an interface nobody
|
||||
# remembers the meaning of.
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force_build:
|
||||
description: 'Rebuild every image even if the published revision matches'
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
# The base-image refresh (milestone 326 step 4, #3154).
|
||||
#
|
||||
# Skip-if-exists is keyed on OUR source, so an artifact whose source stops
|
||||
# moving stops picking up base-image updates. `agent/` last changed
|
||||
# 2026-07-17; every push since has correctly declined to rebuild it, which
|
||||
# also means it will serve that day's `nvidia/cuda` layers forever. Nothing
|
||||
# is wrong until it has been unchanged for months, which is precisely why
|
||||
# this is a calendar trigger and not a condition on the push path.
|
||||
#
|
||||
# Weekly, Sunday 06:00 UTC. Away from CI-runner's Monday security sweep so
|
||||
# the two are never diagnosing each other, and on the quietest day so a
|
||||
# surprise rebuild is not competing with a push.
|
||||
schedule:
|
||||
- cron: '0 6 * * 0'
|
||||
|
||||
# Which branch a run BUILDS, as opposed to which one triggered it.
|
||||
#
|
||||
# They are the same thing on every trigger but `schedule`. Forgejo registers a
|
||||
# cron from the DEFAULT branch — `dev` here — so a scheduled run arrives with
|
||||
# `github.ref` pointing at dev, and a refresh that rebuilt `:dev` would be
|
||||
# refreshing the one channel that gets rebuilt constantly anyway. Production is
|
||||
# `main` (rule 147), and `:latest` is the tag that goes stale.
|
||||
#
|
||||
# So the ref is decided once, here, and every checkout in the file takes it.
|
||||
# Deriving it per job invites the two halves to disagree: sign-extension would
|
||||
# derive dev's extension version while build-web bundled main's, and the
|
||||
# release download would 404 on a version that exists perfectly well.
|
||||
env:
|
||||
BUILD_REF: ${{ github.event_name == 'schedule' && 'main' || github.ref }}
|
||||
|
||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||
# - write:release (for ext-<version> release asset cache)
|
||||
@@ -68,12 +118,43 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history is load-bearing, not a convenience: the version this
|
||||
# job signs is derived from the commit TIME of the newest packaged
|
||||
# extension change. A depth-1 clone sees one commit and derives a
|
||||
# wrong, too-low value rather than failing (ci-requirements.md).
|
||||
fetch-depth: 0
|
||||
|
||||
# BUILD_REF is what makes a scheduled run build `main` rather than the
|
||||
# branch its cron fired from — and it is read through the `env` context
|
||||
# inside `with:`, which this runner is NOT known to evaluate. If it does
|
||||
# not, checkout silently falls back to the triggering ref and the weekly
|
||||
# refresh publishes DEV's source to `:latest`, which is production.
|
||||
# Every lane would stay green; the first sign of it would be production
|
||||
# running code that was never merged.
|
||||
#
|
||||
# So assert the checkout instead of trusting the expression. A red
|
||||
# weekly job is a fine outcome. Shipping dev to production is not.
|
||||
#
|
||||
# `if:` reads the `github` context, which the runner demonstrably does
|
||||
# evaluate — this file already gates steps on it — so the guard cannot
|
||||
# be disabled by the same uncertainty it exists to cover.
|
||||
- name: Guard — a scheduled run must have checked out main
|
||||
if: github.event_name == 'schedule'
|
||||
run: |
|
||||
set -eu
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))"
|
||||
if [ "$BRANCH" != "main" ]; then
|
||||
echo "schedule: expected main, got '$BRANCH'." >&2
|
||||
echo "schedule: BUILD_REF was not honoured by the runner." >&2
|
||||
echo "schedule: refusing to publish a channel tag from it." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# The version is DERIVED, not read from the repo (milestone 271 step 4,
|
||||
# cut over 2026-08-27). `packaging.sh version` returns `YYYY.M.D.HHMM`
|
||||
# UTC — the commit TIME of the newest change to a PACKAGED extension
|
||||
@@ -344,12 +425,30 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history: this job RE-DERIVES the extension version rather than
|
||||
# being handed it, and a depth-1 clone derives a wrong, too-low value
|
||||
# rather than failing — which would 404 the download of a release
|
||||
# that exists perfectly well under its real name.
|
||||
fetch-depth: 0
|
||||
|
||||
# See sign-extension's copy for why this guard exists.
|
||||
- name: Guard — a scheduled run must have checked out main
|
||||
if: github.event_name == 'schedule'
|
||||
run: |
|
||||
set -eu
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))"
|
||||
if [ "$BRANCH" != "main" ]; then
|
||||
echo "schedule: expected main, got '$BRANCH'." >&2
|
||||
echo "schedule: BUILD_REF was not honoured by the runner." >&2
|
||||
echo "schedule: refusing to publish a channel tag from it." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- derived values, one line (milestone 313) ------------------------
|
||||
# These stopped being shadow output at step 3. `revision` decides
|
||||
# whether the build below runs at all and `version` is what the image
|
||||
@@ -409,8 +508,30 @@ jobs:
|
||||
# everywhere). Operator-flagged 2026-06-01 after the first :c-<sha>
|
||||
# main-push build failed at this step.
|
||||
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||
# Mirrors build-web's tag list; see the comment there.
|
||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
|
||||
# A scheduled refresh publishes the CHANNEL and nothing else
|
||||
# (#3154). :c-<sha> for main's HEAD already exists and names the
|
||||
# bytes that commit actually built; re-pushing it over refreshed
|
||||
# base layers would break the one tag rule 145 makes immutable —
|
||||
# and it is the rollback unit, so the breakage would surface on the
|
||||
# day somebody needed it.
|
||||
#
|
||||
# The accepted consequence: between a refresh and the next main
|
||||
# push, :latest and :c-<sha> point at different manifests. That is
|
||||
# the design, not drift. They RE-CONVERGE on that push — it hits
|
||||
# reuse (a refresh does not move fc.revision, because it does not
|
||||
# touch the source), and the repoint step then writes the new
|
||||
# :c-<sha> from the refreshed :latest. So the rollback unit ends up
|
||||
# naming the bytes production is actually running, which is the
|
||||
# property that matters.
|
||||
#
|
||||
# Checked BEFORE the ref test, not after: a scheduled run's
|
||||
# GITHUB_REF is the default branch (dev), so the main test would
|
||||
# never fire on it.
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
@@ -443,6 +564,23 @@ jobs:
|
||||
ACTOR: ${{ github.actor }}
|
||||
run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin
|
||||
|
||||
# A REAL buildx builder, not the default `docker` driver (#3114, #3190).
|
||||
#
|
||||
# The default driver builds through the local dockerd. It cannot export a
|
||||
# registry cache at all — which is why the agent rebuilds a ~6.3 GB CUDA
|
||||
# + torch image from scratch whenever the runner's local cache is cold,
|
||||
# measured at 9m26s against 7s warm. It is also #3190's leading suspect:
|
||||
# after a registry-direct push it resolves image metadata against a local
|
||||
# store the push never filled, and reports `No such image` on an image
|
||||
# that published perfectly well three seconds earlier.
|
||||
#
|
||||
# These jobs run INSIDE a container against a mounted docker socket, so
|
||||
# the buildkit container this starts is a SIBLING of the job container,
|
||||
# not a child. That works over the socket mount; it had never been tried
|
||||
# here before milestone 326 step 1.
|
||||
- name: Set up buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# --- reuse-if-published (milestone 313, step 4) ----------------------
|
||||
# Does the image the channel tag already points at carry THIS commit's
|
||||
# revision? If so the bytes this job would produce are already published
|
||||
@@ -481,6 +619,16 @@ jobs:
|
||||
env:
|
||||
IMAGE: git.fabledsword.com/bvandeusen/fabledcurator
|
||||
CHANNEL: ${{ steps.tag.outputs.channel }}
|
||||
# Empty on a push; the string "true" only from a workflow_dispatch
|
||||
# that asked for it. `github.event.inputs` rather than the `inputs`
|
||||
# context — release.yml already uses that form, and it is the one
|
||||
# this runner is known to evaluate. Read through env rather than
|
||||
# interpolated into the run block, same rule as release.yml's TAG.
|
||||
FORCE: ${{ github.event.inputs.force_build }}
|
||||
# A scheduled refresh has to bypass reuse by construction: it
|
||||
# rebuilds the SAME source, so fc.revision always matches and the
|
||||
# check would skip every refresh there has ever been.
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
set -eu
|
||||
DERIVED=$(sh scripts/artifacts.sh revision web)
|
||||
@@ -520,7 +668,17 @@ jobs:
|
||||
echo "reuse: NOTE tag is being index-wrapped and reuse is dead."
|
||||
fi
|
||||
|
||||
if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||
# FORCE is checked here rather than in the build step's `if:`, so
|
||||
# that one decision drives everything downstream. The repoint step
|
||||
# keys off `hit` too, and a force that bypassed only the build would
|
||||
# leave the two disagreeing about what just happened.
|
||||
if [ "${FORCE:-false}" = "true" ]; then
|
||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: force_build set — building regardless"
|
||||
elif [ "${EVENT:-}" = "schedule" ]; then
|
||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: scheduled base refresh — building regardless"
|
||||
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||
echo "hit=true" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: already published — skipping the build"
|
||||
else
|
||||
@@ -611,6 +769,37 @@ jobs:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
push: true
|
||||
# Re-resolve the FROM references against the registry instead of
|
||||
# trusting whatever digest the cache was built against. This is the
|
||||
# whole mechanism of the scheduled refresh (#3154): if the base tag
|
||||
# moved, the FROM layer's cache key changes, every layer above it
|
||||
# invalidates, and the image genuinely rebuilds.
|
||||
#
|
||||
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
||||
# did NOT move, the build is ~13s and every content step reports
|
||||
# CACHED — but the channel tag STILL gets a new manifest digest.
|
||||
# buildkit mints a fresh image config each run, so identical layers
|
||||
# are republished under a new config blob. All three images moved
|
||||
# that way on 2026-08-30 with nothing whatsoever changed in them.
|
||||
#
|
||||
# So a refresh currently rewrites :latest every Sunday whether or
|
||||
# not there is anything new in it, and :c-<sha> is handed a new
|
||||
# manifest to diverge from on the same cadence. Layers are shared,
|
||||
# so the storage cost is a config blob; the cost that matters is
|
||||
# that a digest change no longer MEANS anything. Tracked in #3265 —
|
||||
# the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
|
||||
# make "same source, same bytes" true and turn the no-op case into
|
||||
# a genuine no-op.
|
||||
#
|
||||
# What `pull` does NOT catch either: a Debian package update inside
|
||||
# the `apt-get install` layer while the base tag itself stands
|
||||
# still. The official python/cuda images rebuild with those updates
|
||||
# baked in, so this is a lag rather than a hole; closing it needs
|
||||
# `no-cache: true`, which is a much larger version of the same
|
||||
# churn #3265 is about.
|
||||
#
|
||||
# Only on the schedule. An ordinary push wants the cached base.
|
||||
pull: ${{ github.event_name == 'schedule' }}
|
||||
# ONE tag, the channel's. Every other tag is written by the step
|
||||
# below, registry-side. buildx here pushes the first tag to the
|
||||
# registry and then re-pushes the rest through the DOCKER driver,
|
||||
@@ -623,6 +812,40 @@ jobs:
|
||||
# decoration — an unstamped image is one that will always rebuild.
|
||||
labels: |
|
||||
fc.revision=${{ steps.reuse.outputs.revision }}
|
||||
# LOAD-BEARING, not a preference. On the default docker driver these
|
||||
# were no-ops; on the docker-container driver above,
|
||||
# build-push-action@v5 defaults provenance to TRUE when pushing.
|
||||
# Provenance attaches an attestation manifest, which makes the pushed
|
||||
# tag a manifest INDEX — and `.Image.Config.Labels` does not resolve
|
||||
# through an index.
|
||||
#
|
||||
# The label directly above IS the reuse key. Wrap the channel tag in
|
||||
# an index and the next push reads fc.revision=<none>, misses, and
|
||||
# rebuilds. Then so does the one after that, forever. Nothing fails,
|
||||
# nothing goes red, and the only symptom is the bill. That is #3183
|
||||
# arriving through a different door, and note #3127 §4 records the
|
||||
# same shape for `platforms:`.
|
||||
provenance: false
|
||||
sbom: false
|
||||
# The ONLY cache this driver can have. `docker-container` gets a
|
||||
# FRESH buildkit instance per job, so unlike the default docker
|
||||
# driver it has no local layer store to fall back on — measured on
|
||||
# run 4896, the first builds after the driver change: web 3m44s
|
||||
# (was 2m23s), ml 3m49s (was 3m20s), agent 11m12s (was 9m26s). The
|
||||
# driver change ALONE is a regression; this is the other half of it.
|
||||
#
|
||||
# mode=max so intermediate stages cache too. web's frontend-builder
|
||||
# stage and the agent's two ~150s pip layers are the whole cost, and
|
||||
# they are exactly what a min-mode cache would drop.
|
||||
#
|
||||
# A `:buildcache` tag is NOT the withdrawn tag scheme coming back.
|
||||
# Rule 145 narrowed against names NOTHING reads; this one is read by
|
||||
# every build that runs, is one moving ref per image rather than one
|
||||
# per build, holds cache blobs rather than a shippable artifact, and
|
||||
# is overwritten in place rather than accumulating. It is closer to
|
||||
# :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.)
|
||||
cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator:buildcache
|
||||
cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator:buildcache,mode=max
|
||||
# Only the web image carries these: it is the one with a UI and an
|
||||
# HTTP surface to report them on. The ml and agent images have
|
||||
# nothing to tell.
|
||||
@@ -699,6 +922,12 @@ jobs:
|
||||
ARGS="$ARGS -t $t"
|
||||
done
|
||||
unset IFS
|
||||
#
|
||||
# This is also the whole of the scheduled refresh's tag handling
|
||||
# (#3154): a refresh's tag list is the channel tag alone, so SOURCE
|
||||
# is the only entry, it gets excluded, and this step correctly does
|
||||
# nothing. No `if:` on the step and no schedule special-case —
|
||||
# excluding the source was already the right rule.
|
||||
if [ -z "$ARGS" ]; then
|
||||
echo "repoint: $SOURCE is the only tag for this channel and"
|
||||
echo "repoint: already holds this revision — nothing to write."
|
||||
@@ -715,6 +944,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history: this job derives its artifact's version from the
|
||||
# commit its shipped files last changed in (milestone 313). A
|
||||
# depth-1 clone cannot see that commit — it either derives a wrong,
|
||||
@@ -722,6 +955,20 @@ jobs:
|
||||
# the build would otherwise notice.
|
||||
fetch-depth: 0
|
||||
|
||||
# See sign-extension's copy for why this guard exists.
|
||||
- name: Guard — a scheduled run must have checked out main
|
||||
if: github.event_name == 'schedule'
|
||||
run: |
|
||||
set -eu
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))"
|
||||
if [ "$BRANCH" != "main" ]; then
|
||||
echo "schedule: expected main, got '$BRANCH'." >&2
|
||||
echo "schedule: BUILD_REF was not honoured by the runner." >&2
|
||||
echo "schedule: refusing to publish a channel tag from it." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- derived values, one line (milestone 313) ------------------------
|
||||
# These stopped being shadow output at step 3. `revision` decides
|
||||
# whether the build below runs at all and `version` is what the image
|
||||
@@ -759,8 +1006,12 @@ jobs:
|
||||
# everywhere). Operator-flagged 2026-06-01 after first :c-<sha>
|
||||
# main-push build failed at this step.
|
||||
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||
# Mirrors build-web's tag list; see the comment there.
|
||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
# Mirrors build-web's tag list and its schedule handling; see
|
||||
# the comments there.
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
@@ -776,6 +1027,23 @@ jobs:
|
||||
ACTOR: ${{ github.actor }}
|
||||
run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin
|
||||
|
||||
# A REAL buildx builder, not the default `docker` driver (#3114, #3190).
|
||||
#
|
||||
# The default driver builds through the local dockerd. It cannot export a
|
||||
# registry cache at all — which is why the agent rebuilds a ~6.3 GB CUDA
|
||||
# + torch image from scratch whenever the runner's local cache is cold,
|
||||
# measured at 9m26s against 7s warm. It is also #3190's leading suspect:
|
||||
# after a registry-direct push it resolves image metadata against a local
|
||||
# store the push never filled, and reports `No such image` on an image
|
||||
# that published perfectly well three seconds earlier.
|
||||
#
|
||||
# These jobs run INSIDE a container against a mounted docker socket, so
|
||||
# the buildkit container this starts is a SIBLING of the job container,
|
||||
# not a child. That works over the socket mount; it had never been tried
|
||||
# here before milestone 326 step 1.
|
||||
- name: Set up buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# --- reuse-if-published (milestone 313, step 4) ----------------------
|
||||
# Does the image the channel tag already points at carry THIS commit's
|
||||
# revision? If so the bytes this job would produce are already published
|
||||
@@ -814,6 +1082,16 @@ jobs:
|
||||
env:
|
||||
IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-ml
|
||||
CHANNEL: ${{ steps.tag.outputs.channel }}
|
||||
# Empty on a push; the string "true" only from a workflow_dispatch
|
||||
# that asked for it. `github.event.inputs` rather than the `inputs`
|
||||
# context — release.yml already uses that form, and it is the one
|
||||
# this runner is known to evaluate. Read through env rather than
|
||||
# interpolated into the run block, same rule as release.yml's TAG.
|
||||
FORCE: ${{ github.event.inputs.force_build }}
|
||||
# A scheduled refresh has to bypass reuse by construction: it
|
||||
# rebuilds the SAME source, so fc.revision always matches and the
|
||||
# check would skip every refresh there has ever been.
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
set -eu
|
||||
DERIVED=$(sh scripts/artifacts.sh revision ml)
|
||||
@@ -849,7 +1127,17 @@ jobs:
|
||||
echo "reuse: NOTE tag is being index-wrapped and reuse is dead."
|
||||
fi
|
||||
|
||||
if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||
# FORCE is checked here rather than in the build step's `if:`, so
|
||||
# that one decision drives everything downstream. The repoint step
|
||||
# keys off `hit` too, and a force that bypassed only the build would
|
||||
# leave the two disagreeing about what just happened.
|
||||
if [ "${FORCE:-false}" = "true" ]; then
|
||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: force_build set — building regardless"
|
||||
elif [ "${EVENT:-}" = "schedule" ]; then
|
||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: scheduled base refresh — building regardless"
|
||||
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||
echo "hit=true" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: already published — skipping the build"
|
||||
else
|
||||
@@ -864,6 +1152,37 @@ jobs:
|
||||
context: .
|
||||
file: Dockerfile.ml
|
||||
push: true
|
||||
# Re-resolve the FROM references against the registry instead of
|
||||
# trusting whatever digest the cache was built against. This is the
|
||||
# whole mechanism of the scheduled refresh (#3154): if the base tag
|
||||
# moved, the FROM layer's cache key changes, every layer above it
|
||||
# invalidates, and the image genuinely rebuilds.
|
||||
#
|
||||
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
||||
# did NOT move, the build is ~13s and every content step reports
|
||||
# CACHED — but the channel tag STILL gets a new manifest digest.
|
||||
# buildkit mints a fresh image config each run, so identical layers
|
||||
# are republished under a new config blob. All three images moved
|
||||
# that way on 2026-08-30 with nothing whatsoever changed in them.
|
||||
#
|
||||
# So a refresh currently rewrites :latest every Sunday whether or
|
||||
# not there is anything new in it, and :c-<sha> is handed a new
|
||||
# manifest to diverge from on the same cadence. Layers are shared,
|
||||
# so the storage cost is a config blob; the cost that matters is
|
||||
# that a digest change no longer MEANS anything. Tracked in #3265 —
|
||||
# the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
|
||||
# make "same source, same bytes" true and turn the no-op case into
|
||||
# a genuine no-op.
|
||||
#
|
||||
# What `pull` does NOT catch either: a Debian package update inside
|
||||
# the `apt-get install` layer while the base tag itself stands
|
||||
# still. The official python/cuda images rebuild with those updates
|
||||
# baked in, so this is a lag rather than a hole; closing it needs
|
||||
# `no-cache: true`, which is a much larger version of the same
|
||||
# churn #3265 is about.
|
||||
#
|
||||
# Only on the schedule. An ordinary push wants the cached base.
|
||||
pull: ${{ github.event_name == 'schedule' }}
|
||||
# ONE tag, the channel's. Every other tag is written by the step
|
||||
# below, registry-side. buildx here pushes the first tag to the
|
||||
# registry and then re-pushes the rest through the DOCKER driver,
|
||||
@@ -876,6 +1195,40 @@ jobs:
|
||||
# decoration — an unstamped image is one that will always rebuild.
|
||||
labels: |
|
||||
fc.revision=${{ steps.reuse.outputs.revision }}
|
||||
# LOAD-BEARING, not a preference. On the default docker driver these
|
||||
# were no-ops; on the docker-container driver above,
|
||||
# build-push-action@v5 defaults provenance to TRUE when pushing.
|
||||
# Provenance attaches an attestation manifest, which makes the pushed
|
||||
# tag a manifest INDEX — and `.Image.Config.Labels` does not resolve
|
||||
# through an index.
|
||||
#
|
||||
# The label directly above IS the reuse key. Wrap the channel tag in
|
||||
# an index and the next push reads fc.revision=<none>, misses, and
|
||||
# rebuilds. Then so does the one after that, forever. Nothing fails,
|
||||
# nothing goes red, and the only symptom is the bill. That is #3183
|
||||
# arriving through a different door, and note #3127 §4 records the
|
||||
# same shape for `platforms:`.
|
||||
provenance: false
|
||||
sbom: false
|
||||
# The ONLY cache this driver can have. `docker-container` gets a
|
||||
# FRESH buildkit instance per job, so unlike the default docker
|
||||
# driver it has no local layer store to fall back on — measured on
|
||||
# run 4896, the first builds after the driver change: web 3m44s
|
||||
# (was 2m23s), ml 3m49s (was 3m20s), agent 11m12s (was 9m26s). The
|
||||
# driver change ALONE is a regression; this is the other half of it.
|
||||
#
|
||||
# mode=max so intermediate stages cache too. web's frontend-builder
|
||||
# stage and the agent's two ~150s pip layers are the whole cost, and
|
||||
# they are exactly what a min-mode cache would drop.
|
||||
#
|
||||
# A `:buildcache` tag is NOT the withdrawn tag scheme coming back.
|
||||
# Rule 145 narrowed against names NOTHING reads; this one is read by
|
||||
# every build that runs, is one moving ref per image rather than one
|
||||
# per build, holds cache blobs rather than a shippable artifact, and
|
||||
# is overwritten in place rather than accumulating. It is closer to
|
||||
# :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.)
|
||||
cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache
|
||||
cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-ml:buildcache,mode=max
|
||||
|
||||
# Every tag but the channel's own is written HERE, registry-side,
|
||||
# whether or not a build ran. Each -t becomes another reference to the
|
||||
@@ -965,6 +1318,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history: this job derives its artifact's version from the
|
||||
# commit its shipped files last changed in (milestone 313). A
|
||||
# depth-1 clone cannot see that commit — it either derives a wrong,
|
||||
@@ -972,6 +1329,20 @@ jobs:
|
||||
# the build would otherwise notice.
|
||||
fetch-depth: 0
|
||||
|
||||
# See sign-extension's copy for why this guard exists.
|
||||
- name: Guard — a scheduled run must have checked out main
|
||||
if: github.event_name == 'schedule'
|
||||
run: |
|
||||
set -eu
|
||||
BRANCH=$(git rev-parse --abbrev-ref HEAD)
|
||||
echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))"
|
||||
if [ "$BRANCH" != "main" ]; then
|
||||
echo "schedule: expected main, got '$BRANCH'." >&2
|
||||
echo "schedule: BUILD_REF was not honoured by the runner." >&2
|
||||
echo "schedule: refusing to publish a channel tag from it." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# --- derived values, one line (milestone 313) ------------------------
|
||||
# These stopped being shadow output at step 3. `revision` decides
|
||||
# whether the build below runs at all and `version` is what the image
|
||||
@@ -1004,8 +1375,12 @@ jobs:
|
||||
id: tag
|
||||
run: |
|
||||
SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7)
|
||||
# Mirrors build-web's tag list; see the comment there.
|
||||
if [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
# Mirrors build-web's tag list and its schedule handling; see
|
||||
# the comments there.
|
||||
if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${GITHUB_REF##*/}" = "main" ]; then
|
||||
echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:latest,git.fabledsword.com/bvandeusen/fabledcurator-agent:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
|
||||
echo "channel=main" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
@@ -1021,6 +1396,23 @@ jobs:
|
||||
ACTOR: ${{ github.actor }}
|
||||
run: echo "$TOKEN" | docker login git.fabledsword.com -u "$ACTOR" --password-stdin
|
||||
|
||||
# A REAL buildx builder, not the default `docker` driver (#3114, #3190).
|
||||
#
|
||||
# The default driver builds through the local dockerd. It cannot export a
|
||||
# registry cache at all — which is why the agent rebuilds a ~6.3 GB CUDA
|
||||
# + torch image from scratch whenever the runner's local cache is cold,
|
||||
# measured at 9m26s against 7s warm. It is also #3190's leading suspect:
|
||||
# after a registry-direct push it resolves image metadata against a local
|
||||
# store the push never filled, and reports `No such image` on an image
|
||||
# that published perfectly well three seconds earlier.
|
||||
#
|
||||
# These jobs run INSIDE a container against a mounted docker socket, so
|
||||
# the buildkit container this starts is a SIBLING of the job container,
|
||||
# not a child. That works over the socket mount; it had never been tried
|
||||
# here before milestone 326 step 1.
|
||||
- name: Set up buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
# --- reuse-if-published (milestone 313, step 4) ----------------------
|
||||
# Does the image the channel tag already points at carry THIS commit's
|
||||
# revision? If so the bytes this job would produce are already published
|
||||
@@ -1059,6 +1451,16 @@ jobs:
|
||||
env:
|
||||
IMAGE: git.fabledsword.com/bvandeusen/fabledcurator-agent
|
||||
CHANNEL: ${{ steps.tag.outputs.channel }}
|
||||
# Empty on a push; the string "true" only from a workflow_dispatch
|
||||
# that asked for it. `github.event.inputs` rather than the `inputs`
|
||||
# context — release.yml already uses that form, and it is the one
|
||||
# this runner is known to evaluate. Read through env rather than
|
||||
# interpolated into the run block, same rule as release.yml's TAG.
|
||||
FORCE: ${{ github.event.inputs.force_build }}
|
||||
# A scheduled refresh has to bypass reuse by construction: it
|
||||
# rebuilds the SAME source, so fc.revision always matches and the
|
||||
# check would skip every refresh there has ever been.
|
||||
EVENT: ${{ github.event_name }}
|
||||
run: |
|
||||
set -eu
|
||||
DERIVED=$(sh scripts/artifacts.sh revision agent)
|
||||
@@ -1094,7 +1496,17 @@ jobs:
|
||||
echo "reuse: NOTE tag is being index-wrapped and reuse is dead."
|
||||
fi
|
||||
|
||||
if [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||
# FORCE is checked here rather than in the build step's `if:`, so
|
||||
# that one decision drives everything downstream. The repoint step
|
||||
# keys off `hit` too, and a force that bypassed only the build would
|
||||
# leave the two disagreeing about what just happened.
|
||||
if [ "${FORCE:-false}" = "true" ]; then
|
||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: force_build set — building regardless"
|
||||
elif [ "${EVENT:-}" = "schedule" ]; then
|
||||
echo "hit=false" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: scheduled base refresh — building regardless"
|
||||
elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then
|
||||
echo "hit=true" >> "$GITHUB_OUTPUT"
|
||||
echo "reuse: already published — skipping the build"
|
||||
else
|
||||
@@ -1109,6 +1521,37 @@ jobs:
|
||||
context: agent
|
||||
file: agent/Dockerfile
|
||||
push: true
|
||||
# Re-resolve the FROM references against the registry instead of
|
||||
# trusting whatever digest the cache was built against. This is the
|
||||
# whole mechanism of the scheduled refresh (#3154): if the base tag
|
||||
# moved, the FROM layer's cache key changes, every layer above it
|
||||
# invalidates, and the image genuinely rebuilds.
|
||||
#
|
||||
# MEASURED on the first real fire, run 4934 (#3265): when the base
|
||||
# did NOT move, the build is ~13s and every content step reports
|
||||
# CACHED — but the channel tag STILL gets a new manifest digest.
|
||||
# buildkit mints a fresh image config each run, so identical layers
|
||||
# are republished under a new config blob. All three images moved
|
||||
# that way on 2026-08-30 with nothing whatsoever changed in them.
|
||||
#
|
||||
# So a refresh currently rewrites :latest every Sunday whether or
|
||||
# not there is anything new in it, and :c-<sha> is handed a new
|
||||
# manifest to diverge from on the same cadence. Layers are shared,
|
||||
# so the storage cost is a config blob; the cost that matters is
|
||||
# that a digest change no longer MEANS anything. Tracked in #3265 —
|
||||
# the likely fix is a deterministic SOURCE_DATE_EPOCH, which would
|
||||
# make "same source, same bytes" true and turn the no-op case into
|
||||
# a genuine no-op.
|
||||
#
|
||||
# What `pull` does NOT catch either: a Debian package update inside
|
||||
# the `apt-get install` layer while the base tag itself stands
|
||||
# still. The official python/cuda images rebuild with those updates
|
||||
# baked in, so this is a lag rather than a hole; closing it needs
|
||||
# `no-cache: true`, which is a much larger version of the same
|
||||
# churn #3265 is about.
|
||||
#
|
||||
# Only on the schedule. An ordinary push wants the cached base.
|
||||
pull: ${{ github.event_name == 'schedule' }}
|
||||
# ONE tag, the channel's. Every other tag is written by the step
|
||||
# below, registry-side. buildx here pushes the first tag to the
|
||||
# registry and then re-pushes the rest through the DOCKER driver,
|
||||
@@ -1121,6 +1564,40 @@ jobs:
|
||||
# decoration — an unstamped image is one that will always rebuild.
|
||||
labels: |
|
||||
fc.revision=${{ steps.reuse.outputs.revision }}
|
||||
# LOAD-BEARING, not a preference. On the default docker driver these
|
||||
# were no-ops; on the docker-container driver above,
|
||||
# build-push-action@v5 defaults provenance to TRUE when pushing.
|
||||
# Provenance attaches an attestation manifest, which makes the pushed
|
||||
# tag a manifest INDEX — and `.Image.Config.Labels` does not resolve
|
||||
# through an index.
|
||||
#
|
||||
# The label directly above IS the reuse key. Wrap the channel tag in
|
||||
# an index and the next push reads fc.revision=<none>, misses, and
|
||||
# rebuilds. Then so does the one after that, forever. Nothing fails,
|
||||
# nothing goes red, and the only symptom is the bill. That is #3183
|
||||
# arriving through a different door, and note #3127 §4 records the
|
||||
# same shape for `platforms:`.
|
||||
provenance: false
|
||||
sbom: false
|
||||
# The ONLY cache this driver can have. `docker-container` gets a
|
||||
# FRESH buildkit instance per job, so unlike the default docker
|
||||
# driver it has no local layer store to fall back on — measured on
|
||||
# run 4896, the first builds after the driver change: web 3m44s
|
||||
# (was 2m23s), ml 3m49s (was 3m20s), agent 11m12s (was 9m26s). The
|
||||
# driver change ALONE is a regression; this is the other half of it.
|
||||
#
|
||||
# mode=max so intermediate stages cache too. web's frontend-builder
|
||||
# stage and the agent's two ~150s pip layers are the whole cost, and
|
||||
# they are exactly what a min-mode cache would drop.
|
||||
#
|
||||
# A `:buildcache` tag is NOT the withdrawn tag scheme coming back.
|
||||
# Rule 145 narrowed against names NOTHING reads; this one is read by
|
||||
# every build that runs, is one moving ref per image rather than one
|
||||
# per build, holds cache blobs rather than a shippable artifact, and
|
||||
# is overwritten in place rather than accumulating. It is closer to
|
||||
# :dev than to the :2026.8.28 tags milestone 318 deleted. (#3114.)
|
||||
cache-from: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache
|
||||
cache-to: type=registry,ref=git.fabledsword.com/bvandeusen/fabledcurator-agent:buildcache,mode=max
|
||||
|
||||
# Every tag but the channel's own is written HERE, registry-side,
|
||||
# whether or not a build ran. Each -t becomes another reference to the
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
# Contributing
|
||||
|
||||
FabledCurator is developed by a single maintainer for their own use, and
|
||||
published because it may be useful to others. That shapes what contribution
|
||||
looks like here.
|
||||
|
||||
**Issues are welcome** — bug reports, and questions about running it, are
|
||||
genuinely useful and often the fastest way to find out that something is
|
||||
broken outside the one environment it was built in.
|
||||
|
||||
**Open an issue before writing a pull request.** Not as a formality: the
|
||||
project has opinions that are not obvious from the code, and it is unpleasant
|
||||
for everyone when a finished patch turns out to conflict with one. A short
|
||||
issue first costs you nothing and may save you an evening.
|
||||
|
||||
**Contributions are licensed under the AGPL-3.0**, like the rest of the
|
||||
project. By submitting one you agree it ships under that licence. There is no
|
||||
CLA and no copyright assignment.
|
||||
|
||||
## Running it for development
|
||||
|
||||
```bash
|
||||
docker compose up -d # UI on http://localhost:8080
|
||||
```
|
||||
|
||||
The dev override (`docker-compose.override.yml`) is auto-merged and builds the
|
||||
app images locally from source, so this needs no `.env` and no registry
|
||||
access. Postgres and Redis ports are exposed on the host.
|
||||
|
||||
## What CI checks
|
||||
|
||||
Every push runs these, and they are the definition of done for a change:
|
||||
|
||||
```bash
|
||||
ruff check backend/ tests/ alembic/ agent/ scripts/ # lint (and import order)
|
||||
pytest tests/ -m "not integration" # backend unit tests
|
||||
pytest tests/ -m integration # needs pgvector + redis
|
||||
cd frontend && npm run test:unit && npm run build # frontend
|
||||
```
|
||||
|
||||
The integration lane builds its schema by running the real migrations
|
||||
(`alembic upgrade head`), never from ORM metadata — so a migration that does
|
||||
not apply cleanly fails CI rather than being discovered later.
|
||||
|
||||
Note for the linter: ruff's isort runs with `order-by-type`, which sorts
|
||||
ALL-CAPS names ahead of CamelCase. `from sqlalchemy import JSON, DateTime, ...`
|
||||
is correct; putting `JSON` alphabetically between `Integer` and `String` is
|
||||
not. This catches people out.
|
||||
|
||||
## Database changes
|
||||
|
||||
The ORM models and the migration chain must agree. This is enforced, and it is
|
||||
enforced because they silently diverged for a long time and nobody noticed
|
||||
until they were compared: the models were missing indexes, defaults and
|
||||
uniqueness guarantees that only ever existed inside a migration, which made
|
||||
`alembic revision --autogenerate` actively unsafe to run.
|
||||
|
||||
So: if you change a model, write the migration; if you write a migration,
|
||||
change the model to match. Both, in the same commit.
|
||||
|
||||
Adding a value to a CHECK-constrained column means swapping the constraint in
|
||||
the same change — the constraint is not documentation, and a new value without
|
||||
it fails at insert time.
|
||||
|
||||
## Branch model
|
||||
|
||||
`dev` is where work happens. `main` is production and is only reached by a
|
||||
merge from `dev`, never pushed to directly. If you are sending a pull request,
|
||||
target `dev`.
|
||||
|
||||
## Style
|
||||
|
||||
Match the surrounding code. The one convention worth stating explicitly is
|
||||
that comments here explain *why*, especially where a choice looks wrong at a
|
||||
glance — a comment recording which migration a constraint came from, or why a
|
||||
default is a `text()` rather than a string, is the kind that has repeatedly
|
||||
turned out to be worth its space.
|
||||
@@ -0,0 +1,661 @@
|
||||
GNU AFFERO GENERAL PUBLIC LICENSE
|
||||
Version 3, 19 November 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU Affero General Public License is a free, copyleft license for
|
||||
software and other kinds of works, specifically designed to ensure
|
||||
cooperation with the community in the case of network server software.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
our General Public Licenses are intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
Developers that use our General Public Licenses protect your rights
|
||||
with two steps: (1) assert copyright on the software, and (2) offer
|
||||
you this License which gives you legal permission to copy, distribute
|
||||
and/or modify the software.
|
||||
|
||||
A secondary benefit of defending all users' freedom is that
|
||||
improvements made in alternate versions of the program, if they
|
||||
receive widespread use, become available for other developers to
|
||||
incorporate. Many developers of free software are heartened and
|
||||
encouraged by the resulting cooperation. However, in the case of
|
||||
software used on network servers, this result may fail to come about.
|
||||
The GNU General Public License permits making a modified version and
|
||||
letting the public access it on a server without ever releasing its
|
||||
source code to the public.
|
||||
|
||||
The GNU Affero General Public License is designed specifically to
|
||||
ensure that, in such cases, the modified source code becomes available
|
||||
to the community. It requires the operator of a network server to
|
||||
provide the source code of the modified version running there to the
|
||||
users of that server. Therefore, public use of a modified version, on
|
||||
a publicly accessible server, gives the public access to the source
|
||||
code of the modified version.
|
||||
|
||||
An older license, called the Affero General Public License and
|
||||
published by Affero, was designed to accomplish similar goals. This is
|
||||
a different license, not a version of the Affero GPL, but Affero has
|
||||
released a new version of the Affero GPL which permits relicensing under
|
||||
this license.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU Affero General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Remote Network Interaction; Use with the GNU General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, if you modify the
|
||||
Program, your modified version must prominently offer all users
|
||||
interacting with it remotely through a computer network (if your version
|
||||
supports such interaction) an opportunity to receive the Corresponding
|
||||
Source of your version by providing access to the Corresponding Source
|
||||
from a network server at no charge, through some standard or customary
|
||||
means of facilitating copying of software. This Corresponding Source
|
||||
shall include the Corresponding Source for any work covered by version 3
|
||||
of the GNU General Public License that is incorporated pursuant to the
|
||||
following paragraph.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the work with which it is combined will remain governed by version
|
||||
3 of the GNU General Public License.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU Affero General Public License from time to time. Such new versions
|
||||
will be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU Affero General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU Affero General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU Affero General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If your software can interact with users remotely through a computer
|
||||
network, you should also make sure that it provides a way for users to
|
||||
get its source. For example, if your program is a web application, its
|
||||
interface could display a "Source" link that leads users to an archive
|
||||
of the code. There are many ways you could offer source, and different
|
||||
solutions will be better for different programs; see section 13 for the
|
||||
specific requirements.
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU AGPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
@@ -69,9 +69,19 @@ or a `.env` file (see `.env.example` for the variable names) and use:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.yml up -d
|
||||
# (skips the override so containers pull registry images)
|
||||
# (skips the dev override, so containers pull published :latest images)
|
||||
```
|
||||
|
||||
`-f` is doing real work there: it tells Compose to use *only* that file, which
|
||||
skips `docker-compose.override.yml` and its local builds. What you get is the
|
||||
`:latest` images — the stable channel, built from `main`. This is the install
|
||||
path, and it is the one to use if you are running FabledCurator rather than
|
||||
working on it.
|
||||
|
||||
`:dev` is the other channel: rebuilt from the `dev` branch several times a day,
|
||||
bleeding edge, no stability promise. Nothing in this repo points an installer at
|
||||
it, and nothing should.
|
||||
|
||||
The GPU agent is deployed separately, on the machine with the card —
|
||||
`agent/docker-compose.yml`, not this stack.
|
||||
|
||||
@@ -110,4 +120,18 @@ version.
|
||||
|
||||
## License
|
||||
|
||||
Personal project; use at your own discretion.
|
||||
**GNU Affero General Public License v3.0** — see [LICENSE](LICENSE).
|
||||
|
||||
You may run, study, modify and redistribute this software. The condition is
|
||||
reciprocity: if you distribute a modified version, or **run one as a network
|
||||
service that other people use**, you must offer those users the corresponding
|
||||
source under the same licence. That second clause (AGPL §13) is the reason this
|
||||
licence rather than the GPL — for a self-hosted web application, "distribution"
|
||||
otherwise never happens, and the obligation would never bite.
|
||||
|
||||
Running an unmodified copy for yourself, your household or your organisation
|
||||
carries no obligation at all. Neither does modifying it privately. The licence
|
||||
asks something of you only when you hand your modified version to others.
|
||||
|
||||
Contributions ship under the same licence — see [CONTRIBUTING](CONTRIBUTING.md).
|
||||
Security reports: [SECURITY.md](SECURITY.md).
|
||||
|
||||
+63
@@ -0,0 +1,63 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a vulnerability
|
||||
|
||||
**Please do not put vulnerability details in a public issue.**
|
||||
|
||||
This project has no private disclosure channel yet. Until it does, open an
|
||||
issue on the repository that says only that you have a security report — no
|
||||
reproduction steps, no affected endpoint, no payload — and a maintainer will
|
||||
reply with a private contact to send the details to.
|
||||
|
||||
That is a deliberately awkward first step, and it exists because the
|
||||
alternative is worse: an issue tracker is public the moment it is written to,
|
||||
and every self-hosted instance stays vulnerable until its operator has had a
|
||||
chance to update.
|
||||
|
||||
Please include, once you have a private channel:
|
||||
|
||||
- what an attacker can do, and what access they need to start
|
||||
- the version or commit you tested
|
||||
- reproduction steps
|
||||
|
||||
## Scope — what this software actually handles
|
||||
|
||||
FabledCurator is self-hosted and holds things worth stating plainly, because
|
||||
they shape what counts as a serious bug here:
|
||||
|
||||
- **Platform credentials.** The app captures and stores session cookies for
|
||||
third-party subscription sites (Patreon, SubscribeStar, Pixiv) so it can
|
||||
download on the operator's behalf. These are live credentials for accounts
|
||||
that usually carry a payment method. Anything that discloses them, decrypts
|
||||
them, or lets one user of a shared instance read another's is high severity.
|
||||
- **An extension API key.** The Firefox extension authenticates to the backend
|
||||
with a shared key. Anything that leaks it or lets it be bypassed is a way in.
|
||||
- **A multi-user sharing ACL.** Instances can be shared. A bug that lets one
|
||||
account see content another has not shared is an access-control failure, not
|
||||
a cosmetic one.
|
||||
- **Arbitrary media from the internet.** Downloaded files are decoded, hashed,
|
||||
thumbnailed and fed to ML models. Anything that turns a hostile file into
|
||||
code execution is in scope.
|
||||
|
||||
## Deployment posture — read this before reporting
|
||||
|
||||
FabledCurator is designed to run **inside a private network, over plain HTTP**.
|
||||
It does not terminate TLS, redirect to HTTPS, or set HSTS; if you want
|
||||
transport security, terminate it at your reverse proxy. This is a documented
|
||||
design decision, not an oversight.
|
||||
|
||||
Reports that reduce to "the application is served over HTTP" or "there is no
|
||||
HSTS header" describe that decision rather than a vulnerability. Reports that
|
||||
an authenticated operator can cause the software to do something destructive
|
||||
are usually also by design — the operator is the administrator of their own
|
||||
instance.
|
||||
|
||||
What remains in scope is everything that crosses a boundary the software is
|
||||
supposed to hold: between one user and another, between an unauthenticated
|
||||
visitor and any of it, and between untrusted downloaded content and the host.
|
||||
|
||||
## Supported versions
|
||||
|
||||
Fixes land on the `main` branch and reach the `:latest` image. There are no
|
||||
maintained release branches — the supported version is the current one, and
|
||||
the remedy for a security issue is to update.
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Reconcile the database with what the models have always claimed (#3275).
|
||||
|
||||
Milestone 328 discovered ~130 places where the ORM models and the deployed
|
||||
schema disagreed. Almost all of them were the MODEL being wrong — missing
|
||||
`server_default`s, indexes and CHECK constraints that only ever existed in a
|
||||
migration — and those are fixed in the model files with no DDL at all, because
|
||||
the database already had them.
|
||||
|
||||
This migration carries the remainder — the two places where DDL is actually
|
||||
needed, because the database is what is wrong.
|
||||
|
||||
`tag.fandom_id` is declared `index=True` on the model, but no migration ever
|
||||
created that index. Every autogenerate run since would have proposed adding
|
||||
it; nobody ran one, so the model and the database simply drifted apart and
|
||||
stayed that way.
|
||||
|
||||
Deliberately NOT in this migration: anything about `image_record.sha256`. An
|
||||
earlier draft of this file claimed sha256 was not unique in the database and
|
||||
that duplicate rows were therefore possible. That was WRONG, and it was wrong
|
||||
because it was read off `op.create_index("ix_image_record_sha256", ...)` at
|
||||
0001 line 151 without reading line 149 two lines above it:
|
||||
|
||||
sa.UniqueConstraint("sha256", name="uq_image_record_sha256"),
|
||||
|
||||
Uniqueness has been enforced since the initial schema. The database simply
|
||||
expresses it as a CONSTRAINT plus a separate non-unique lookup index, where
|
||||
the model expressed it as one `unique=True, index=True` column — the same
|
||||
guarantee built from different objects, which is why the two schemas did not
|
||||
line up. The model now declares the constraint and the plain index separately,
|
||||
so it describes what is actually there. No DDL is needed for it.
|
||||
|
||||
Also here: six CHECK constraints whose names carry their table prefix TWICE.
|
||||
|
||||
`base.py`'s naming convention is `ck_%(table_name)s_%(constraint_name)s`, and
|
||||
unlike the uq/fk/ix entries it applies even to a constraint that already has a
|
||||
name. Six migrations passed an already-prefixed name, so the convention
|
||||
prefixed it again:
|
||||
|
||||
ck_external_link_ck_external_link_host
|
||||
ck_external_link_ck_external_link_status
|
||||
ck_import_settings_ck_import_settings_singleton
|
||||
ck_ml_settings_ck_ml_settings_singleton
|
||||
ck_post_ck_post_translation_override
|
||||
ck_tag_ck_tag_fandom_requires_character
|
||||
|
||||
Nothing reads a CHECK constraint by name, so this has never done any harm —
|
||||
but it is exactly the development-era residue the collapsed baseline exists to
|
||||
leave behind, and a public schema should not ship it. The models now declare
|
||||
bare names, which the convention renders into the single-prefix form; this
|
||||
renames the deployed constraints to match.
|
||||
|
||||
RENAME CONSTRAINT is a catalog-only operation: no table scan, no rewrite, no
|
||||
validation of existing rows. It takes a brief ACCESS EXCLUSIVE lock and
|
||||
returns. That is why this is safe to do on `post` and `tag`, which are the two
|
||||
large tables in the schema.
|
||||
|
||||
Revision ID: 0088
|
||||
Revises: 0087
|
||||
Create Date: 2026-08-30
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0088"
|
||||
down_revision: Union[str, None] = "0087"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# (table, doubled name, single-prefix name)
|
||||
#
|
||||
# Six, not the four a first read of the migrations turned up. The list that
|
||||
# settles it is the one extracted from the chain's pg_dump by matching
|
||||
# `ck_(\w+?)_ck_\1_` — reading the migrations by eye missed external_link
|
||||
# twice over, in the same way an earlier pass missed a UNIQUE constraint two
|
||||
# lines above the index it was looking at (see the sha256 note above).
|
||||
DOUBLED_CHECKS = (
|
||||
("external_link", "ck_external_link_ck_external_link_host",
|
||||
"ck_external_link_host"),
|
||||
("external_link", "ck_external_link_ck_external_link_status",
|
||||
"ck_external_link_status"),
|
||||
("import_settings", "ck_import_settings_ck_import_settings_singleton",
|
||||
"ck_import_settings_singleton"),
|
||||
("ml_settings", "ck_ml_settings_ck_ml_settings_singleton",
|
||||
"ck_ml_settings_singleton"),
|
||||
("post", "ck_post_ck_post_translation_override",
|
||||
"ck_post_translation_override"),
|
||||
("tag", "ck_tag_ck_tag_fandom_requires_character",
|
||||
"ck_tag_fandom_requires_character"),
|
||||
)
|
||||
|
||||
|
||||
def _rename_check(table: str, old: str, new: str) -> None:
|
||||
# Guarded on pg_constraint rather than run bare: a database built from the
|
||||
# models (a fresh install, or the CI integration schema) already has the
|
||||
# single-prefix name, and this migration must be a no-op there rather than
|
||||
# an error. Same reasoning as the CREATE INDEX IF NOT EXISTS below.
|
||||
op.execute(
|
||||
f"""
|
||||
DO $$
|
||||
BEGIN
|
||||
IF EXISTS (
|
||||
SELECT 1 FROM pg_constraint
|
||||
WHERE conname = '{old}' AND conrelid = '{table}'::regclass
|
||||
) THEN
|
||||
ALTER TABLE {table} RENAME CONSTRAINT {old} TO {new};
|
||||
END IF;
|
||||
END $$;
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# IF NOT EXISTS because the index is what the model already asks for: any
|
||||
# database built from metadata rather than from this chain will have it,
|
||||
# and this migration must be a no-op there rather than an error.
|
||||
op.execute("CREATE INDEX IF NOT EXISTS ix_tag_fandom_id ON tag (fandom_id)")
|
||||
|
||||
for table, old, new in DOUBLED_CHECKS:
|
||||
_rename_check(table, old, new)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
for table, old, new in DOUBLED_CHECKS:
|
||||
_rename_check(table, new, old)
|
||||
|
||||
op.execute("DROP INDEX IF EXISTS ix_tag_fandom_id")
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Index the seven unindexed FKs; drop the seven redundant indexes (#3300, #3301).
|
||||
|
||||
Found by a structural sweep of the deployed schema done AFTER 0088 brought the
|
||||
models and the migration chain into exact agreement. That agreement is what
|
||||
0088 achieved, and it is worth being precise about what it does NOT prove: a
|
||||
models-vs-chain diff shows the two describe the same schema. It says nothing
|
||||
about whether that schema is right. Everything here was wrong in BOTH, which is
|
||||
exactly the class of problem the reconciliation could not see.
|
||||
|
||||
## Added: seven FK indexes
|
||||
|
||||
`image_tag.tag_id` is the one that matters. The table's only index is
|
||||
PRIMARY KEY (image_record_id, tag_id), which leads with the wrong column for
|
||||
the two hottest things done with it:
|
||||
|
||||
* the gallery's tag filter — services/tag_query.py builds
|
||||
`image_tag.c.tag_id == tid` (and `.in_(tids)`) on every tag-scoped browse;
|
||||
* ON DELETE CASCADE from `tag` — deleting or merging a tag makes Postgres
|
||||
find that tag's rows before it can remove them.
|
||||
|
||||
Both had to scan the largest table in the schema. The other six are the same
|
||||
shape on much smaller tables; `presentation_review.tag_id` is the notable one,
|
||||
since it also CASCADEs.
|
||||
|
||||
## Dropped: seven redundant indexes
|
||||
|
||||
`ix_image_record_sha256` was an exact duplicate. A UNIQUE constraint builds its
|
||||
own index, so `uq_image_record_sha256` already covered the column and
|
||||
`image_record` carried two btrees on `sha256` — on the highest-insert-rate
|
||||
table in the system.
|
||||
|
||||
The other six are single-column indexes that a later composite superseded
|
||||
without the narrow one being retired. A btree on (a, b) already serves lookups
|
||||
on `a`, so each was pure write amplification. `task_run` and `backup_run` are
|
||||
append-heavy operational logs, which is where that cost lands hardest.
|
||||
|
||||
Note for anyone reading 0088 next to this: 0088 deliberately taught the models
|
||||
to declare BOTH sha256 indexes, so they would describe reality. That was right.
|
||||
This migration changes the reality instead, and the models change with it.
|
||||
|
||||
## CONCURRENTLY, and why this migration has no transaction
|
||||
|
||||
`CREATE INDEX` takes an ACCESS EXCLUSIVE lock for the whole build, which on
|
||||
`image_tag` means stalling every write for as long as it takes. CONCURRENTLY
|
||||
builds without blocking writers, at the cost of two table passes and an
|
||||
inability to run inside a transaction — hence `autocommit_block()`.
|
||||
|
||||
The consequence to know about: this migration is NOT atomic. If it fails
|
||||
partway, the work already done stays done. Every statement is therefore written
|
||||
IF NOT EXISTS / IF EXISTS so that re-running it after a failure is safe rather
|
||||
than an error.
|
||||
|
||||
A failed CONCURRENTLY build also leaves an INVALID index behind — it is not
|
||||
used by the planner and not repaired automatically. Find them with:
|
||||
|
||||
SELECT c.relname FROM pg_index i
|
||||
JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE NOT i.indisvalid;
|
||||
|
||||
Drop what that returns and re-run; nothing else is needed.
|
||||
|
||||
Revision ID: 0089
|
||||
Revises: 0088
|
||||
Create Date: 2026-08-31
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
|
||||
revision: str = "0089"
|
||||
down_revision: Union[str, None] = "0088"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
# (index name, table, column) — names match what the models render under
|
||||
# base.py's naming convention, so autogenerate stays quiet after this.
|
||||
MISSING_FK_INDEXES = (
|
||||
("ix_image_tag_tag_id", "image_tag", "tag_id"),
|
||||
("ix_presentation_review_tag_id", "presentation_review", "tag_id"),
|
||||
("ix_presentation_review_conflict_tag_id", "presentation_review", "conflict_tag_id"),
|
||||
("ix_import_task_result_image_id", "import_task", "result_image_id"),
|
||||
("ix_external_link_attachment_id", "external_link", "attachment_id"),
|
||||
("ix_character_prototype_region_id", "character_prototype", "region_id"),
|
||||
("ix_backup_run_restored_from_id", "backup_run", "restored_from_id"),
|
||||
)
|
||||
|
||||
# (index name, table, column) — redundant; the second element of each pair in
|
||||
# the docstring is what still covers the column after the drop.
|
||||
REDUNDANT_INDEXES = (
|
||||
("ix_image_record_sha256", "image_record", "sha256"),
|
||||
("ix_backup_run_kind", "backup_run", "kind"),
|
||||
("ix_backup_run_status", "backup_run", "status"),
|
||||
("ix_task_run_queue", "task_run", "queue"),
|
||||
("ix_task_run_status", "task_run", "status"),
|
||||
("ix_task_run_task_name", "task_run", "task_name"),
|
||||
("ix_external_link_post_id", "external_link", "post_id"),
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
for name, table, column in MISSING_FK_INDEXES:
|
||||
op.execute(
|
||||
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} "
|
||||
f"ON {table} ({column})"
|
||||
)
|
||||
for name, _table, _column in REDUNDANT_INDEXES:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
with op.get_context().autocommit_block():
|
||||
for name, table, column in REDUNDANT_INDEXES:
|
||||
op.execute(
|
||||
f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} "
|
||||
f"ON {table} ({column})"
|
||||
)
|
||||
for name, _table, _column in MISSING_FK_INDEXES:
|
||||
op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}")
|
||||
@@ -27,10 +27,10 @@ class Artist(Base):
|
||||
notes: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
# True once a Source is attached; flips false if all sources removed.
|
||||
is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
|
||||
# Per-artist scheduling overrides; null means "use global default".
|
||||
auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||
check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
|
||||
@@ -20,7 +20,7 @@ feedback_check_existing_enums):
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Integer, String, Text
|
||||
from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, Integer, String, Text, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -29,10 +29,21 @@ from .base import Base
|
||||
class BackupRun(Base):
|
||||
__tablename__ = "backup_run"
|
||||
|
||||
|
||||
__table_args__ = (
|
||||
# alembic 0017: reporting indexes, never declared on the model (#3275).
|
||||
Index("ix_backup_run_kind_started", "kind", text("started_at DESC")),
|
||||
Index("ix_backup_run_status_finished", "status", text("finished_at DESC")),
|
||||
Index("ix_backup_run_tag_partial", "tag", postgresql_where=text("tag IS NOT NULL")),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True)
|
||||
# No index=True: ix_backup_run_kind_started (above) already leads with
|
||||
# `kind`, so a single-column index on it was pure write cost (#3301).
|
||||
kind: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="pending", index=True,
|
||||
# No index=True — ix_backup_run_status_finished leads with `status`.
|
||||
String(16), nullable=False, default="pending",
|
||||
server_default="pending",
|
||||
)
|
||||
tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
triggered_by: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
@@ -49,7 +60,9 @@ class BackupRun(Base):
|
||||
manifest: Mapped[dict] = mapped_column(
|
||||
JSON, nullable=False, default=dict, server_default="{}",
|
||||
)
|
||||
# Self-referential FK, unindexed until 0089 (#3300): SET NULL has to find
|
||||
# the rows pointing at a deleted run before it can null them.
|
||||
restored_from_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("backup_run.id", ondelete="SET NULL"),
|
||||
nullable=True,
|
||||
nullable=True, index=True,
|
||||
)
|
||||
|
||||
@@ -40,8 +40,10 @@ class CharacterPrototype(Base):
|
||||
)
|
||||
# Provenance: the region this vector was copied from. SET NULL so pruning a
|
||||
# region doesn't delete the prototype mid-cycle (the next refresh reconciles).
|
||||
# index=True added in 0089 — the FK was unindexed (#3300).
|
||||
region_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True
|
||||
ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True,
|
||||
index=True,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -25,8 +25,8 @@ class DownloadEvent(Base):
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)
|
||||
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default="0")
|
||||
files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
metadata_: Mapped[dict] = mapped_column(
|
||||
"metadata", JSONB, nullable=False, default=dict,
|
||||
|
||||
@@ -16,6 +16,7 @@ doesn't delete the link record).
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
@@ -38,15 +39,33 @@ STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead")
|
||||
class ExternalLink(Base):
|
||||
__tablename__ = "external_link"
|
||||
__table_args__ = (
|
||||
# alembic 0028 enum CHECKs. Rule 36 territory: a new host or status value
|
||||
# needs its constraint swapped in the same migration (#3275).
|
||||
CheckConstraint(
|
||||
"host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')",
|
||||
# Bare name: Base.metadata's naming convention prepends
|
||||
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
||||
# alembic 0088, which renames the four constraints that shipped
|
||||
# that way (#3275).
|
||||
name="host",
|
||||
),
|
||||
CheckConstraint(
|
||||
"status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')",
|
||||
name="status",
|
||||
),
|
||||
# One row per (post, url). The full url (incl. #fragment) is the identity
|
||||
# — the same file linked twice in a post collapses to one row.
|
||||
Index("uq_external_link_post_url", "post_id", "url", unique=True),
|
||||
Index("ix_external_link_status", "status"),
|
||||
# Unindexed FK (#3300).
|
||||
Index("ix_external_link_attachment_id", "attachment_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
# No index=True: uq_external_link_post_url (post_id, url) already leads
|
||||
# with post_id (#3301).
|
||||
post_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
ForeignKey("post.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
artist_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
|
||||
@@ -50,7 +50,8 @@ class GpuJob(Base):
|
||||
# What to compute, e.g. 'ccip' (detect figures + CCIP-embed) or 'siglip_region'.
|
||||
task: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="pending", index=True
|
||||
String(16), nullable=False, default="pending", index=True,
|
||||
server_default="pending",
|
||||
)
|
||||
# pending | leased | done | error
|
||||
lease_token: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
@@ -60,7 +61,7 @@ class GpuJob(Base):
|
||||
lease_expires_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True
|
||||
)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Triage verdict for an ERRORED job (#125): NULL = not yet probed;
|
||||
# 'defect' = the integrity probe says the FILE itself is bad (surfaced for
|
||||
|
||||
@@ -24,10 +24,11 @@ class HeadAutoApplyRun(Base):
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
# dry_run=True is a PREVIEW: scores + counts what WOULD apply, writes nothing
|
||||
# (preview/apply parity, rule 93).
|
||||
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
server_default="running",
|
||||
)
|
||||
# running | ready | error
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
|
||||
@@ -24,9 +24,9 @@ class HeadMetric(Base):
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
# An auto-applied (source='head_auto') tag the operator later REMOVED.
|
||||
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
# A tag with a head that the operator added by HAND (the head missed it).
|
||||
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
|
||||
@@ -19,8 +19,14 @@ class HeadMetricsSnapshot(Base):
|
||||
__tablename__ = "head_metrics_snapshot"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), index=True
|
||||
# Nullable, matching alembic 0060, which declared this column without
|
||||
# `nullable=False`. The model had it as `Mapped[int]` — NOT NULL — which
|
||||
# was simply never true of the database (#3275). Left nullable rather than
|
||||
# tightened: a snapshot of a tag that is later hard-deleted is a row worth
|
||||
# keeping, and the FK is ON DELETE CASCADE, so tightening it would only
|
||||
# change behaviour, not correct a bug.
|
||||
tag_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=True, index=True
|
||||
)
|
||||
# Denormalized so a snapshot stays readable even if the tag is later renamed.
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
@@ -28,9 +34,9 @@ class HeadMetricsSnapshot(Base):
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now(), index=True
|
||||
)
|
||||
# Current count of source='head_auto' applications still standing.
|
||||
n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
# The head's measured quality at snapshot time (null if no head exists).
|
||||
ap: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
precision_cv: Mapped[float | None] = mapped_column(Float, nullable=True)
|
||||
|
||||
@@ -24,7 +24,8 @@ class HeadTrainingRun(Base):
|
||||
# Training parameters: {min_positives, neg_ratio, precision_target, ...}.
|
||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
server_default="running",
|
||||
)
|
||||
# running | ready | error
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
|
||||
@@ -47,8 +47,15 @@ class ImageProvenance(Base):
|
||||
# attachment on the post. NULL for loose downloads and pre-backfill rows.
|
||||
# SET NULL so deleting the archive attachment never destroys the (image,
|
||||
# post) edge — it just forgets which archive it came from.
|
||||
# FK named explicitly: the convention renders this
|
||||
# `fk_image_provenance_from_attachment_id_post_attachment`, but alembic
|
||||
# 0055 created it as `fk_image_provenance_from_attachment` (#3275).
|
||||
from_attachment_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("post_attachment.id", ondelete="SET NULL"),
|
||||
ForeignKey(
|
||||
"post_attachment.id",
|
||||
ondelete="SET NULL",
|
||||
name="fk_image_provenance_from_attachment",
|
||||
),
|
||||
nullable=True, index=True,
|
||||
)
|
||||
captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
|
||||
@@ -14,10 +14,13 @@ from sqlalchemy import (
|
||||
Enum,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
@@ -29,11 +32,38 @@ ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded")
|
||||
class ImageRecord(Base):
|
||||
__tablename__ = "image_record"
|
||||
|
||||
|
||||
__table_args__ = (
|
||||
# alembic 0001. The database enforces sha256 uniqueness with a
|
||||
# CONSTRAINT and carries a SEPARATE non-unique btree index; the model
|
||||
# said `unique=True, index=True`, which collapses both into a single
|
||||
# UNIQUE index under a different name. Same guarantee either way, but
|
||||
# not the same objects, so autogenerate saw a drop and an add (#3275).
|
||||
UniqueConstraint("sha256", name="uq_image_record_sha256"),
|
||||
# alembic 0036, and the last thing in this schema that lived only in a
|
||||
# migration. SQLAlchemy CAN express an hnsw index with an operator
|
||||
# class, so there is no reason for it to be invisible to the models —
|
||||
# and its absence was the quietest failure of the lot: everything
|
||||
# works, similarity search just silently stops using an index.
|
||||
Index(
|
||||
"ix_image_record_siglip_hnsw",
|
||||
"siglip_embedding",
|
||||
postgresql_using="hnsw",
|
||||
postgresql_ops={"siglip_embedding": "vector_cosine_ops"},
|
||||
),
|
||||
# alembic 0035/0071: the date-ordered browse indexes (#3275).
|
||||
Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")),
|
||||
Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
|
||||
# On-disk identity
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False, unique=True)
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True)
|
||||
# Neither unique= nor index=: uq_image_record_sha256 in __table_args__
|
||||
# above creates its own index, and the separate ix_image_record_sha256
|
||||
# that 0001 also built was an exact duplicate of it — dropped in 0089
|
||||
# (#3301). Lookups by sha256 use the constraint's index.
|
||||
sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False)
|
||||
mime: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
@@ -47,7 +77,8 @@ class ImageRecord(Base):
|
||||
# Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'.
|
||||
# Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'.
|
||||
integrity_status: Mapped[str] = mapped_column(
|
||||
String(24), nullable=False, default="unknown", index=True
|
||||
String(24), nullable=False, default="unknown", index=True,
|
||||
server_default="unknown",
|
||||
)
|
||||
|
||||
# Thumbnail (populated by FC-2)
|
||||
@@ -72,8 +103,15 @@ class ImageRecord(Base):
|
||||
)
|
||||
# FC-2d-vii-c: canonical per-image artist (the single source of truth
|
||||
# for attribution; provenance posts remain lineage detail).
|
||||
# FK named explicitly: the naming convention renders this
|
||||
# `fk_image_record_artist_id_artist`, but alembic 0008 created it as
|
||||
# `fk_image_record_artist_id` (#3275).
|
||||
artist_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
ForeignKey(
|
||||
"artist.id", ondelete="SET NULL", name="fk_image_record_artist_id"
|
||||
),
|
||||
nullable=True,
|
||||
index=True,
|
||||
)
|
||||
|
||||
# ML fields (populated by the ml-worker / GPU agent). 1152 = SigLIP-so400m
|
||||
|
||||
@@ -21,17 +21,17 @@ class ImportBatch(Base):
|
||||
)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
|
||||
total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
# Deep-scan only: count of already-imported files whose sidecar metadata
|
||||
# got re-applied this run (post/source/provenance upsert). Stays 0 on
|
||||
# quick-scan batches. See `Importer.import_one(deep_scan=True)`.
|
||||
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True, server_default="running")
|
||||
# running | complete | cancelled
|
||||
|
||||
tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan")
|
||||
|
||||
@@ -4,7 +4,15 @@ Enforced as a single row via a CHECK (id = 1) constraint. The application
|
||||
always SELECTs id=1 and never inserts/deletes after the initial migration.
|
||||
"""
|
||||
|
||||
from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text, select
|
||||
from sqlalchemy import (
|
||||
Boolean,
|
||||
CheckConstraint,
|
||||
Float,
|
||||
Integer,
|
||||
Text,
|
||||
select,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -14,63 +22,79 @@ class ImportSettings(Base):
|
||||
__tablename__ = "import_settings"
|
||||
# Bare constraint name — Base.metadata's naming convention applies the
|
||||
# ck_<table>_<name> prefix, producing the final ck_import_settings_singleton.
|
||||
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
|
||||
# producing ck_import_settings_singleton. The chain shipped the DOUBLED
|
||||
# ck_import_settings_ck_import_settings_singleton, because the migration
|
||||
# pre-prefixed the name and the convention prefixed it again; alembic
|
||||
# 0088 renames it to what this line has always produced (#3275).
|
||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import")
|
||||
import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import", server_default="/import")
|
||||
|
||||
min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
|
||||
skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9)
|
||||
skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
|
||||
|
||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95)
|
||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30)
|
||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
|
||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
|
||||
|
||||
phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10)
|
||||
phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10, server_default="10")
|
||||
|
||||
# FC-3c downloader knobs
|
||||
download_rate_limit_seconds: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=3.0
|
||||
Float, nullable=False, default=3.0,
|
||||
server_default="3",
|
||||
)
|
||||
download_validate_files: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
|
||||
# FC-3d scheduling knobs
|
||||
download_schedule_default_seconds: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=28800
|
||||
Integer, nullable=False, default=28800,
|
||||
server_default="28800",
|
||||
)
|
||||
download_event_retention_days: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=90
|
||||
Integer, nullable=False, default=90,
|
||||
server_default="90",
|
||||
)
|
||||
download_failure_warning_threshold: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=5
|
||||
Integer, nullable=False, default=5,
|
||||
server_default="5",
|
||||
)
|
||||
|
||||
# FC-3h backup knobs.
|
||||
backup_db_nightly_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False,
|
||||
server_default="false",
|
||||
)
|
||||
backup_db_nightly_hour_utc: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=3,
|
||||
server_default="3",
|
||||
)
|
||||
backup_db_keep_last_n: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=14,
|
||||
server_default="14",
|
||||
)
|
||||
backup_images_keep_last_n: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=3,
|
||||
server_default="3",
|
||||
)
|
||||
|
||||
# FC-6.3 series continuation matcher. enabled gates the rescan; threshold is
|
||||
# the weighted-score cut-off (0..1) above which a pending suggestion is made.
|
||||
series_suggest_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
series_suggest_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.5,
|
||||
server_default="0.5",
|
||||
)
|
||||
|
||||
# #830 off-platform file-host downloads — per-host enable lever (default on,
|
||||
@@ -113,7 +137,9 @@ class ImportSettings(Base):
|
||||
# English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays
|
||||
# trusted regardless (script-detected). Per-post overrides handle the misses.
|
||||
translation_min_confidence: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.9, server_default="0.9",
|
||||
# text() because alembic 0084 used sa.text(); see ml_settings for why
|
||||
# the form matters and why it is per-column (#3275).
|
||||
Float, nullable=False, default=0.9, server_default=text("0.9"),
|
||||
)
|
||||
|
||||
# Title-based WIP auto-tagging (task #1458). When a freshly-imported post's
|
||||
|
||||
@@ -13,10 +13,12 @@ from sqlalchemy import (
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
@@ -26,6 +28,12 @@ from .base import Base
|
||||
class ImportTask(Base):
|
||||
__tablename__ = "import_task"
|
||||
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_import_task_created_at_desc", text("created_at DESC")),
|
||||
# Unindexed FK (#3300).
|
||||
Index("ix_import_task_result_image_id", "result_image_id"),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
batch_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
@@ -33,14 +41,14 @@ class ImportTask(Base):
|
||||
|
||||
source_path: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True)
|
||||
status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True, server_default="pending")
|
||||
|
||||
# Poison-pill circuit breaker (alembic 0026). recovery_count tracks
|
||||
# how many times the stuck-task sweep has re-queued this row; after
|
||||
# the cap it's failed with a diagnostic instead of looping. refetched
|
||||
# bounds the one-shot re-download remediation to a single attempt.
|
||||
recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
|
||||
result_image_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True
|
||||
|
||||
@@ -8,7 +8,7 @@ reads it and routes through cleanup_service.delete_images.
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text, func
|
||||
from sqlalchemy import DateTime, Integer, String, Text, func, text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
@@ -23,6 +23,7 @@ class LibraryAuditRun(Base):
|
||||
params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
server_default="running",
|
||||
)
|
||||
# running | ready | applied | cancelled | error
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
@@ -31,14 +32,16 @@ class LibraryAuditRun(Base):
|
||||
finished_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list)
|
||||
scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
matched_ids: Mapped[list[int]] = mapped_column(
|
||||
JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb")
|
||||
)
|
||||
error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
# Chunked-scan state (alembic 0039): keyset cursor the next chunk resumes
|
||||
# from, and the last time a chunk made progress (so the recovery sweep can
|
||||
# tell a progressing multi-chunk audit from a stuck one).
|
||||
resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
last_progress_at: Mapped[datetime | None] = mapped_column(
|
||||
DateTime(timezone=True), nullable=True,
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ from sqlalchemy import (
|
||||
String,
|
||||
func,
|
||||
select,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
@@ -20,7 +21,10 @@ from .base import Base
|
||||
class MLSettings(Base):
|
||||
__tablename__ = "ml_settings"
|
||||
# Bare name — Base.metadata's naming convention prepends ck_<table>_,
|
||||
# producing the final ck_ml_settings_singleton (matches migration 0003).
|
||||
# producing ck_ml_settings_singleton. The chain shipped the DOUBLED
|
||||
# ck_ml_settings_ck_ml_settings_singleton, because the migration
|
||||
# pre-prefixed the name and the convention prefixed it again; alembic
|
||||
# 0088 renames it to what this line has always produced (#3275).
|
||||
__table_args__ = (CheckConstraint("id = 1", name="singleton"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
@@ -31,17 +35,20 @@ class MLSettings(Base):
|
||||
# queueing embed work nothing will consume (the daily GPU 'embed' backfill
|
||||
# covers those images instead).
|
||||
cpu_embed_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
# Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not
|
||||
# a fixed count) so coverage reflects real screen time regardless of length;
|
||||
# cap the total so a long video can't explode into hundreds of embeds. The
|
||||
# per-frame SigLIP embeddings are mean-pooled. Operator-tunable.
|
||||
video_frame_interval_seconds: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=4.0
|
||||
Float, nullable=False, default=4.0,
|
||||
server_default="4",
|
||||
)
|
||||
video_max_frames: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=64
|
||||
Integer, nullable=False, default=64,
|
||||
server_default="64",
|
||||
)
|
||||
# Tagging-v2 head training (#114). The head is the suggestion source that
|
||||
# LEARNS from the operator's tags (replacing Camie + centroid). A concept
|
||||
@@ -49,10 +56,12 @@ class MLSettings(Base):
|
||||
# head_auto_apply_precision is the precision bar a head must clear (at some
|
||||
# operating point) to "graduate" into earned auto-apply. Operator-tunable.
|
||||
head_min_positives: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=8
|
||||
Integer, nullable=False, default=8,
|
||||
server_default="8",
|
||||
)
|
||||
head_auto_apply_precision: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.97
|
||||
Float, nullable=False, default=0.97,
|
||||
server_default="0.97",
|
||||
)
|
||||
# Earned auto-apply (#114). A graduated head fires (tags images without a
|
||||
# human) when this master switch is on AND the head has at least
|
||||
@@ -61,29 +70,34 @@ class MLSettings(Base):
|
||||
# default (operator-asked 2026-06-29: opt-OUT, not opt-in); the support +
|
||||
# measured-precision gates keep it safe, and every auto-tag is reversible.
|
||||
head_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
head_auto_apply_min_positives: Mapped[int] = mapped_column(
|
||||
# Support floor raised 30→50 (operator-asked 2026-07-06): a head needs
|
||||
# more human labels before it may fire without a human.
|
||||
Integer, nullable=False, default=50
|
||||
Integer, nullable=False, default=50,
|
||||
server_default="30",
|
||||
)
|
||||
# CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75
|
||||
# over-fired (high-reference characters matched a scatter of images); 0.85
|
||||
# keeps the confident single-character matches. Tunable from the agent card.
|
||||
ccip_match_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.85
|
||||
Float, nullable=False, default=0.85,
|
||||
server_default="0.85",
|
||||
)
|
||||
# CCIP auto-apply (#114). Confident matches (>= ccip_auto_apply_threshold,
|
||||
# above the suggest cut) auto-tag on a daily sweep. ON by default (opt-out);
|
||||
# single-character references + the high bar keep it safe, every tag reversible.
|
||||
ccip_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
ccip_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||
# Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident
|
||||
# character matches auto-tag.
|
||||
Float, nullable=False, default=0.95
|
||||
Float, nullable=False, default=0.95,
|
||||
server_default="0.92",
|
||||
)
|
||||
# -- Presentation chrome auto-hide (#141) -------------------------------
|
||||
# `banner` (chrome — clusters on UI, not content) auto-applies on the sweep
|
||||
@@ -95,13 +109,21 @@ class MLSettings(Base):
|
||||
# (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor
|
||||
# screenshot` are no longer chrome — they went to the PROCESS path below.
|
||||
presentation_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
presentation_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.90
|
||||
Float, nullable=False, default=0.90,
|
||||
# text(), not a string, because alembic 0082 used sa.text(): a bare
|
||||
# string renders DEFAULT '0.90'::double precision while text() renders
|
||||
# DEFAULT 0.90, and the chain is MIXED — some migrations used one,
|
||||
# some the other. Same value, different stored expression, so each
|
||||
# column here mirrors whichever form its own migration used (#3275).
|
||||
server_default=text("0.90"),
|
||||
)
|
||||
presentation_conflict_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
Float, nullable=False, default=0.50,
|
||||
server_default=text("0.50"),
|
||||
)
|
||||
# -- Process auto-apply (#1464) ----------------------------------------
|
||||
# `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program
|
||||
@@ -115,24 +137,29 @@ class MLSettings(Base):
|
||||
# (PresentationReview, mode='process') rather than silently marked. OFF by
|
||||
# default — a new whole-library auto-tagger is opt-in; every auto-tag reversible.
|
||||
process_auto_apply_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False
|
||||
Boolean, nullable=False, default=False,
|
||||
server_default="false",
|
||||
)
|
||||
process_auto_apply_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.90
|
||||
Float, nullable=False, default=0.90,
|
||||
server_default="0.90",
|
||||
)
|
||||
process_conflict_threshold: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.50
|
||||
Float, nullable=False, default=0.50,
|
||||
server_default="0.50",
|
||||
)
|
||||
# Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069);
|
||||
# existing libraries keep their stored value until the operator re-embeds.
|
||||
embedder_model_version: Mapped[str] = mapped_column(
|
||||
String(128), nullable=False, default="siglip2-so400m-patch16-512"
|
||||
String(128), nullable=False, default="siglip2-so400m-patch16-512",
|
||||
server_default="siglip2-so400m-patch16-512",
|
||||
)
|
||||
# The HF model NAME the embedder loads (server CPU embed + announced to the
|
||||
# GPU agent in the lease). Operator-settable so the embedder is a choice, not
|
||||
# a hardcode (#1190): set name + version together, then re-embed + retrain.
|
||||
embedder_model_name: Mapped[str] = mapped_column(
|
||||
String(128), nullable=False, default="google/siglip2-so400m-patch16-512"
|
||||
String(128), nullable=False, default="google/siglip2-so400m-patch16-512",
|
||||
server_default="google/siglip2-so400m-patch16-512",
|
||||
)
|
||||
# -- Crop proposers / detectors (#1202, #134) --------------------------
|
||||
# WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config
|
||||
@@ -145,20 +172,24 @@ class MLSettings(Base):
|
||||
# person: general COCO figure detector for Western/realistic art the anime
|
||||
# person-detector misses → NMS-merged with imgutils → CCIP + concept.
|
||||
detector_person_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
detector_person_weights: Mapped[str] = mapped_column(
|
||||
String(512), nullable=False, default="yolo11n.pt"
|
||||
String(512), nullable=False, default="yolo11n.pt",
|
||||
server_default="yolo11n.pt",
|
||||
)
|
||||
detector_person_conf: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.35
|
||||
Float, nullable=False, default=0.35,
|
||||
server_default=text("0.35"),
|
||||
)
|
||||
# anatomy: booru_yolo anime/furry/NSFW torso components → concept crops.
|
||||
# Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the
|
||||
# upstream repo so the URL resolves. License UNSTATED — fine for a private
|
||||
# homelab (operator accepted #1202).
|
||||
detector_anatomy_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
detector_anatomy_weights: Mapped[str] = mapped_column(
|
||||
String(512), nullable=False,
|
||||
@@ -166,37 +197,47 @@ class MLSettings(Base):
|
||||
"https://github.com/aperveyev/booru_yolo/raw/main/models/"
|
||||
"yolov11m_aa22.pt"
|
||||
),
|
||||
server_default="https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt",
|
||||
)
|
||||
detector_anatomy_conf: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.30
|
||||
Float, nullable=False, default=0.30,
|
||||
server_default=text("0.30"),
|
||||
)
|
||||
# panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x).
|
||||
detector_panel_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True
|
||||
Boolean, nullable=False, default=True,
|
||||
server_default="true",
|
||||
)
|
||||
detector_panel_weights: Mapped[str] = mapped_column(
|
||||
String(512), nullable=False,
|
||||
default="mosesb/best-comic-panel-detection::best.pt",
|
||||
server_default="mosesb/best-comic-panel-detection::best.pt",
|
||||
)
|
||||
detector_panel_conf: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.30
|
||||
Float, nullable=False, default=0.30,
|
||||
server_default=text("0.30"),
|
||||
)
|
||||
# Per-frame caps bound the crop→embed explosion; max_regions is the hard
|
||||
# per-job backstop; dedupe_iou drops near-duplicate crops before the embed.
|
||||
detector_max_figures: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=8
|
||||
Integer, nullable=False, default=8,
|
||||
server_default="8",
|
||||
)
|
||||
detector_max_components: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=8
|
||||
Integer, nullable=False, default=8,
|
||||
server_default="8",
|
||||
)
|
||||
detector_max_panels: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=8
|
||||
Integer, nullable=False, default=8,
|
||||
server_default="8",
|
||||
)
|
||||
detector_max_regions: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=128
|
||||
Integer, nullable=False, default=128,
|
||||
server_default="128",
|
||||
)
|
||||
detector_dedupe_iou: Mapped[float] = mapped_column(
|
||||
Float, nullable=False, default=0.85
|
||||
Float, nullable=False, default=0.85,
|
||||
server_default=text("0.85"),
|
||||
)
|
||||
# -- CCIP character prototypes (#1317) ---------------------------------
|
||||
# The per-character reference set is precomputed + refreshed INCREMENTALLY
|
||||
@@ -208,7 +249,8 @@ class MLSettings(Base):
|
||||
String(128), nullable=True
|
||||
)
|
||||
ccip_prototype_cap: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=64
|
||||
Integer, nullable=False, default=64,
|
||||
server_default="64",
|
||||
)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
|
||||
@@ -35,7 +35,7 @@ class PatreonFailedMedia(Base):
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
|
||||
@@ -35,7 +35,7 @@ class PixivFailedMedia(Base):
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
|
||||
@@ -13,11 +13,13 @@ from sqlalchemy import (
|
||||
CheckConstraint,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
@@ -27,6 +29,10 @@ from .base import Base
|
||||
class Post(Base):
|
||||
__tablename__ = "post"
|
||||
__table_args__ = (
|
||||
# alembic 0030. The comment above described this index; nothing declared
|
||||
# it, so autogenerate proposed dropping it (#3275).
|
||||
Index("uq_post_artist_external_id_null_source", "artist_id", "external_post_id",
|
||||
unique=True, postgresql_where=text("source_id IS NULL")),
|
||||
# Source-bound dedup. Postgres treats NULL != NULL so rows
|
||||
# with source_id IS NULL aren't deduped by this constraint;
|
||||
# the partial unique index `uq_post_artist_external_id_null_source`
|
||||
@@ -35,7 +41,11 @@ class Post(Base):
|
||||
UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"),
|
||||
CheckConstraint(
|
||||
"translation_override IN ('auto', 'force', 'original')",
|
||||
name="ck_post_translation_override",
|
||||
# Bare name: Base.metadata's naming convention prepends
|
||||
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
||||
# alembic 0088, which renames the four constraints that shipped
|
||||
# that way (#3275).
|
||||
name="translation_override",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ are pruned by retention.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Float, ForeignKey, String, func
|
||||
from sqlalchemy import DateTime, Float, ForeignKey, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -20,6 +20,14 @@ from .base import Base
|
||||
class PresentationReview(Base):
|
||||
__tablename__ = "presentation_review"
|
||||
|
||||
|
||||
__table_args__ = (
|
||||
Index("ix_presentation_review_resolved_at", "resolved_at"),
|
||||
# Both FKs to tag were unindexed (#3300); tag_id CASCADEs, so a tag
|
||||
# delete had to scan this table to find its rows.
|
||||
Index("ix_presentation_review_tag_id", "tag_id"),
|
||||
Index("ix_presentation_review_conflict_tag_id", "conflict_tag_id"),
|
||||
)
|
||||
image_record_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
|
||||
@@ -16,7 +16,14 @@ title is the optional chapter name; stated_part is the optional operator-facing
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer, Text, func
|
||||
from sqlalchemy import (
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -25,14 +32,26 @@ from .base import Base
|
||||
class SeriesChapter(Base):
|
||||
__tablename__ = "series_chapter"
|
||||
|
||||
__table_args__ = (
|
||||
# alembic 0047 named the UNIQUE `uq_series_chapter_anchor_page`, not
|
||||
# the `uq_series_chapter_anchor_page_id` a bare `unique=True` would
|
||||
# render (#3275).
|
||||
UniqueConstraint("anchor_page_id", name="uq_series_chapter_anchor_page"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
series_tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
# Both the UNIQUE (above) and the FK carry the names 0047 gave them; the
|
||||
# convention would render the FK `fk_series_chapter_anchor_page_id_series_page`.
|
||||
anchor_page_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("series_page.id", ondelete="CASCADE"),
|
||||
ForeignKey(
|
||||
"series_page.id",
|
||||
ondelete="CASCADE",
|
||||
name="fk_series_chapter_anchor_page",
|
||||
),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
title: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
stated_part: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
|
||||
@@ -14,7 +14,14 @@ number parsed from the source post, nullable when unknown.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, Integer, String, func
|
||||
from sqlalchemy import (
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -23,14 +30,22 @@ from .base import Base
|
||||
class SeriesPage(Base):
|
||||
__tablename__ = "series_page"
|
||||
|
||||
__table_args__ = (
|
||||
# alembic 0005 named this `uq_series_page_image`; a bare `unique=True`
|
||||
# on the column renders `uq_series_page_image_id` under the naming
|
||||
# convention, which is a different object from the one the database
|
||||
# has (#3275).
|
||||
UniqueConstraint("image_id", name="uq_series_page_image"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
series_tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
# UNIQUE lives in __table_args__ above, under the name 0005 gave it.
|
||||
image_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
unique=True,
|
||||
)
|
||||
# 'placed' = in the series-global run (page_number set); 'pending' = staged
|
||||
# from a post awaiting the operator's sort (page_number NULL). (#789 P2)
|
||||
|
||||
@@ -5,7 +5,16 @@ Multiple sources per artist support creators with cross-platform presence.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Boolean,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from .base import Base
|
||||
@@ -14,13 +23,27 @@ from .base import Base
|
||||
class Source(Base):
|
||||
__tablename__ = "source"
|
||||
|
||||
__table_args__ = (
|
||||
# alembic 0010. One row per (artist, platform, url): re-adding a source
|
||||
# the artist already has is an update, not a second row. The model had
|
||||
# never declared it (#3275), so autogenerate would have proposed
|
||||
# DROPPING it — the guarantee existed only in the migration chain.
|
||||
#
|
||||
# Named explicitly because the naming convention would render this
|
||||
# `uq_source_artist_id` (uq keys off column_0_name), which is both
|
||||
# wrong about the shape and not what the database actually has.
|
||||
UniqueConstraint(
|
||||
"artist_id", "platform", "url", name="uq_source_artist_platform_url"
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
artist_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
platform: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
url: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||
|
||||
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
|
||||
@@ -32,7 +55,7 @@ class Source(Base):
|
||||
# by _update_source_health alongside last_error; cleared on 'ok'.
|
||||
error_type: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True)
|
||||
check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0")
|
||||
|
||||
# alembic 0031: sticky deep-scan budget. When > 0, the next N download
|
||||
# runs use gallery-dl's full-walk config (skip: True + 1800s timeout);
|
||||
|
||||
@@ -34,7 +34,7 @@ class SubscribeStarFailedMedia(Base):
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
|
||||
@@ -15,11 +15,13 @@ from sqlalchemy import (
|
||||
Column,
|
||||
DateTime,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Table,
|
||||
false,
|
||||
func,
|
||||
text,
|
||||
)
|
||||
from sqlalchemy import (
|
||||
Enum as SQLEnum,
|
||||
@@ -67,17 +69,31 @@ image_tag = Table(
|
||||
primary_key=True,
|
||||
),
|
||||
Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True),
|
||||
Column("source", String(32), nullable=False, default="manual"),
|
||||
Column("source", String(32), nullable=False, default="manual", server_default="manual"),
|
||||
Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()),
|
||||
# The PK is (image_record_id, tag_id), which leads with the WRONG column
|
||||
# for the two things that matter most here (#3300): the gallery's tag
|
||||
# filter (tag_query.py builds `image_tag.c.tag_id == tid`) and the
|
||||
# ON DELETE CASCADE from tag, which has to find a tag's rows to remove
|
||||
# them. Without this index both scan the largest table in the schema.
|
||||
Index("ix_image_tag_tag_id", "tag_id"),
|
||||
)
|
||||
|
||||
|
||||
class Tag(Base):
|
||||
__tablename__ = "tag"
|
||||
__table_args__ = (
|
||||
# alembic 0002. An EXPRESSION index — COALESCE cannot be expressed as a
|
||||
# UniqueConstraint, which is why it only ever existed in a migration (#3275).
|
||||
Index("uq_tag_name_kind_fandom", "name", "kind", text("COALESCE(fandom_id, 0)"),
|
||||
unique=True),
|
||||
CheckConstraint(
|
||||
"(fandom_id IS NULL) OR (kind = 'character')",
|
||||
name="ck_tag_fandom_requires_character",
|
||||
# Bare name: Base.metadata's naming convention prepends
|
||||
# ck_<table>_. Pre-prefixing it here doubles the prefix — see
|
||||
# alembic 0088, which renames the four constraints that shipped
|
||||
# that way (#3275).
|
||||
name="fandom_requires_character",
|
||||
),
|
||||
)
|
||||
|
||||
@@ -87,6 +103,7 @@ class Tag(Base):
|
||||
SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]),
|
||||
nullable=False,
|
||||
default=TagKind.general,
|
||||
server_default="general",
|
||||
)
|
||||
fandom_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True
|
||||
|
||||
@@ -5,7 +5,7 @@ in image_prediction stay unmolested.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, String, func
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -14,10 +14,17 @@ from .base import Base
|
||||
class TagAlias(Base):
|
||||
__tablename__ = "tag_alias"
|
||||
|
||||
|
||||
__table_args__ = (
|
||||
# Named explicitly: the database calls this ix_tag_alias_canonical, while
|
||||
# a bare index=True on the column would generate ix_tag_alias_canonical_tag_id
|
||||
# and silently propose a drop+create on the next autogenerate (#3275).
|
||||
Index("ix_tag_alias_canonical", "canonical_tag_id"),
|
||||
)
|
||||
alias_string: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
alias_category: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
canonical_tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
|
||||
@@ -5,7 +5,7 @@ Prevents re-suggestion AND prevents allowlist auto-apply on that image.
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, ForeignKey, func
|
||||
from sqlalchemy import DateTime, ForeignKey, Index, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -14,11 +14,24 @@ from .base import Base
|
||||
class TagSuggestionRejection(Base):
|
||||
__tablename__ = "tag_suggestion_rejection"
|
||||
|
||||
|
||||
__table_args__ = (
|
||||
# Named explicitly; see tag_alias for why (#3275).
|
||||
Index("ix_tag_suggestion_rejection_tag", "tag_id"),
|
||||
)
|
||||
# Both FKs named explicitly. alembic 0003 used a hand-shortened `tsr`
|
||||
# prefix; the convention would render the full table name (#3275).
|
||||
image_record_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True
|
||||
ForeignKey(
|
||||
"image_record.id",
|
||||
ondelete="CASCADE",
|
||||
name="fk_tsr_image_record_id_image_record",
|
||||
),
|
||||
primary_key=True,
|
||||
)
|
||||
tag_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True
|
||||
ForeignKey("tag.id", ondelete="CASCADE", name="fk_tsr_tag_id_tag"),
|
||||
primary_key=True,
|
||||
)
|
||||
rejected_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
|
||||
@@ -15,7 +15,7 @@ backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min).
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import DateTime, Integer, String, Text
|
||||
from sqlalchemy import DateTime, Index, Integer, String, Text, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .base import Base
|
||||
@@ -24,12 +24,21 @@ from .base import Base
|
||||
class TaskRun(Base):
|
||||
__tablename__ = "task_run"
|
||||
|
||||
|
||||
__table_args__ = (
|
||||
# alembic 0016: the three task-history indexes (#3275).
|
||||
Index("ix_task_run_name_started", "task_name", text("started_at DESC")),
|
||||
Index("ix_task_run_queue_started", "queue", text("started_at DESC")),
|
||||
Index("ix_task_run_status_started", "status", text("started_at DESC")),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
celery_task_id: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, index=True,
|
||||
)
|
||||
queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True)
|
||||
task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True)
|
||||
# Neither carries index=True: ix_task_run_queue_started and
|
||||
# ix_task_run_name_started already lead with these columns (#3301).
|
||||
queue: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
task_name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
target_id: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
started_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, index=True,
|
||||
@@ -39,7 +48,9 @@ class TaskRun(Base):
|
||||
)
|
||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(16), nullable=False, default="running", index=True,
|
||||
# No index=True — ix_task_run_status_started leads with `status`.
|
||||
String(16), nullable=False, default="running",
|
||||
server_default="running",
|
||||
)
|
||||
error_type: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
@@ -133,6 +133,82 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
||||
already ran, so both paths now share one proven route. The cost: `:c-<sha>`
|
||||
is an index rather than a plain image, so `fc.revision` does not resolve
|
||||
through it — nothing reads it there, and the index names the same manifest.
|
||||
- **The image builds run on a `docker-container` buildx builder, and
|
||||
`provenance`/`sbom` are explicitly OFF** (milestone 326 step 1). The builder
|
||||
is what makes a registry layer cache possible at all — the default `docker`
|
||||
driver cannot export one (#3114) — and it is #3190's leading suspect, since
|
||||
it is the driver that resolves image metadata against a local store a
|
||||
registry-direct push never fills. **The attestation flags are load-bearing,
|
||||
not tidiness:** on the container driver `build-push-action@v5` defaults
|
||||
`provenance` to true when pushing, an attestation manifest makes the pushed
|
||||
tag a manifest INDEX, and config labels do not resolve through an index — so
|
||||
leaving them on would make every push read `fc.revision=<none>`, miss, and
|
||||
rebuild forever with every lane green. Same failure as #3183, different door.
|
||||
These jobs run inside a container against a mounted docker socket, so the
|
||||
buildkit container is a sibling rather than a child.
|
||||
- **All three images import and export a registry layer cache**
|
||||
(`<image>:buildcache`, `mode=max`). This is not an optimisation bolted onto
|
||||
the driver change — it is the other half of it. The `docker-container`
|
||||
driver gets a fresh buildkit instance per job and therefore has **no local
|
||||
layer store at all**, where the old `docker` driver at least reused whatever
|
||||
the runner's dockerd happened to hold. Measured on run 4896, the first builds
|
||||
after the driver moved: web 3m44s (was 2m23s), ml 3m49s (was 3m20s), agent
|
||||
11m12s (was 9m26s) — every one slower. A `:buildcache` tag is read by every
|
||||
build that runs, is one moving ref per image, holds cache blobs rather than a
|
||||
shippable artifact, and is overwritten in place, so it is not a return of the
|
||||
per-version tags milestone 318 withdrew (#3114).
|
||||
- **`build.yml` accepts a `workflow_dispatch` with `force_build`**, which
|
||||
bypasses the reuse check for all three images. It exists because
|
||||
skip-if-exists makes its own build path untestable: `agent/` has not changed
|
||||
since 2026-07-17, so the agent build has not run in six weeks and cannot be
|
||||
exercised on demand — and #3190 lives on exactly that path. Editing
|
||||
`build.yml` does not force a build either, deliberately: the workflow is not
|
||||
shipped bytes and is in no artifact's path set. The flag is read through
|
||||
`github.event.inputs` into an env var rather than interpolated into a run
|
||||
block, and it is checked inside the reuse step so that one decision drives
|
||||
both the build and the repoint.
|
||||
- **A weekly `schedule` rebuilds all three images against fresh base layers**
|
||||
(Sunday 06:00 UTC, milestone 326 step 4, #3154). Skip-if-exists is keyed on
|
||||
OUR source, so an artifact whose source stops moving stops picking up base
|
||||
updates — `agent/` has not changed since 2026-07-17 and would otherwise serve
|
||||
that day's `nvidia/cuda` layers forever. Four things make it work:
|
||||
- It **builds `main`, not the branch that triggered it.** Forgejo registers a
|
||||
cron from the DEFAULT branch (`dev` here), so a scheduled run arrives with
|
||||
`github.ref` on dev. The ref is decided once in a top-level `env:
|
||||
BUILD_REF` that every checkout in the file takes, rather than per job —
|
||||
otherwise `sign-extension` would derive dev's extension version while
|
||||
`build-web` bundled main's, and the release download would 404 on a version
|
||||
that exists perfectly well. Every job then ASSERTS its checkout is `main`
|
||||
before doing anything, because `env` inside `with:` is not a context this
|
||||
runner is known to evaluate — if it silently resolved to empty, checkout
|
||||
would fall back to the triggering ref and the refresh would publish dev's
|
||||
source to `:latest` with every lane green.
|
||||
- It **publishes only `:latest`.** `:c-<sha>` for main's HEAD already names
|
||||
the bytes that commit built; re-pushing it over refreshed layers would
|
||||
break the one tag rule 145 makes immutable, and it is the rollback unit.
|
||||
The repoint step needs no schedule case for this — the tag list is the
|
||||
channel tag alone, so SOURCE is the only entry, it is excluded as always,
|
||||
and the step correctly does nothing.
|
||||
- **`:latest` and `:c-<sha>` therefore diverge between a refresh and the next
|
||||
`main` push, by design.** They re-converge on that push: it hits reuse (a
|
||||
refresh does not move `fc.revision`, because it does not touch the source),
|
||||
and the repoint writes the NEW `:c-<sha>` from the refreshed `:latest`. The
|
||||
push path needed no change for this, because the repoint already excluded
|
||||
the source tag — the same rule that keeps the label readable also keeps a
|
||||
refresh from being undone.
|
||||
- **`pull: true` on the scheduled path only** is the mechanism: a moved base
|
||||
tag changes the `FROM` layer's cache key and everything above it rebuilds.
|
||||
**It does not currently make the unmoved case free.** Measured on the first
|
||||
real fire (run 4934, 2026-08-30): every content step reported `CACHED` and
|
||||
the bases resolved to unchanged digests, yet all three `:latest` tags got a
|
||||
NEW manifest digest, because buildkit mints a fresh image config per run and
|
||||
republishes identical layers under it. So `:latest` is rewritten weekly
|
||||
whether or not anything changed, and `:c-<sha>` is handed a new manifest to
|
||||
diverge from on the same cadence — a digest change stops meaning anything.
|
||||
Tracked as #3265; the likely fix is a deterministic `SOURCE_DATE_EPOCH`.
|
||||
Separately not caught: a Debian package update inside the `apt-get install`
|
||||
layer while the base tag stands still — a lag rather than a hole, since the
|
||||
official python/cuda images rebuild with those updates baked in.
|
||||
- **`FC_CHANNEL` and `FC_VERSION` are build args, not runtime settings.**
|
||||
`build.yml` passes them to the web image only — the ml and agent images have
|
||||
nothing to report them to. `/api/health` returns both, the foot of Settings
|
||||
|
||||
+23
-5
@@ -74,7 +74,25 @@ services:
|
||||
retries: 5
|
||||
|
||||
web:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
# :latest, NOT :dev — this file IS the install path.
|
||||
#
|
||||
# `docker compose up -d` merges docker-compose.override.yml, which sets
|
||||
# build: for all five app services, and a build: wins over image:. So a
|
||||
# contributor never pulls this tag and is unaffected by what it says.
|
||||
#
|
||||
# The tag is consulted only on `docker compose -f docker-compose.yml up -d`
|
||||
# — the documented production path, which skips the override. That is a
|
||||
# stranger installing the product, and they must land on the stable channel.
|
||||
#
|
||||
# :latest is main, which IS production (rule 147). :dev is the rolling
|
||||
# bleeding-edge channel we work out of, republished several times a day with
|
||||
# no stability promise. This file pinned :dev on all five services until
|
||||
# 2026-08-31 (#3270), so the documented install shipped development builds.
|
||||
# It went unnoticed because nobody who works on the project takes this path:
|
||||
# the operator deploys from a swarm stack file, contributors get the
|
||||
# override. Do not "fix" this back to :dev while debugging — use the
|
||||
# override, or -f with an explicit tag on the command line.
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||
command: ["web"]
|
||||
# Graceful shutdown: give the container time to drain in-flight work on a
|
||||
# deploy (docker SIGTERMs, then SIGKILLs after this window — default is only
|
||||
@@ -122,7 +140,7 @@ services:
|
||||
redis: { condition: service_healthy }
|
||||
|
||||
worker:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||
command: ["worker"]
|
||||
# Drain in-flight import/thumbnail/download tasks before SIGKILL on deploy.
|
||||
stop_grace_period: 90s
|
||||
@@ -142,7 +160,7 @@ services:
|
||||
redis: { condition: service_healthy }
|
||||
|
||||
scheduler:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||
command: ["scheduler"]
|
||||
# Quick maintenance/scan lane + beat — short tasks, modest drain window.
|
||||
stop_grace_period: 60s
|
||||
@@ -163,7 +181,7 @@ services:
|
||||
# 30-min backup or a multi-chunk audit can never starve the 5-min recovery
|
||||
# sweeps / vacuum (operator-flagged 2026-06-07). One slot — these are heavy.
|
||||
maintenance-long:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:dev
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator:latest
|
||||
command: ["worker"]
|
||||
# Longest lane (DB backups, library audits, translation backfill) — give it
|
||||
# the most room to finish a chunk gracefully. Chunked + idempotent, so a job
|
||||
@@ -184,7 +202,7 @@ services:
|
||||
redis: { condition: service_healthy }
|
||||
|
||||
ml-worker:
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator-ml:dev
|
||||
image: git.fabledsword.com/bvandeusen/fabledcurator-ml:latest
|
||||
command: ["ml-worker"]
|
||||
# A single GPU inference pass can run tens of seconds — let it finish.
|
||||
stop_grace_period: 120s
|
||||
|
||||
@@ -34,6 +34,15 @@ is genuinely the commit those paths last changed in.
|
||||
emit. Two nearly-identical formats are more dangerous than two obviously
|
||||
different ones, and the only thing keeping them identical is a test.
|
||||
|
||||
**The extension is the one exception, and it is a rendering exception only.**
|
||||
AMO's version grammar forbids a leading zero, so the extension emits the same
|
||||
numbers unpadded — `2026.8.29.201` where the family says `2026.08.29.0201`
|
||||
(#3138, milestone 318 step 8). Rule 148 defines comparison as numeric per
|
||||
dot-segment, under which the two are equal, so this is pinned in both
|
||||
directions below: the extension must satisfy AMO's grammar, and every artifact
|
||||
must derive the same NUMBERS its own commit stamps. An exception left as "the
|
||||
extension is different" would drift into being differently different.
|
||||
|
||||
The identity-TAG tests this file used to hold are gone with the tag. There is
|
||||
no longer a `CHANNELLED` list to drift (the channel is which tag you inspect),
|
||||
and no `identity` subcommand to refuse an unqualified call.
|
||||
@@ -57,6 +66,26 @@ _REVISION = re.compile(r"^[0-9a-f]{12}$")
|
||||
# YYYY.MM.DD.HHMM, every segment zero-padded to its full width.
|
||||
_VERSION = re.compile(r"^\d{4}\.\d{2}\.\d{2}\.\d{4}$")
|
||||
|
||||
# The artifacts that cannot use the padded rendering. Exactly one, and the
|
||||
# reason is external: `packaging.sh` derives the extension's version and AMO
|
||||
# refuses to sign a padded one.
|
||||
AMO_UNPADDED = frozenset({"extension"})
|
||||
|
||||
# Mozilla's published grammar for addons.mozilla.org, transcribed from MDN's
|
||||
# manifest.json/version page. A segment is the single digit `0` or starts 1-9,
|
||||
# and there are at most four. This is the constraint the exception exists for,
|
||||
# so it is what the exception is tested against — `2026.08.29.0201` fails it.
|
||||
_AMO = re.compile(r"^(0|[1-9][0-9]{0,8})(\.(0|[1-9][0-9]{0,8})){0,3}$")
|
||||
|
||||
# YYYY.M.D.HHMM — four segments, none of them zero-padded.
|
||||
_UNPADDED = re.compile(r"^\d{4}(\.(0|[1-9]\d*)){3}$")
|
||||
|
||||
|
||||
def segments(value: str) -> tuple[int, ...]:
|
||||
"""A version as the numbers it denotes, which is how rule 148 says to
|
||||
compare one. `2026.08.29.0201` and `2026.8.29.201` are one value here."""
|
||||
return tuple(int(part) for part in value.split("."))
|
||||
|
||||
|
||||
# Everything here goes through artifacts.sh rather than importing a sibling
|
||||
# test module. That is the interface build.yml actually calls, so the tests
|
||||
@@ -126,7 +155,7 @@ def test_revision_is_a_legal_label_value_and_is_stable(artifact):
|
||||
assert first == revision(artifact), "revision is not stable across calls"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("artifact", ARTIFACTS)
|
||||
@pytest.mark.parametrize("artifact", sorted(set(ARTIFACTS) - AMO_UNPADDED))
|
||||
def test_version_is_zero_padded_calver(artifact):
|
||||
"""The family shape, pinned.
|
||||
|
||||
@@ -148,6 +177,31 @@ def test_version_is_zero_padded_calver(artifact):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("artifact", sorted(AMO_UNPADDED))
|
||||
def test_the_unpadded_artifacts_derive_something_amo_will_sign(artifact):
|
||||
"""The other half of the family shape: the documented exception, tested
|
||||
against the constraint that justifies it rather than against itself.
|
||||
|
||||
A padded value passes `_UNPADDED` on any date with no leading zeros, so
|
||||
that pattern alone would let a regression sit unnoticed until the first
|
||||
single-digit month — at which point the failure is a burned AMO version,
|
||||
not a red lane. AMO's grammar is the assertion that fires immediately.
|
||||
"""
|
||||
value = artifacts("version", artifact).strip()
|
||||
assert _AMO.match(value), (
|
||||
f"{artifact} derives {value!r}, which AMO refuses: a segment must be "
|
||||
f"the single digit `0` or start 1-9, and there are at most four. "
|
||||
f"Almost certainly a zero-padded segment — the family pads and this "
|
||||
f"artifact must not (#3138). AMO 409s on re-signing, so a version it "
|
||||
f"rejects is burned."
|
||||
)
|
||||
assert _UNPADDED.match(value), (
|
||||
f"{artifact} derives {value!r}, which is not YYYY.M.D.HHMM. AMO would "
|
||||
f"also accept the pre-318 `1.0.<minutes>`, and that orders below every "
|
||||
f"ext-2026.* release already signed."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("artifact", ARTIFACTS)
|
||||
def test_version_and_revision_describe_the_same_commit(artifact):
|
||||
"""They are derived independently and must not be able to disagree.
|
||||
@@ -162,5 +216,21 @@ def test_version_and_revision_describe_the_same_commit(artifact):
|
||||
capture_output=True, text=True, check=True, cwd=ROOT,
|
||||
env={"TZ": "UTC", "PATH": os.environ.get("PATH", "")},
|
||||
).stdout.strip()
|
||||
assert artifacts("version", artifact).strip() == stamped
|
||||
derived = artifacts("version", artifact).strip()
|
||||
|
||||
# Compared as NUMBERS, which is how rule 148 defines comparison and the
|
||||
# only way one assertion can cover both renderings. This is what makes the
|
||||
# extension's exception cosmetic rather than semantic: it must denote
|
||||
# exactly the value its own commit stamps, whatever the padding.
|
||||
assert segments(derived) == segments(stamped), (
|
||||
f"{artifact} derives {derived!r}, but its newest shipped commit "
|
||||
f"{sha[:12]} is {stamped!r}. The instance would name one commit while "
|
||||
f"carrying another's bytes."
|
||||
)
|
||||
if artifact not in AMO_UNPADDED:
|
||||
assert derived == stamped, (
|
||||
f"{artifact} derives {derived!r} where the family shape is "
|
||||
f"{stamped!r} — same numbers, wrong rendering. Only the artifacts "
|
||||
f"in AMO_UNPADDED may differ here."
|
||||
)
|
||||
assert sha.startswith(revision(artifact))
|
||||
|
||||
Reference in New Issue
Block a user