From 63e0a423d72fc292275a18e6e94185bd5271042c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 29 Aug 2026 22:53:48 -0400 Subject: [PATCH 01/17] ci: a weekly base-image refresh on the channel tags (milestone 326 step 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Skip-if-exists is keyed on our own source, so an artifact whose source stops moving stops picking up base-image updates. `agent/` last changed 2026-07-17; every push since has correctly declined to rebuild it, which also means it will serve that day's nvidia/cuda layers indefinitely. A `schedule:` trigger, Sunday 06:00 UTC, away from CI-runner's Monday security sweep so the two are never diagnosing each other. #3154's blocking open question is dissolved rather than answered. It was written when the identity was a `r-` TAG, and asked how the next ordinary push could avoid repointing :latest back off the refresh. Milestone 318 replaced that tag with a LABEL, and #3183 made the repoint step exclude its source tag so the label stays readable. Excluding the source is what also keeps a refresh from being undone: on the next main push the reuse check hits, :latest is not rewritten, and the new :c- is written FROM the refreshed :latest. To be verified by digest, not by this argument. Four decisions, each commented where it lives: * It builds `main`, not the branch that triggered it. Forgejo registers a cron from the default branch — `dev` here — so a scheduled run arrives with github.ref on dev, and a refresh of :dev would be refreshing the one channel that is rebuilt constantly anyway. The ref is decided once in a top-level `env: BUILD_REF` that all four checkouts take. Deriving it per job would let the halves disagree: sign-extension would derive dev's extension version while build-web bundled main's, and the release download would 404 on a version that exists perfectly well. * It publishes only the channel tag. :c- for main's HEAD already names the bytes that commit built; re-pushing it over refreshed layers would break the one tag rule 145 makes immutable, and it is the rollback unit — so the breakage would surface on the day somebody needed it. The repoint step needs no schedule case: the tag list is the channel tag alone, SOURCE is the only entry, it is excluded as always, and the step correctly does nothing. * It bypasses reuse by construction, since it rebuilds the same source and fc.revision always matches. Checked in the reuse step beside force_build, so one decision still drives both the build and the repoint. * `pull: true`, on the scheduled path only, is the actual mechanism. A moved base tag changes the FROM layer's cache key and everything above it rebuilds; an unmoved one is satisfied by the registry cache and the refresh is a ~13s no-op that republishes nothing. That no-op is the point — :latest should change when there is something new in it, not every Sunday. The known lag, left deliberately: an apt package update while the base tag stands still is not caught, and closing it needs no-cache: true, which buys weekly churn for it. --- .forgejo/workflows/build.yml | 187 +++++++++++++++++++++++++++++++++-- ci-requirements.md | 35 +++++++ 2 files changed, 216 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index eb4299b..3c6d666 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -45,6 +45,36 @@ on: type: boolean default: false + # The base-image refresh (milestone 326 step 4, #3154). + # + # Skip-if-exists is keyed on OUR source, so an artifact whose source stops + # moving stops picking up base-image updates. `agent/` last changed + # 2026-07-17; every push since has correctly declined to rebuild it, which + # also means it will serve that day's `nvidia/cuda` layers forever. Nothing + # is wrong until it has been unchanged for months, which is precisely why + # this is a calendar trigger and not a condition on the push path. + # + # Weekly, Sunday 06:00 UTC. Away from CI-runner's Monday security sweep so + # the two are never diagnosing each other, and on the quietest day so a + # surprise rebuild is not competing with a push. + schedule: + - cron: '0 6 * * 0' + +# Which branch a run BUILDS, as opposed to which one triggered it. +# +# They are the same thing on every trigger but `schedule`. Forgejo registers a +# cron from the DEFAULT branch — `dev` here — so a scheduled run arrives with +# `github.ref` pointing at dev, and a refresh that rebuilt `:dev` would be +# refreshing the one channel that gets rebuilt constantly anyway. Production is +# `main` (rule 147), and `:latest` is the tag that goes stale. +# +# So the ref is decided once, here, and every checkout in the file takes it. +# Deriving it per job invites the two halves to disagree: sign-extension would +# derive dev's extension version while build-web bundled main's, and the +# release download would 404 on a version that exists perfectly well. +env: + BUILD_REF: ${{ github.event_name == 'schedule' && 'main' || github.ref }} + # Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes: # - write:package, read:package (for docker push to git.fabledsword.com) # - write:release (for ext- release asset cache) @@ -88,6 +118,10 @@ jobs: steps: - uses: actions/checkout@v4 with: + # Not the triggering ref — see the `env:` block at the top. On a + # scheduled refresh this is `main`; on everything else it is the ref + # that fired, so this is a no-op on every ordinary path. + ref: ${{ env.BUILD_REF }} # Full history is load-bearing, not a convenience: the version this # job signs is derived from the commit TIME of the newest packaged # extension change. A depth-1 clone sees one commit and derives a @@ -364,6 +398,10 @@ jobs: steps: - uses: actions/checkout@v4 with: + # Not the triggering ref — see the `env:` block at the top. On a + # scheduled refresh this is `main`; on everything else it is the ref + # that fired, so this is a no-op on every ordinary path. + ref: ${{ env.BUILD_REF }} # Full history: this job RE-DERIVES the extension version rather than # being handed it, and a depth-1 clone derives a wrong, too-low value # rather than failing — which would 404 the download of a release @@ -429,8 +467,30 @@ jobs: # everywhere). Operator-flagged 2026-06-01 after the first :c- # main-push build failed at this step. SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # Mirrors build-web's tag list; see the comment there. - if [ "${GITHUB_REF##*/}" = "main" ]; then + + # A scheduled refresh publishes the CHANNEL and nothing else + # (#3154). :c- for main's HEAD already exists and names the + # bytes that commit actually built; re-pushing it over refreshed + # base layers would break the one tag rule 145 makes immutable — + # and it is the rollback unit, so the breakage would surface on the + # day somebody needed it. + # + # The accepted consequence: between a refresh and the next main + # push, :latest and :c- point at different manifests. That is + # the design, not drift. They RE-CONVERGE on that push — it hits + # reuse (a refresh does not move fc.revision, because it does not + # touch the source), and the repoint step then writes the new + # :c- from the refreshed :latest. So the rollback unit ends up + # naming the bytes production is actually running, which is the + # property that matters. + # + # Checked BEFORE the ref test, not after: a scheduled run's + # GITHUB_REF is the default branch (dev), so the main test would + # never fire on it. + if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:latest" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" + elif [ "${GITHUB_REF##*/}" = "main" ]; then echo "tags=git.fabledsword.com/bvandeusen/fabledcurator:latest,git.fabledsword.com/bvandeusen/fabledcurator:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "channel=main" >> "$GITHUB_OUTPUT" else @@ -524,6 +584,10 @@ jobs: # this runner is known to evaluate. Read through env rather than # interpolated into the run block, same rule as release.yml's TAG. FORCE: ${{ github.event.inputs.force_build }} + # A scheduled refresh has to bypass reuse by construction: it + # rebuilds the SAME source, so fc.revision always matches and the + # check would skip every refresh there has ever been. + EVENT: ${{ github.event_name }} run: | set -eu DERIVED=$(sh scripts/artifacts.sh revision web) @@ -570,6 +634,9 @@ jobs: if [ "${FORCE:-false}" = "true" ]; then echo "hit=false" >> "$GITHUB_OUTPUT" echo "reuse: force_build set — building regardless" + elif [ "${EVENT:-}" = "schedule" ]; then + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: scheduled base refresh — building regardless" elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" echo "reuse: already published — skipping the build" @@ -661,6 +728,30 @@ jobs: context: . file: Dockerfile push: true + # Re-resolve the FROM references against the registry instead of + # trusting whatever digest the cache was built against. This is the + # whole mechanism of the scheduled refresh (#3154): if the base tag + # moved, the FROM layer's cache key changes, every layer above it + # invalidates, and the image genuinely rebuilds. If it did not move, + # the registry cache satisfies the entire graph and the refresh is a + # ~13s no-op that republishes nothing. + # + # That no-op is the POINT, not a shortfall: :latest should change + # when there is something new in it and not otherwise. A refresh + # that rewrote the image weekly regardless would churn the registry + # and hand :c- a new manifest to diverge from every Sunday, for + # no gain. + # + # What it therefore does NOT catch: a Debian package update inside + # the `apt-get install` layer while the base tag itself stands + # still. The official python/cuda images rebuild with those updates + # baked in, so this is a lag rather than a hole — but closing it + # would take `no-cache: true` on the scheduled path, which is the + # weekly-churn trade above. Left as the cheaper of the two on + # purpose. + # + # Only on the schedule. An ordinary push wants the cached base. + pull: ${{ github.event_name == 'schedule' }} # ONE tag, the channel's. Every other tag is written by the step # below, registry-side. buildx here pushes the first tag to the # registry and then re-pushes the rest through the DOCKER driver, @@ -783,6 +874,12 @@ jobs: ARGS="$ARGS -t $t" done unset IFS + # + # This is also the whole of the scheduled refresh's tag handling + # (#3154): a refresh's tag list is the channel tag alone, so SOURCE + # is the only entry, it gets excluded, and this step correctly does + # nothing. No `if:` on the step and no schedule special-case — + # excluding the source was already the right rule. if [ -z "$ARGS" ]; then echo "repoint: $SOURCE is the only tag for this channel and" echo "repoint: already holds this revision — nothing to write." @@ -799,6 +896,10 @@ jobs: steps: - uses: actions/checkout@v4 with: + # Not the triggering ref — see the `env:` block at the top. On a + # scheduled refresh this is `main`; on everything else it is the ref + # that fired, so this is a no-op on every ordinary path. + ref: ${{ env.BUILD_REF }} # Full history: this job derives its artifact's version from the # commit its shipped files last changed in (milestone 313). A # depth-1 clone cannot see that commit — it either derives a wrong, @@ -843,8 +944,12 @@ jobs: # everywhere). Operator-flagged 2026-06-01 after first :c- # main-push build failed at this step. SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # Mirrors build-web's tag list; see the comment there. - if [ "${GITHUB_REF##*/}" = "main" ]; then + # Mirrors build-web's tag list and its schedule handling; see + # the comments there. + if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" + elif [ "${GITHUB_REF##*/}" = "main" ]; then echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-ml:latest,git.fabledsword.com/bvandeusen/fabledcurator-ml:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "channel=main" >> "$GITHUB_OUTPUT" else @@ -921,6 +1026,10 @@ jobs: # this runner is known to evaluate. Read through env rather than # interpolated into the run block, same rule as release.yml's TAG. FORCE: ${{ github.event.inputs.force_build }} + # A scheduled refresh has to bypass reuse by construction: it + # rebuilds the SAME source, so fc.revision always matches and the + # check would skip every refresh there has ever been. + EVENT: ${{ github.event_name }} run: | set -eu DERIVED=$(sh scripts/artifacts.sh revision ml) @@ -963,6 +1072,9 @@ jobs: if [ "${FORCE:-false}" = "true" ]; then echo "hit=false" >> "$GITHUB_OUTPUT" echo "reuse: force_build set — building regardless" + elif [ "${EVENT:-}" = "schedule" ]; then + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: scheduled base refresh — building regardless" elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" echo "reuse: already published — skipping the build" @@ -978,6 +1090,30 @@ jobs: context: . file: Dockerfile.ml push: true + # Re-resolve the FROM references against the registry instead of + # trusting whatever digest the cache was built against. This is the + # whole mechanism of the scheduled refresh (#3154): if the base tag + # moved, the FROM layer's cache key changes, every layer above it + # invalidates, and the image genuinely rebuilds. If it did not move, + # the registry cache satisfies the entire graph and the refresh is a + # ~13s no-op that republishes nothing. + # + # That no-op is the POINT, not a shortfall: :latest should change + # when there is something new in it and not otherwise. A refresh + # that rewrote the image weekly regardless would churn the registry + # and hand :c- a new manifest to diverge from every Sunday, for + # no gain. + # + # What it therefore does NOT catch: a Debian package update inside + # the `apt-get install` layer while the base tag itself stands + # still. The official python/cuda images rebuild with those updates + # baked in, so this is a lag rather than a hole — but closing it + # would take `no-cache: true` on the scheduled path, which is the + # weekly-churn trade above. Left as the cheaper of the two on + # purpose. + # + # Only on the schedule. An ordinary push wants the cached base. + pull: ${{ github.event_name == 'schedule' }} # ONE tag, the channel's. Every other tag is written by the step # below, registry-side. buildx here pushes the first tag to the # registry and then re-pushes the rest through the DOCKER driver, @@ -1113,6 +1249,10 @@ jobs: steps: - uses: actions/checkout@v4 with: + # Not the triggering ref — see the `env:` block at the top. On a + # scheduled refresh this is `main`; on everything else it is the ref + # that fired, so this is a no-op on every ordinary path. + ref: ${{ env.BUILD_REF }} # Full history: this job derives its artifact's version from the # commit its shipped files last changed in (milestone 313). A # depth-1 clone cannot see that commit — it either derives a wrong, @@ -1152,8 +1292,12 @@ jobs: id: tag run: | SHORT_SHA=$(printf '%s' "$GITHUB_SHA" | cut -c1-7) - # Mirrors build-web's tag list; see the comment there. - if [ "${GITHUB_REF##*/}" = "main" ]; then + # Mirrors build-web's tag list and its schedule handling; see + # the comments there. + if [ "${GITHUB_EVENT_NAME:-}" = "schedule" ]; then + echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:latest" >> "$GITHUB_OUTPUT" + echo "channel=main" >> "$GITHUB_OUTPUT" + elif [ "${GITHUB_REF##*/}" = "main" ]; then echo "tags=git.fabledsword.com/bvandeusen/fabledcurator-agent:latest,git.fabledsword.com/bvandeusen/fabledcurator-agent:c-${SHORT_SHA}" >> "$GITHUB_OUTPUT" echo "channel=main" >> "$GITHUB_OUTPUT" else @@ -1230,6 +1374,10 @@ jobs: # this runner is known to evaluate. Read through env rather than # interpolated into the run block, same rule as release.yml's TAG. FORCE: ${{ github.event.inputs.force_build }} + # A scheduled refresh has to bypass reuse by construction: it + # rebuilds the SAME source, so fc.revision always matches and the + # check would skip every refresh there has ever been. + EVENT: ${{ github.event_name }} run: | set -eu DERIVED=$(sh scripts/artifacts.sh revision agent) @@ -1272,6 +1420,9 @@ jobs: if [ "${FORCE:-false}" = "true" ]; then echo "hit=false" >> "$GITHUB_OUTPUT" echo "reuse: force_build set — building regardless" + elif [ "${EVENT:-}" = "schedule" ]; then + echo "hit=false" >> "$GITHUB_OUTPUT" + echo "reuse: scheduled base refresh — building regardless" elif [ -n "$PUBLISHED" ] && [ "$PUBLISHED" = "$DERIVED" ]; then echo "hit=true" >> "$GITHUB_OUTPUT" echo "reuse: already published — skipping the build" @@ -1287,6 +1438,30 @@ jobs: context: agent file: agent/Dockerfile push: true + # Re-resolve the FROM references against the registry instead of + # trusting whatever digest the cache was built against. This is the + # whole mechanism of the scheduled refresh (#3154): if the base tag + # moved, the FROM layer's cache key changes, every layer above it + # invalidates, and the image genuinely rebuilds. If it did not move, + # the registry cache satisfies the entire graph and the refresh is a + # ~13s no-op that republishes nothing. + # + # That no-op is the POINT, not a shortfall: :latest should change + # when there is something new in it and not otherwise. A refresh + # that rewrote the image weekly regardless would churn the registry + # and hand :c- a new manifest to diverge from every Sunday, for + # no gain. + # + # What it therefore does NOT catch: a Debian package update inside + # the `apt-get install` layer while the base tag itself stands + # still. The official python/cuda images rebuild with those updates + # baked in, so this is a lag rather than a hole — but closing it + # would take `no-cache: true` on the scheduled path, which is the + # weekly-churn trade above. Left as the cheaper of the two on + # purpose. + # + # Only on the schedule. An ordinary push wants the cached base. + pull: ${{ github.event_name == 'schedule' }} # ONE tag, the channel's. Every other tag is written by the step # below, registry-side. buildx here pushes the first tag to the # registry and then re-pushes the rest through the DOCKER driver, diff --git a/ci-requirements.md b/ci-requirements.md index 340e277..330471c 100644 --- a/ci-requirements.md +++ b/ci-requirements.md @@ -167,6 +167,41 @@ per `docs/process.md`'s "add deps to the image when used by >1 project". `github.event.inputs` into an env var rather than interpolated into a run block, and it is checked inside the reuse step so that one decision drives both the build and the repoint. +- **A weekly `schedule` rebuilds all three images against fresh base layers** + (Sunday 06:00 UTC, milestone 326 step 4, #3154). Skip-if-exists is keyed on + OUR source, so an artifact whose source stops moving stops picking up base + updates — `agent/` has not changed since 2026-07-17 and would otherwise serve + that day's `nvidia/cuda` layers forever. Four things make it work: + - It **builds `main`, not the branch that triggered it.** Forgejo registers a + cron from the DEFAULT branch (`dev` here), so a scheduled run arrives with + `github.ref` on dev. The ref is decided once in a top-level `env: + BUILD_REF` that every checkout in the file takes, rather than per job — + otherwise `sign-extension` would derive dev's extension version while + `build-web` bundled main's, and the release download would 404 on a version + that exists perfectly well. + - It **publishes only `:latest`.** `:c-` for main's HEAD already names + the bytes that commit built; re-pushing it over refreshed layers would + break the one tag rule 145 makes immutable, and it is the rollback unit. + The repoint step needs no schedule case for this — the tag list is the + channel tag alone, so SOURCE is the only entry, it is excluded as always, + and the step correctly does nothing. + - **`:latest` and `:c-` therefore diverge between a refresh and the next + `main` push, by design.** They re-converge on that push: it hits reuse (a + refresh does not move `fc.revision`, because it does not touch the source), + and the repoint writes the NEW `:c-` from the refreshed `:latest`. The + push path needed no change for this, because the repoint already excluded + the source tag — the same rule that keeps the label readable also keeps a + refresh from being undone. + - **`pull: true` on the scheduled path only** is the actual mechanism. If a + base tag moved, the `FROM` layer's cache key changes and everything above + it rebuilds; if it did not, the registry cache satisfies the whole graph + and the refresh is a ~13s no-op that republishes nothing. That no-op is the + point — `:latest` should change when there is something new in it, not + every Sunday. The known lag: a Debian package update inside the `apt-get + install` layer while the base tag stands still is not caught. Closing it + needs `no-cache: true`, which buys weekly churn for it; the official + python/cuda images rebuild with those updates baked in, so this is a lag + rather than a hole. - **`FC_CHANNEL` and `FC_VERSION` are build args, not runtime settings.** `build.yml` passes them to the web image only — the ml and agent images have nothing to report them to. `/api/health` returns both, the foot of Settings -- 2.54.0 From 6663e06aa65940814857c8e06490a70aeb4afaba Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sat, 29 Aug 2026 22:57:41 -0400 Subject: [PATCH 02/17] ci: assert the scheduled refresh actually checked out main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BUILD_REF is read through the `env` context inside `with:`, which this runner is not known to evaluate. `${{ steps.* }}` and `${{ secrets.* }}` in `with:`/`env:` are proven here; `env` is not, and run 4915's checkout log (`git checkout -B dev refs/remotes/origin/dev`) cannot tell an honoured `refs/heads/dev` from an empty value falling back to the same place — the two are indistinguishable on every path except the one that matters. If it does resolve empty, the weekly refresh checks out dev and pushes its source to :latest, which is production. Every lane stays green and the first symptom is production running code that was never merged. So each of the four jobs now asserts its own checkout before doing anything, gated on `github.event_name` — the `github` context is demonstrably evaluated in `if:`, so the guard cannot be disabled by the same uncertainty it covers. A red weekly job is an acceptable outcome; shipping dev to production is not. --- .forgejo/workflows/build.yml | 69 ++++++++++++++++++++++++++++++++++++ ci-requirements.md | 6 +++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 3c6d666..7498f3e 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -128,6 +128,33 @@ jobs: # wrong, too-low value rather than failing (ci-requirements.md). fetch-depth: 0 + # BUILD_REF is what makes a scheduled run build `main` rather than the + # branch its cron fired from — and it is read through the `env` context + # inside `with:`, which this runner is NOT known to evaluate. If it does + # not, checkout silently falls back to the triggering ref and the weekly + # refresh publishes DEV's source to `:latest`, which is production. + # Every lane would stay green; the first sign of it would be production + # running code that was never merged. + # + # So assert the checkout instead of trusting the expression. A red + # weekly job is a fine outcome. Shipping dev to production is not. + # + # `if:` reads the `github` context, which the runner demonstrably does + # evaluate — this file already gates steps on it — so the guard cannot + # be disabled by the same uncertainty it exists to cover. + - name: Guard — a scheduled run must have checked out main + if: github.event_name == 'schedule' + run: | + set -eu + BRANCH=$(git rev-parse --abbrev-ref HEAD) + echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))" + if [ "$BRANCH" != "main" ]; then + echo "schedule: expected main, got '$BRANCH'." >&2 + echo "schedule: BUILD_REF was not honoured by the runner." >&2 + echo "schedule: refusing to publish a channel tag from it." >&2 + exit 1 + fi + # The version is DERIVED, not read from the repo (milestone 271 step 4, # cut over 2026-08-27). `packaging.sh version` returns `YYYY.M.D.HHMM` # UTC — the commit TIME of the newest change to a PACKAGED extension @@ -408,6 +435,20 @@ jobs: # that exists perfectly well under its real name. fetch-depth: 0 + # See sign-extension's copy for why this guard exists. + - name: Guard — a scheduled run must have checked out main + if: github.event_name == 'schedule' + run: | + set -eu + BRANCH=$(git rev-parse --abbrev-ref HEAD) + echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))" + if [ "$BRANCH" != "main" ]; then + echo "schedule: expected main, got '$BRANCH'." >&2 + echo "schedule: BUILD_REF was not honoured by the runner." >&2 + echo "schedule: refusing to publish a channel tag from it." >&2 + exit 1 + fi + # --- derived values, one line (milestone 313) ------------------------ # These stopped being shadow output at step 3. `revision` decides # whether the build below runs at all and `version` is what the image @@ -907,6 +948,20 @@ jobs: # the build would otherwise notice. fetch-depth: 0 + # See sign-extension's copy for why this guard exists. + - name: Guard — a scheduled run must have checked out main + if: github.event_name == 'schedule' + run: | + set -eu + BRANCH=$(git rev-parse --abbrev-ref HEAD) + echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))" + if [ "$BRANCH" != "main" ]; then + echo "schedule: expected main, got '$BRANCH'." >&2 + echo "schedule: BUILD_REF was not honoured by the runner." >&2 + echo "schedule: refusing to publish a channel tag from it." >&2 + exit 1 + fi + # --- derived values, one line (milestone 313) ------------------------ # These stopped being shadow output at step 3. `revision` decides # whether the build below runs at all and `version` is what the image @@ -1260,6 +1315,20 @@ jobs: # the build would otherwise notice. fetch-depth: 0 + # See sign-extension's copy for why this guard exists. + - name: Guard — a scheduled run must have checked out main + if: github.event_name == 'schedule' + run: | + set -eu + BRANCH=$(git rev-parse --abbrev-ref HEAD) + echo "schedule: HEAD is $BRANCH ($(git rev-parse --short HEAD))" + if [ "$BRANCH" != "main" ]; then + echo "schedule: expected main, got '$BRANCH'." >&2 + echo "schedule: BUILD_REF was not honoured by the runner." >&2 + echo "schedule: refusing to publish a channel tag from it." >&2 + exit 1 + fi + # --- derived values, one line (milestone 313) ------------------------ # These stopped being shadow output at step 3. `revision` decides # whether the build below runs at all and `version` is what the image diff --git a/ci-requirements.md b/ci-requirements.md index 330471c..016fdd3 100644 --- a/ci-requirements.md +++ b/ci-requirements.md @@ -178,7 +178,11 @@ per `docs/process.md`'s "add deps to the image when used by >1 project". BUILD_REF` that every checkout in the file takes, rather than per job — otherwise `sign-extension` would derive dev's extension version while `build-web` bundled main's, and the release download would 404 on a version - that exists perfectly well. + that exists perfectly well. Every job then ASSERTS its checkout is `main` + before doing anything, because `env` inside `with:` is not a context this + runner is known to evaluate — if it silently resolved to empty, checkout + would fall back to the triggering ref and the refresh would publish dev's + source to `:latest` with every lane green. - It **publishes only `:latest`.** `:c-` for main's HEAD already names the bytes that commit built; re-pushing it over refreshed layers would break the one tag rule 145 makes immutable, and it is the rollback unit. -- 2.54.0 From 0a5bbe81dc1e5483cd98e4c9f59af57b54f68c8b Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 12:57:06 -0400 Subject: [PATCH 03/17] docs: the scheduled refresh does NOT republish nothing (#3265) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 4 asserted that when the base has not moved the refresh is "a ~13s no-op that republishes nothing", and that this no-op was the point. The first half is false and was written without being tested. Run 4934, the first real fire: every content step reported CACHED and both bases resolved to unchanged pinned digests, yet all three :latest tags took a new manifest digest anyway. fabledcurator 4ea5265ba017 -> 380e504de0fa fabledcurator-ml 6e7cfc0c09fd -> 6b2eefc301d8 fabledcurator-agent 44920e0af1f3 -> 54accbeb52ed buildkit mints a fresh image config per run, so identical layers get republished under a new config blob. Storage cost is trivial; the cost that matters is that a :latest digest change stops meaning "something is different", and :c- is handed a new manifest to diverge from every Sunday for no reason. Corrects the workflow comment (x3) and ci-requirements.md to say what actually happens. Filed as #3265 with the candidate fixes; the likely one is a deterministic SOURCE_DATE_EPOCH off the value artifacts.sh already derives, which would make "same source, same version" into "same source, same bytes". The rest of step 4 verified clean on the same run: the guard passed (HEAD is main (499720d), `git checkout -B main`) — so this runner DOES evaluate the env context inside `with:` — the tag list was :latest alone with no :c-, and the repoint step correctly found nothing to write. --- .forgejo/workflows/build.yml | 99 ++++++++++++++++++++++-------------- ci-requirements.md | 23 +++++---- 2 files changed, 73 insertions(+), 49 deletions(-) diff --git a/.forgejo/workflows/build.yml b/.forgejo/workflows/build.yml index 7498f3e..7bd8066 100644 --- a/.forgejo/workflows/build.yml +++ b/.forgejo/workflows/build.yml @@ -773,23 +773,30 @@ jobs: # trusting whatever digest the cache was built against. This is the # whole mechanism of the scheduled refresh (#3154): if the base tag # moved, the FROM layer's cache key changes, every layer above it - # invalidates, and the image genuinely rebuilds. If it did not move, - # the registry cache satisfies the entire graph and the refresh is a - # ~13s no-op that republishes nothing. + # invalidates, and the image genuinely rebuilds. # - # That no-op is the POINT, not a shortfall: :latest should change - # when there is something new in it and not otherwise. A refresh - # that rewrote the image weekly regardless would churn the registry - # and hand :c- a new manifest to diverge from every Sunday, for - # no gain. + # MEASURED on the first real fire, run 4934 (#3265): when the base + # did NOT move, the build is ~13s and every content step reports + # CACHED — but the channel tag STILL gets a new manifest digest. + # buildkit mints a fresh image config each run, so identical layers + # are republished under a new config blob. All three images moved + # that way on 2026-08-30 with nothing whatsoever changed in them. # - # What it therefore does NOT catch: a Debian package update inside + # So a refresh currently rewrites :latest every Sunday whether or + # not there is anything new in it, and :c- is handed a new + # manifest to diverge from on the same cadence. Layers are shared, + # so the storage cost is a config blob; the cost that matters is + # that a digest change no longer MEANS anything. Tracked in #3265 — + # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would + # make "same source, same bytes" true and turn the no-op case into + # a genuine no-op. + # + # What `pull` does NOT catch either: a Debian package update inside # the `apt-get install` layer while the base tag itself stands # still. The official python/cuda images rebuild with those updates - # baked in, so this is a lag rather than a hole — but closing it - # would take `no-cache: true` on the scheduled path, which is the - # weekly-churn trade above. Left as the cheaper of the two on - # purpose. + # baked in, so this is a lag rather than a hole; closing it needs + # `no-cache: true`, which is a much larger version of the same + # churn #3265 is about. # # Only on the schedule. An ordinary push wants the cached base. pull: ${{ github.event_name == 'schedule' }} @@ -1149,23 +1156,30 @@ jobs: # trusting whatever digest the cache was built against. This is the # whole mechanism of the scheduled refresh (#3154): if the base tag # moved, the FROM layer's cache key changes, every layer above it - # invalidates, and the image genuinely rebuilds. If it did not move, - # the registry cache satisfies the entire graph and the refresh is a - # ~13s no-op that republishes nothing. + # invalidates, and the image genuinely rebuilds. # - # That no-op is the POINT, not a shortfall: :latest should change - # when there is something new in it and not otherwise. A refresh - # that rewrote the image weekly regardless would churn the registry - # and hand :c- a new manifest to diverge from every Sunday, for - # no gain. + # MEASURED on the first real fire, run 4934 (#3265): when the base + # did NOT move, the build is ~13s and every content step reports + # CACHED — but the channel tag STILL gets a new manifest digest. + # buildkit mints a fresh image config each run, so identical layers + # are republished under a new config blob. All three images moved + # that way on 2026-08-30 with nothing whatsoever changed in them. # - # What it therefore does NOT catch: a Debian package update inside + # So a refresh currently rewrites :latest every Sunday whether or + # not there is anything new in it, and :c- is handed a new + # manifest to diverge from on the same cadence. Layers are shared, + # so the storage cost is a config blob; the cost that matters is + # that a digest change no longer MEANS anything. Tracked in #3265 — + # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would + # make "same source, same bytes" true and turn the no-op case into + # a genuine no-op. + # + # What `pull` does NOT catch either: a Debian package update inside # the `apt-get install` layer while the base tag itself stands # still. The official python/cuda images rebuild with those updates - # baked in, so this is a lag rather than a hole — but closing it - # would take `no-cache: true` on the scheduled path, which is the - # weekly-churn trade above. Left as the cheaper of the two on - # purpose. + # baked in, so this is a lag rather than a hole; closing it needs + # `no-cache: true`, which is a much larger version of the same + # churn #3265 is about. # # Only on the schedule. An ordinary push wants the cached base. pull: ${{ github.event_name == 'schedule' }} @@ -1511,23 +1525,30 @@ jobs: # trusting whatever digest the cache was built against. This is the # whole mechanism of the scheduled refresh (#3154): if the base tag # moved, the FROM layer's cache key changes, every layer above it - # invalidates, and the image genuinely rebuilds. If it did not move, - # the registry cache satisfies the entire graph and the refresh is a - # ~13s no-op that republishes nothing. + # invalidates, and the image genuinely rebuilds. # - # That no-op is the POINT, not a shortfall: :latest should change - # when there is something new in it and not otherwise. A refresh - # that rewrote the image weekly regardless would churn the registry - # and hand :c- a new manifest to diverge from every Sunday, for - # no gain. + # MEASURED on the first real fire, run 4934 (#3265): when the base + # did NOT move, the build is ~13s and every content step reports + # CACHED — but the channel tag STILL gets a new manifest digest. + # buildkit mints a fresh image config each run, so identical layers + # are republished under a new config blob. All three images moved + # that way on 2026-08-30 with nothing whatsoever changed in them. # - # What it therefore does NOT catch: a Debian package update inside + # So a refresh currently rewrites :latest every Sunday whether or + # not there is anything new in it, and :c- is handed a new + # manifest to diverge from on the same cadence. Layers are shared, + # so the storage cost is a config blob; the cost that matters is + # that a digest change no longer MEANS anything. Tracked in #3265 — + # the likely fix is a deterministic SOURCE_DATE_EPOCH, which would + # make "same source, same bytes" true and turn the no-op case into + # a genuine no-op. + # + # What `pull` does NOT catch either: a Debian package update inside # the `apt-get install` layer while the base tag itself stands # still. The official python/cuda images rebuild with those updates - # baked in, so this is a lag rather than a hole — but closing it - # would take `no-cache: true` on the scheduled path, which is the - # weekly-churn trade above. Left as the cheaper of the two on - # purpose. + # baked in, so this is a lag rather than a hole; closing it needs + # `no-cache: true`, which is a much larger version of the same + # churn #3265 is about. # # Only on the schedule. An ordinary push wants the cached base. pull: ${{ github.event_name == 'schedule' }} diff --git a/ci-requirements.md b/ci-requirements.md index 016fdd3..ccd3081 100644 --- a/ci-requirements.md +++ b/ci-requirements.md @@ -196,16 +196,19 @@ per `docs/process.md`'s "add deps to the image when used by >1 project". push path needed no change for this, because the repoint already excluded the source tag — the same rule that keeps the label readable also keeps a refresh from being undone. - - **`pull: true` on the scheduled path only** is the actual mechanism. If a - base tag moved, the `FROM` layer's cache key changes and everything above - it rebuilds; if it did not, the registry cache satisfies the whole graph - and the refresh is a ~13s no-op that republishes nothing. That no-op is the - point — `:latest` should change when there is something new in it, not - every Sunday. The known lag: a Debian package update inside the `apt-get - install` layer while the base tag stands still is not caught. Closing it - needs `no-cache: true`, which buys weekly churn for it; the official - python/cuda images rebuild with those updates baked in, so this is a lag - rather than a hole. + - **`pull: true` on the scheduled path only** is the mechanism: a moved base + tag changes the `FROM` layer's cache key and everything above it rebuilds. + **It does not currently make the unmoved case free.** Measured on the first + real fire (run 4934, 2026-08-30): every content step reported `CACHED` and + the bases resolved to unchanged digests, yet all three `:latest` tags got a + NEW manifest digest, because buildkit mints a fresh image config per run and + republishes identical layers under it. So `:latest` is rewritten weekly + whether or not anything changed, and `:c-` is handed a new manifest to + diverge from on the same cadence — a digest change stops meaning anything. + Tracked as #3265; the likely fix is a deterministic `SOURCE_DATE_EPOCH`. + Separately not caught: a Debian package update inside the `apt-get install` + layer while the base tag stands still — a lag rather than a hole, since the + official python/cuda images rebuild with those updates baked in. - **`FC_CHANNEL` and `FC_VERSION` are build args, not runtime settings.** `build.yml` passes them to the web image only — the ml and agent images have nothing to report them to. `/api/health` returns both, the foot of Settings -- 2.54.0 From 62583791d8dd17aa4a23f6fc82e41b3c8c4d6e82 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 13:31:16 -0400 Subject: [PATCH 04/17] ci: a workflow that proves a collapsed alembic chain matches the old one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone 328 step 1 needs a baseline generated from the models, and step 2 must not stamp the operator's live database until that baseline is proven to reproduce what the 87-revision chain produced. `alembic stamp` validates nothing, so an unproven baseline fails silently now and loudly later, on real data. There is no local Python environment and rules 10/12 point away from standing one up, so the comparison runs in CI, where a pgvector Postgres is already built from the chain on every integration run and nothing is at risk. It builds two databases and diffs their pg_dump --schema-only output: one from `alembic upgrade head` on the revisions read out of git at `chain_ref`, one from the current tree. Reading the chain from git via a worktree — rather than from the working tree — is what keeps this usable AFTER the old revisions are deleted, so it is the proof for step 1 and the pre-flight for step 2 rather than a one-shot script. Both sides use `alembic upgrade head`, never metadata.create_all, per rule 82 — and that rule's reasoning is exactly the hazard here. `create_all` emits plain CREATE TABLE and skips everything else, which is why the optional autogenerated candidate CANNOT be trusted as the answer. Three things in this schema are invisible to SQLAlchemy metadata: CREATE EXTENSION vector (0001) CREATE EXTENSION tsm_system_rows (0004) the HNSW index on image_record.siglip_embedding, raw SQL because alembic's create_index cannot express USING hnsw (...) (0036) plus any CHECK constraint or server_default a migration added without the model declaring it — 4 model files declare CheckConstraints against 6 migrations that touch them. The candidate is a starting point to hand finish; the diff is what proves nothing was missed. Results are printed to the job log rather than uploaded: ci-requirements records that this runner cannot do actions/upload-artifact@v4+, and the repo dropped the action entirely in 2026-05. Run it first with the chain still present, as a control — the diff compares the chain against itself and must come back clean. A clean diff after the squash only means something if the harness was shown to be capable of producing one beforehand. Temporary. Delete once the baseline is stamped. --- .forgejo/workflows/baseline.yml | 178 ++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 .forgejo/workflows/baseline.yml diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml new file mode 100644 index 0000000..da9fa9d --- /dev/null +++ b/.forgejo/workflows/baseline.yml @@ -0,0 +1,178 @@ + +# TEMPORARY — milestone 328 steps 1-2. Delete once the baseline is stamped. +# +# Squashing 87 alembic revisions into one baseline has exactly one dangerous +# failure: the generated baseline does not reproduce the schema the chain +# produced, `alembic stamp` writes a version string anyway (it validates +# NOTHING), and the divergence surfaces on the next real migration against the +# operator's live data. +# +# So this workflow does the comparison in CI, where a pgvector Postgres already +# gets built from the chain on every integration run, and nothing is at risk. +# It answers one question: does `upgrade head` on the collapsed chain produce a +# byte-identical schema to `upgrade head` on the 87-revision chain? +# +# The chain is read from git rather than from the working tree, so this keeps +# working AFTER the old revisions are deleted — `chain_ref` names a commit that +# still has them. That is what makes this the proof for step 1 and the +# pre-flight for step 2, rather than a one-shot script. +# +# `generate: true` additionally autogenerates a candidate baseline from the +# models and uploads it. That is a starting point, NOT the answer: autogenerate +# reads SQLAlchemy metadata, and three things in this schema do not live there — +# * CREATE EXTENSION vector (0001) +# * CREATE EXTENSION tsm_system_rows (0004) +# * the HNSW index on image_record.siglip_embedding, which is raw SQL +# because alembic's create_index cannot express `USING hnsw (...)` (0036) +# plus any CHECK constraint or server_default that a migration added without +# the model declaring it. Those must be hand-added, and the diff below is what +# proves none were missed. +name: Alembic baseline + +on: + workflow_dispatch: + inputs: + chain_ref: + description: 'Commit/tag that still carries the full 0001..0087 chain' + type: string + default: '0a5bbe8' + generate: + description: 'Also autogenerate a candidate baseline from the models' + type: boolean + default: false + +jobs: + compare: + runs-on: python-ci + container: + image: git.fabledsword.com/bvandeusen/ci-python:3.14 + env: + DB_USER: fabledcurator + DB_PASSWORD: ci_integration + DB_PORT: "5432" + DB_NAME: fabledcurator_test + SECRET_KEY: ci_integration_placeholder + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: fabledcurator + POSTGRES_PASSWORD: ci_integration + POSTGRES_DB: fabledcurator_test + options: >- + --health-cmd "pg_isready -U fabledcurator" + --health-interval 10s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v4 + with: + # Full history is the point: `chain_ref` is read out of git, so a + # shallow clone would not have the revisions to compare against. + fetch-depth: 0 + + - name: Resolve the Postgres service and install deps + run: | + set -eux + # Same service-IP dance as ci.yml's integration job; see the long + # comment there for why the job name must stay separator-free. + PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1) + test -n "$PG" + PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG") + test -n "$PG_IP" + echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV" + echo "DB_HOST=$PG_IP" >> "$GITHUB_ENV" + for i in $(seq 1 60); do + (echo > "/dev/tcp/$PG_IP/5432") >/dev/null 2>&1 && break + sleep 2 + done + if command -v uv >/dev/null 2>&1; then + uv pip install --system -r requirements.txt + else + pip install -r requirements.txt + fi + + # DB 1: the 87-revision chain, read out of git at `chain_ref`. + # + # A git worktree rather than a checkout, so the current tree — which is + # what we are testing — is left completely alone. + - name: Build the schema the OLD chain produces + env: + CHAIN_REF: ${{ github.event.inputs.chain_ref }} + run: | + set -eux + docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_chain + git worktree add /tmp/chain "$CHAIN_REF" + ls /tmp/chain/alembic/versions/*.py | wc -l + cd /tmp/chain + DB_NAME=fc_chain alembic upgrade head + cd - + docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \ + --no-owner --no-privileges -d fc_chain > chain.sql + wc -l chain.sql + + # Optional: a candidate baseline, autogenerated from the models against an + # EMPTY database so every table shows up as a create. Uploaded for a human + # to finish — it will be missing the three raw-SQL items named at the top. + - name: Autogenerate a candidate baseline + if: ${{ github.event.inputs.generate == 'true' }} + run: | + set -eux + docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_gen + # Hide the existing revisions so alembic sees an empty history and + # emits the whole schema rather than a delta. + mkdir -p /tmp/versions_held + mv alembic/versions/*.py /tmp/versions_held/ 2>/dev/null || true + DB_NAME=fc_gen alembic revision --autogenerate -m "baseline" || true + # Printed in full rather than uploaded. ci-requirements.md records + # that this runner cannot do actions/upload-artifact@v4+, and the + # repo dropped the action entirely in 2026-05; the job log is the + # retrieval channel that is actually proven here. + echo "===== BEGIN CANDIDATE BASELINE =====" + cat alembic/versions/*.py + echo "===== END CANDIDATE BASELINE =====" + # Put the tree back exactly as it was; this job never mutates state. + rm -f alembic/versions/*.py + mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true + + # DB 2: whatever the CURRENT tree's alembic/versions produces. Before the + # squash that is the same 87 revisions and the diff is trivially clean — + # which is worth running once as a control, so a clean diff after the + # squash means something. + - name: Build the schema the CURRENT tree produces + run: | + set -eux + docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base + ls alembic/versions/*.py | wc -l + DB_NAME=fc_base alembic upgrade head + docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \ + --no-owner --no-privileges -d fc_base > baseline.sql + wc -l baseline.sql + + # The verdict. + # + # pg_dump orders dumpable objects by name within type, not by creation + # order, so two schemas built by different routes are directly + # comparable. The only normalisation applied is dropping blank lines, + # comment lines and the alembic_version row-count noise — deliberately + # minimal, because a filter that hides a real difference is the one way + # this check passes when it should fail. Whatever is normalised is + # printed, so the filtering itself is reviewable. + - name: Diff + run: | + set -eu + norm() { + grep -vE '^\s*(--|$)' "$1" | sed 's/[[:space:]]*$//' + } + norm chain.sql > a.txt + norm baseline.sql > b.txt + echo "normalised: chain=$(wc -l < a.txt) lines, current=$(wc -l < b.txt) lines" + if diff -u a.txt b.txt > schema.diff; then + echo "SCHEMAS IDENTICAL — the collapsed chain reproduces the old one." + else + echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' schema.diff) changed lines:" + cat schema.diff + echo + echo "The baseline is wrong, not the database. Do not stamp." + exit 1 + fi -- 2.54.0 From 5fd171a54479bdb30039435984de8e8f8f6fd461 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 13:35:26 -0400 Subject: [PATCH 05/17] ci: fix two things the baseline control run found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 4960 was the control — the chain compared against itself, which must come back clean before a clean diff after the squash means anything. It did its job and failed on both counts. 1. The harness is sound. Both dumps came back 1123 normalised lines and differed on EXACTLY two, the \restrict / \unrestrict pair that newer pg_dump emits to fence a dump against injection during restore. It is a fresh random nonce per invocation, so it differs by construction and is noise by definition. Now filtered — and the control is what licenses that filter: it was OBSERVED to be the only false positive rather than assumed to be one, which matters for a check whose whole value is that its normalisation does not hide a real difference. 2. The candidate-baseline step never ran. `if: github.event.inputs .generate == 'true'` on a `type: boolean` input silently evaluated false — no diagnostic, step skipped, job carried on. The same `github.event.inputs` typing quirk build.yml already works around for force_build. Rather than fight the input typing, the gate is now the tree itself: skip if alembic/versions holds one file. That is the real question anyway — there is nothing to generate once the chain is collapsed — and it cannot be silently wrong the way an unevaluated expression can. Worth noting what the control also proved incidentally: the two schemas were byte-identical across 1123 lines despite being built by separate alembic runs into separate databases, so pg_dump's object ordering is stable enough to diff directly and no sort normalisation is needed. --- .forgejo/workflows/baseline.yml | 50 ++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index da9fa9d..96e193e 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -17,9 +17,10 @@ # still has them. That is what makes this the proof for step 1 and the # pre-flight for step 2, rather than a one-shot script. # -# `generate: true` additionally autogenerates a candidate baseline from the -# models and uploads it. That is a starting point, NOT the answer: autogenerate -# reads SQLAlchemy metadata, and three things in this schema do not live there — +# While the chain is still present it also autogenerates a candidate baseline +# from the models and prints it. That is a starting point, NOT the answer: +# autogenerate reads SQLAlchemy metadata, and three things here do not live +# there — # * CREATE EXTENSION vector (0001) # * CREATE EXTENSION tsm_system_rows (0004) # * the HNSW index on image_record.siglip_embedding, which is raw SQL @@ -36,10 +37,6 @@ on: description: 'Commit/tag that still carries the full 0001..0087 chain' type: string default: '0a5bbe8' - generate: - description: 'Also autogenerate a candidate baseline from the models' - type: boolean - default: false jobs: compare: @@ -111,13 +108,23 @@ jobs: --no-owner --no-privileges -d fc_chain > chain.sql wc -l chain.sql - # Optional: a candidate baseline, autogenerated from the models against an - # EMPTY database so every table shows up as a create. Uploaded for a human - # to finish — it will be missing the three raw-SQL items named at the top. + # A candidate baseline, autogenerated from the models against an EMPTY + # database so every table shows up as a create. Printed for a human to + # finish — it will be missing the three raw-SQL items named at the top. + # + # Gated on the TREE, not on a workflow input. A `type: boolean` input + # read back as `github.event.inputs.generate == 'true'` silently + # evaluated false on this runner (run 4960 skipped this step entirely + # with no diagnostic) — the same `github.event.inputs` typing quirk + # build.yml already works around. The file count is the real question + # anyway: there is nothing to generate once the chain is collapsed. - name: Autogenerate a candidate baseline - if: ${{ github.event.inputs.generate == 'true' }} run: | set -eux + if [ "$(ls alembic/versions/*.py | wc -l)" -le 1 ]; then + echo "already collapsed — nothing to generate" + exit 0 + fi docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_gen # Hide the existing revisions so alembic sees an empty history and # emits the whole schema rather than a delta. @@ -153,16 +160,25 @@ jobs: # # pg_dump orders dumpable objects by name within type, not by creation # order, so two schemas built by different routes are directly - # comparable. The only normalisation applied is dropping blank lines, - # comment lines and the alembic_version row-count noise — deliberately - # minimal, because a filter that hides a real difference is the one way - # this check passes when it should fail. Whatever is normalised is - # printed, so the filtering itself is reviewable. + # comparable. Normalisation is deliberately minimal, because a filter + # that hides a real difference is the one way this check passes when it + # should fail — blank lines, SQL comments, trailing whitespace, and: + # + # \restrict / \unrestrict — a per-invocation RANDOM NONCE that newer + # pg_dump emits to fence the dump against injection during restore. It + # differs on every run by construction, so it is noise by definition, + # not a schema difference. Measured on run 4960, the control: two dumps + # of the SAME schema came back 1123 lines each and differed on exactly + # these two lines and nothing else. That control is what licenses this + # filter — it was observed to be the only false positive, rather than + # assumed to be one. - name: Diff run: | set -eu norm() { - grep -vE '^\s*(--|$)' "$1" | sed 's/[[:space:]]*$//' + grep -vE '^\s*(--|$)' "$1" \ + | grep -vE '^\\(un)?restrict ' \ + | sed 's/[[:space:]]*$//' } norm chain.sql > a.txt norm baseline.sql > b.txt -- 2.54.0 From 8f1ac0c96acd3db4502677c2400b79deade77da0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 13:40:15 -0400 Subject: [PATCH 06/17] ci: transport the candidate baseline as verifiable base64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 4964 passed the control (1121 normalised lines, schemas identical) but its candidate print was silently truncated. `cat` of the ~33KB generated file stopped mid-line inside sa.Column('mime', sa.String(length=128) and the runner carried straight on to the next traced command with the step still green. The captured text was 484 lines and 29 tables, and looked entirely plausible — which is exactly what makes it dangerous: a schema definition cut in half is still syntactically suggestive, and nothing in the log says it was cut. Now emitted as base64 at a fixed 120-column width, followed by a sha256, a byte count and a base64 line count. Short lines instead of long ones, and more importantly the receiving end can PROVE it got the whole file rather than trusting that it did. Also found in that output, and the reason the candidate could never have been committed as-is: it references pgvector.sqlalchemy.vector.VECTOR(dim=1152) for head_training_run.weights and image_record.siglip_embedding, but autogenerate does not add the corresponding import. The file would die with NameError on the first run. That is the fourth item on the list of things the generator cannot be trusted with, alongside the two CREATE EXTENSIONs and the HNSW index. --- .forgejo/workflows/baseline.yml | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index 96e193e..d7b4b66 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -131,13 +131,32 @@ jobs: mkdir -p /tmp/versions_held mv alembic/versions/*.py /tmp/versions_held/ 2>/dev/null || true DB_NAME=fc_gen alembic revision --autogenerate -m "baseline" || true - # Printed in full rather than uploaded. ci-requirements.md records - # that this runner cannot do actions/upload-artifact@v4+, and the - # repo dropped the action entirely in 2026-05; the job log is the - # retrieval channel that is actually proven here. - echo "===== BEGIN CANDIDATE BASELINE =====" - cat alembic/versions/*.py + # Printed rather than uploaded: ci-requirements.md records that this + # runner cannot do actions/upload-artifact@v4+, and the repo dropped + # the action entirely in 2026-05, so the job log is the retrieval + # channel actually proven here. + # + # base64, not the raw file. A plain `cat` of the ~33KB candidate was + # TRUNCATED MID-LINE by the runner on run 4964 — it stopped inside + # `sa.Column('mime', sa.String(length=128)` and carried straight on + # to the next traced command, with the step still green. A silent + # cut in the middle of a schema definition is the worst possible + # failure here, because the truncated text still looks like a + # plausible file. + # + # base64 at a fixed narrow width gives many short lines instead of + # few long ones, and — the actual point — a checksum and a line + # count that make truncation DETECTABLE rather than invisible. + set +x + F=$(ls alembic/versions/*.py | head -1) + B64=$(base64 -w 120 "$F") + echo "===== BEGIN CANDIDATE BASELINE (base64) =====" + echo "$B64" echo "===== END CANDIDATE BASELINE =====" + echo "candidate-sha256: $(sha256sum "$F" | cut -d' ' -f1)" + echo "candidate-bytes: $(wc -c < "$F")" + echo "candidate-b64-lines: $(echo "$B64" | wc -l)" + set -x # Put the tree back exactly as it was; this job never mutates state. rm -f alembic/versions/*.py mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true -- 2.54.0 From 2529b516e60d07689d8460339da93b51332256af Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 13:45:29 -0400 Subject: [PATCH 07/17] db: collapse alembic 0001..0087 into one baseline (milestone 328 step 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 87 revisions narrating this project's build-out become one file that creates the schema in a single step. They cost nothing at runtime — all 86 upgrade steps ran in 0.2s (note #3260) — so this is a presentation change, not a performance one: a new installer should not inherit our development history to stand up a database. Deleted: 87 revisions (6,052 lines), the 10 tests/test_migration_*.py files (483 lines) that asserted intermediate states and backfills which no longer exist, and backend/app/utils/artist_backfill.py — the only live module a migration imported, with no other consumer anywhere. That last one satisfies the operator's separate request to inline it into 0008 and delete the module; the squash removes both outright. THE REVISION ID IS "0087", NOT "0001", ON PURPOSE. It is the id of the last revision collapsed, so an existing database is already at head and `alembic upgrade head` does nothing. The alternative is `alembic stamp` against live data, and stamp validates NOTHING — it writes a version string whether or not the schema matches, so a wrong baseline surfaces later, via the next real migration, with no clean way back. This removes that operation rather than making it safe. Future revisions run from 0088. Four things are hand-written because SQLAlchemy metadata does not carry them, and none fail at generation time: 1. CREATE EXTENSION vector — the VECTOR columns cannot be created without it, so it is ordered first in upgrade(). 2. CREATE EXTENSION tsm_system_rows — surfaces only when the random sample query runs. 3. the HNSW index on image_record.siglip_embedding, raw SQL because create_index cannot express USING hnsw (... vector_cosine_ops). The quietest of the four: everything works, similarity search just stops using an index. 4. import pgvector.sqlalchemy.vector — autogenerate EMITS pgvector.sqlalchemy.vector.VECTOR references without importing it, so the generated file dies with NameError on first run. The candidate came out of CI (run 4967) as checksummed base64 rather than a plain cat, because run 4964's cat was truncated mid-line inside a column definition with the step still green — 29 tables instead of 42, and it looked entirely plausible. Verified here: 56,582 bytes, sha256 471acfca69c0…, 42 tables, 66 indexes, 42 drops. NOT YET PROVEN against the old chain. baseline.yml does that, and it is step 2's gate; this commit does not claim the schemas match. --- .../versions/0001_initial_unified_schema.py | 277 ------ .../0002_fc2a_tag_kinds_and_import_tasks.py | 208 ----- alembic/versions/0003_fc2b_ml_pipeline.py | 172 ---- .../versions/0004_fc2c_i_tsm_system_rows.py | 23 - .../versions/0005_fc2c_iii_a_series_page.py | 50 - alembic/versions/0006_fc2d_phash_threshold.py | 30 - .../0007_fc2d_post_metadata_fields.py | 31 - .../0008_fc2d_vii_c_artist_deconfliction.py | 52 -- .../versions/0009_fc2d_iii_post_attachment.py | 68 -- ..._fc3a_source_unique_artist_platform_url.py | 32 - .../0011_fc3b_credential_schema_alignment.py | 41 - alembic/versions/0012_fc3b_app_setting.py | 36 - .../0013_fc3c_download_event_metadata.py | 52 -- alembic/versions/0014_fc3d_scheduling.py | 58 -- alembic/versions/0015_fc5_migration_run.py | 51 - alembic/versions/0016_fc3i_task_run.py | 86 -- alembic/versions/0017_fc3h_backup_run.py | 82 -- alembic/versions/0018_fc3h_backup_settings.py | 62 -- .../versions/0019_import_batch_refreshed.py | 38 - alembic/versions/0020_library_audit_run.py | 65 -- .../versions/0021_image_provenance_unique.py | 54 -- .../0022_source_per_artist_platform.py | 223 ----- .../0023_drop_meta_rating_tag_kinds.py | 99 -- ...24_backfill_post_title_from_description.py | 80 -- .../0025_fix_subscribestar_post_ids.py | 288 ------ ...26_import_task_recovery_count_refetched.py | 53 -- alembic/versions/0027_drop_migration_run.py | 50 - ...se_sidecar_synthetics_into_real_sources.py | 190 ---- ...029_drop_artist_copyright_ml_thresholds.py | 71 -- ...ullable_post_source_id_denorm_artist_id.py | 145 --- .../0031_source_backfill_runs_remaining.py | 45 - alembic/versions/0032_source_error_type.py | 41 - .../0033_suggestion_threshold_default_070.py | 48 - alembic/versions/0034_artist_visit.py | 53 -- .../0035_image_record_effective_date.py | 70 -- .../0036_siglip_embedding_hnsw_index.py | 41 - alembic/versions/0037_patreon_seen_media.py | 53 -- alembic/versions/0038_patreon_failed_media.py | 58 -- alembic/versions/0039_library_audit_resume.py | 40 - alembic/versions/0040_series_chapters.py | 108 --- alembic/versions/0041_series_suggestions.py | 98 -- .../0042_series_chapter_stated_part.py | 32 - .../0043_post_attachment_per_post_unique.py | 62 -- .../0044_ml_settings_tagger_store_floor.py | 37 - .../versions/0045_image_prediction_table.py | 69 -- .../versions/0046_drop_tagger_predictions.py | 43 - .../versions/0047_series_chapter_dividers.py | 175 ---- .../0048_series_page_pending_status.py | 45 - alembic/versions/0049_external_link_table.py | 90 -- .../0050_external_link_host_toggles.py | 38 - .../versions/0051_image_source_provenance.py | 38 - .../versions/0052_image_duration_seconds.py | 32 - .../0053_ml_settings_video_tagging.py | 49 - .../versions/0054_subscribestar_ledgers.py | 82 -- .../0055_image_provenance_from_attachment.py | 55 -- alembic/versions/0056_tag_eval_run.py | 43 - .../0057_tag_positive_confirmation.py | 40 - alembic/versions/0058_tag_head.py | 95 -- alembic/versions/0059_head_auto_apply.py | 70 -- alembic/versions/0060_head_metrics.py | 74 -- alembic/versions/0061_image_region.py | 59 -- alembic/versions/0062_gpu_job.py | 55 -- alembic/versions/0063_ccip_match_threshold.py | 33 - alembic/versions/0064_ccip_auto_apply.py | 42 - alembic/versions/0065_embedder_model_name.py | 35 - alembic/versions/0066_drop_centroids.py | 57 -- .../versions/0067_retire_camie_allowlist.py | 66 -- .../0068_drop_dead_tagger_settings.py | 80 -- alembic/versions/0069_default_siglip2.py | 51 - .../versions/0070_gpu_job_lease_indexes.py | 44 - .../0071_image_record_earliest_post_date.py | 80 -- .../versions/0072_gpu_job_triage_status.py | 32 - alembic/versions/0073_drop_tag_eval_run.py | 46 - .../0074_ml_settings_cpu_embed_enabled.py | 35 - alembic/versions/0075_tag_is_system.py | 60 -- alembic/versions/0076_pixiv_ledgers.py | 82 -- .../versions/0077_artist_name_not_unique.py | 32 - .../versions/0078_ml_settings_detectors.py | 83 -- alembic/versions/0079_character_prototypes.py | 77 -- .../0080_tag_head_train_fingerprint.py | 31 - .../0081_stricter_auto_apply_defaults.py | 43 - .../versions/0082_presentation_auto_hide.py | 85 -- alembic/versions/0083_post_translation.py | 73 -- .../0084_translation_strictness_override.py | 51 - alembic/versions/0085_wip_title_tagging.py | 35 - .../0086_process_auto_apply_settings.py | 61 -- alembic/versions/0087_baseline.py | 872 ++++++++++++++++++ .../versions/0087_wip_soft_title_tagging.py | 33 - backend/app/utils/artist_backfill.py | 44 - tests/test_migration_0002.py | 58 -- tests/test_migration_0003.py | 46 - tests/test_migration_0004.py | 27 - tests/test_migration_0007.py | 48 - tests/test_migration_0008.py | 137 --- tests/test_migration_0009.py | 37 - tests/test_migration_0010.py | 35 - tests/test_migration_0011.py | 32 - tests/test_migration_0012.py | 32 - tests/test_migration_0013.py | 31 - 99 files changed, 872 insertions(+), 6579 deletions(-) delete mode 100644 alembic/versions/0001_initial_unified_schema.py delete mode 100644 alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py delete mode 100644 alembic/versions/0003_fc2b_ml_pipeline.py delete mode 100644 alembic/versions/0004_fc2c_i_tsm_system_rows.py delete mode 100644 alembic/versions/0005_fc2c_iii_a_series_page.py delete mode 100644 alembic/versions/0006_fc2d_phash_threshold.py delete mode 100644 alembic/versions/0007_fc2d_post_metadata_fields.py delete mode 100644 alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py delete mode 100644 alembic/versions/0009_fc2d_iii_post_attachment.py delete mode 100644 alembic/versions/0010_fc3a_source_unique_artist_platform_url.py delete mode 100644 alembic/versions/0011_fc3b_credential_schema_alignment.py delete mode 100644 alembic/versions/0012_fc3b_app_setting.py delete mode 100644 alembic/versions/0013_fc3c_download_event_metadata.py delete mode 100644 alembic/versions/0014_fc3d_scheduling.py delete mode 100644 alembic/versions/0015_fc5_migration_run.py delete mode 100644 alembic/versions/0016_fc3i_task_run.py delete mode 100644 alembic/versions/0017_fc3h_backup_run.py delete mode 100644 alembic/versions/0018_fc3h_backup_settings.py delete mode 100644 alembic/versions/0019_import_batch_refreshed.py delete mode 100644 alembic/versions/0020_library_audit_run.py delete mode 100644 alembic/versions/0021_image_provenance_unique.py delete mode 100644 alembic/versions/0022_source_per_artist_platform.py delete mode 100644 alembic/versions/0023_drop_meta_rating_tag_kinds.py delete mode 100644 alembic/versions/0024_backfill_post_title_from_description.py delete mode 100644 alembic/versions/0025_fix_subscribestar_post_ids.py delete mode 100644 alembic/versions/0026_import_task_recovery_count_refetched.py delete mode 100644 alembic/versions/0027_drop_migration_run.py delete mode 100644 alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py delete mode 100644 alembic/versions/0029_drop_artist_copyright_ml_thresholds.py delete mode 100644 alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py delete mode 100644 alembic/versions/0031_source_backfill_runs_remaining.py delete mode 100644 alembic/versions/0032_source_error_type.py delete mode 100644 alembic/versions/0033_suggestion_threshold_default_070.py delete mode 100644 alembic/versions/0034_artist_visit.py delete mode 100644 alembic/versions/0035_image_record_effective_date.py delete mode 100644 alembic/versions/0036_siglip_embedding_hnsw_index.py delete mode 100644 alembic/versions/0037_patreon_seen_media.py delete mode 100644 alembic/versions/0038_patreon_failed_media.py delete mode 100644 alembic/versions/0039_library_audit_resume.py delete mode 100644 alembic/versions/0040_series_chapters.py delete mode 100644 alembic/versions/0041_series_suggestions.py delete mode 100644 alembic/versions/0042_series_chapter_stated_part.py delete mode 100644 alembic/versions/0043_post_attachment_per_post_unique.py delete mode 100644 alembic/versions/0044_ml_settings_tagger_store_floor.py delete mode 100644 alembic/versions/0045_image_prediction_table.py delete mode 100644 alembic/versions/0046_drop_tagger_predictions.py delete mode 100644 alembic/versions/0047_series_chapter_dividers.py delete mode 100644 alembic/versions/0048_series_page_pending_status.py delete mode 100644 alembic/versions/0049_external_link_table.py delete mode 100644 alembic/versions/0050_external_link_host_toggles.py delete mode 100644 alembic/versions/0051_image_source_provenance.py delete mode 100644 alembic/versions/0052_image_duration_seconds.py delete mode 100644 alembic/versions/0053_ml_settings_video_tagging.py delete mode 100644 alembic/versions/0054_subscribestar_ledgers.py delete mode 100644 alembic/versions/0055_image_provenance_from_attachment.py delete mode 100644 alembic/versions/0056_tag_eval_run.py delete mode 100644 alembic/versions/0057_tag_positive_confirmation.py delete mode 100644 alembic/versions/0058_tag_head.py delete mode 100644 alembic/versions/0059_head_auto_apply.py delete mode 100644 alembic/versions/0060_head_metrics.py delete mode 100644 alembic/versions/0061_image_region.py delete mode 100644 alembic/versions/0062_gpu_job.py delete mode 100644 alembic/versions/0063_ccip_match_threshold.py delete mode 100644 alembic/versions/0064_ccip_auto_apply.py delete mode 100644 alembic/versions/0065_embedder_model_name.py delete mode 100644 alembic/versions/0066_drop_centroids.py delete mode 100644 alembic/versions/0067_retire_camie_allowlist.py delete mode 100644 alembic/versions/0068_drop_dead_tagger_settings.py delete mode 100644 alembic/versions/0069_default_siglip2.py delete mode 100644 alembic/versions/0070_gpu_job_lease_indexes.py delete mode 100644 alembic/versions/0071_image_record_earliest_post_date.py delete mode 100644 alembic/versions/0072_gpu_job_triage_status.py delete mode 100644 alembic/versions/0073_drop_tag_eval_run.py delete mode 100644 alembic/versions/0074_ml_settings_cpu_embed_enabled.py delete mode 100644 alembic/versions/0075_tag_is_system.py delete mode 100644 alembic/versions/0076_pixiv_ledgers.py delete mode 100644 alembic/versions/0077_artist_name_not_unique.py delete mode 100644 alembic/versions/0078_ml_settings_detectors.py delete mode 100644 alembic/versions/0079_character_prototypes.py delete mode 100644 alembic/versions/0080_tag_head_train_fingerprint.py delete mode 100644 alembic/versions/0081_stricter_auto_apply_defaults.py delete mode 100644 alembic/versions/0082_presentation_auto_hide.py delete mode 100644 alembic/versions/0083_post_translation.py delete mode 100644 alembic/versions/0084_translation_strictness_override.py delete mode 100644 alembic/versions/0085_wip_title_tagging.py delete mode 100644 alembic/versions/0086_process_auto_apply_settings.py create mode 100644 alembic/versions/0087_baseline.py delete mode 100644 alembic/versions/0087_wip_soft_title_tagging.py delete mode 100644 backend/app/utils/artist_backfill.py delete mode 100644 tests/test_migration_0002.py delete mode 100644 tests/test_migration_0003.py delete mode 100644 tests/test_migration_0004.py delete mode 100644 tests/test_migration_0007.py delete mode 100644 tests/test_migration_0008.py delete mode 100644 tests/test_migration_0009.py delete mode 100644 tests/test_migration_0010.py delete mode 100644 tests/test_migration_0011.py delete mode 100644 tests/test_migration_0012.py delete mode 100644 tests/test_migration_0013.py diff --git a/alembic/versions/0001_initial_unified_schema.py b/alembic/versions/0001_initial_unified_schema.py deleted file mode 100644 index 0580b45..0000000 --- a/alembic/versions/0001_initial_unified_schema.py +++ /dev/null @@ -1,277 +0,0 @@ -"""initial unified schema - -Revision ID: 0001 -Revises: -Create Date: 2026-05-13 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from pgvector.sqlalchemy import Vector - -revision: str = "0001" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute("CREATE EXTENSION IF NOT EXISTS vector") - - op.create_table( - "artist", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=255), nullable=False), - sa.Column("slug", sa.String(length=255), nullable=False), - sa.Column("notes", sa.Text(), nullable=True), - sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()), - sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()), - sa.Column("check_interval_seconds", sa.Integer(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.PrimaryKeyConstraint("id", name="pk_artist"), - sa.UniqueConstraint("name", name="uq_artist_name"), - sa.UniqueConstraint("slug", name="uq_artist_slug"), - ) - - op.create_table( - "source", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("artist_id", sa.Integer(), nullable=False), - sa.Column("platform", sa.String(length=64), nullable=False), - sa.Column("url", sa.Text(), nullable=False), - sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), - sa.Column("config_overrides", sa.JSON(), nullable=True), - sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("last_error", sa.Text(), nullable=True), - sa.Column("check_interval_override", sa.Integer(), nullable=True), - sa.ForeignKeyConstraint( - ["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id", name="pk_source"), - ) - op.create_index("ix_source_artist_id", "source", ["artist_id"]) - - op.create_table( - "credential", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("platform", sa.String(length=64), nullable=False), - sa.Column("kind", sa.String(length=32), nullable=False), - sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False), - sa.Column("status", sa.String(length=32), nullable=False, server_default="active"), - sa.Column( - "captured_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint("id", name="pk_credential"), - sa.UniqueConstraint("platform", name="uq_credential_platform"), - ) - - op.create_table( - "post", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("external_post_id", sa.String(length=128), nullable=False), - sa.Column("post_url", sa.Text(), nullable=True), - sa.Column("post_title", sa.Text(), nullable=True), - sa.Column("post_date", sa.DateTime(timezone=True), nullable=True), - sa.Column("raw_metadata", sa.JSON(), nullable=True), - sa.Column( - "downloaded_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id", name="pk_post"), - sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), - ) - op.create_index("ix_post_source_id", "post", ["source_id"]) - - op.create_table( - "image_record", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("path", sa.Text(), nullable=False), - sa.Column("sha256", sa.String(length=64), nullable=False), - sa.Column("phash", sa.String(length=32), nullable=True), - sa.Column("size_bytes", sa.BigInteger(), nullable=False), - sa.Column("mime", sa.String(length=64), nullable=False), - sa.Column("width", sa.Integer(), nullable=True), - sa.Column("height", sa.Integer(), nullable=True), - sa.Column("thumbnail_path", sa.Text(), nullable=True), - sa.Column( - "origin", - sa.Enum( - "downloaded", - "imported_filesystem", - "uploaded", - name="origin_enum", - ), - nullable=False, - ), - sa.Column("primary_post_id", sa.Integer(), nullable=True), - sa.Column("wd14_predictions", sa.JSON(), nullable=True), - sa.Column("wd14_model_version", sa.String(length=128), nullable=True), - sa.Column("siglip_embedding", Vector(1152), nullable=True), - sa.Column("siglip_model_version", sa.String(length=128), nullable=True), - sa.Column("centroid_scores", sa.JSON(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["primary_post_id"], - ["post.id"], - name="fk_image_record_primary_post_id_post", - ondelete="SET NULL", - ), - sa.PrimaryKeyConstraint("id", name="pk_image_record"), - sa.UniqueConstraint("path", name="uq_image_record_path"), - sa.UniqueConstraint("sha256", name="uq_image_record_sha256"), - ) - op.create_index("ix_image_record_sha256", "image_record", ["sha256"]) - op.create_index("ix_image_record_phash", "image_record", ["phash"]) - op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"]) - - op.create_table( - "image_provenance", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("image_record_id", sa.Integer(), nullable=False), - sa.Column("post_id", sa.Integer(), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("captured_metadata", sa.JSON(), nullable=True), - sa.Column( - "captured_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["image_record_id"], - ["image_record.id"], - name="fk_image_provenance_image_record_id_image_record", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["post_id"], - ["post.id"], - name="fk_image_provenance_post_id_post", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["source_id"], - ["source.id"], - name="fk_image_provenance_source_id_source", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("id", name="pk_image_provenance"), - ) - op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"]) - op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"]) - op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"]) - - op.create_table( - "tag", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("name", sa.String(length=255), nullable=False), - sa.Column("namespace", sa.String(length=64), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.PrimaryKeyConstraint("id", name="pk_tag"), - sa.UniqueConstraint("name", name="uq_tag_name"), - ) - op.create_index("ix_tag_name", "tag", ["name"]) - op.create_index("ix_tag_namespace", "tag", ["namespace"]) - - op.create_table( - "image_tag", - sa.Column("image_record_id", sa.Integer(), nullable=False), - sa.Column("tag_id", sa.Integer(), nullable=False), - sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["image_record_id"], - ["image_record.id"], - name="fk_image_tag_image_record_id_image_record", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"), - ) - - op.create_table( - "download_event", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("source_id", sa.Integer(), nullable=False), - sa.Column("post_id", sa.Integer(), nullable=True), - sa.Column("status", sa.String(length=32), nullable=False), - sa.Column( - "started_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"), - sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"), - sa.Column("error", sa.Text(), nullable=True), - sa.ForeignKeyConstraint( - ["source_id"], - ["source.id"], - name="fk_download_event_source_id_source", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["post_id"], - ["post.id"], - name="fk_download_event_post_id_post", - ondelete="SET NULL", - ), - sa.PrimaryKeyConstraint("id", name="pk_download_event"), - ) - op.create_index("ix_download_event_source_id", "download_event", ["source_id"]) - op.create_index("ix_download_event_post_id", "download_event", ["post_id"]) - - -def downgrade() -> None: - op.drop_table("download_event") - op.drop_table("image_tag") - op.drop_table("tag") - op.drop_table("image_provenance") - op.drop_table("image_record") - op.execute("DROP TYPE IF EXISTS origin_enum") - op.drop_table("post") - op.drop_table("credential") - op.drop_table("source") - op.drop_table("artist") - op.execute("DROP EXTENSION IF EXISTS vector") diff --git a/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py b/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py deleted file mode 100644 index b9dac2c..0000000 --- a/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py +++ /dev/null @@ -1,208 +0,0 @@ -"""fc2a: tag kinds, import_task, import_batch, integrity_status - -Revision ID: 0002 -Revises: 0001 -Create Date: 2026-05-14 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0002" -down_revision: Union[str, None] = "0001" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -TAG_KINDS = ( - "artist", - "character", - "fandom", - "general", - "series", - "archive", - "post", - "meta", - "rating", -) - - -def upgrade() -> None: - # --- Tag kind enum + fandom_id --- - tag_kind = sa.Enum(*TAG_KINDS, name="tag_kind") - tag_kind.create(op.get_bind(), checkfirst=True) - - op.add_column( - "tag", - sa.Column("kind", tag_kind, nullable=False, server_default="general"), - ) - op.add_column( - "tag", - sa.Column("fandom_id", sa.Integer(), nullable=True), - ) - op.create_foreign_key( - "fk_tag_fandom_id_tag", - "tag", - "tag", - ["fandom_id"], - ["id"], - ondelete="SET NULL", - ) - - # Drop the old global uniqueness on name; add kind+fandom-aware uniqueness. - op.drop_constraint("uq_tag_name", "tag", type_="unique") - op.drop_index("ix_tag_name", table_name="tag") - op.execute( - """ - CREATE UNIQUE INDEX uq_tag_name_kind_fandom - ON tag (name, kind, COALESCE(fandom_id, 0)) - """ - ) - - # CHECK: fandom_id is only allowed for character kind. - op.create_check_constraint( - "ck_tag_fandom_requires_character", - "tag", - "(fandom_id IS NULL) OR (kind = 'character')", - ) - - # Drop the old namespace column — superseded by kind. - op.drop_index("ix_tag_namespace", table_name="tag") - op.drop_column("tag", "namespace") - - # --- ImportBatch --- - op.create_table( - "import_batch", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("triggered_by", sa.String(length=32), nullable=False), - sa.Column("source_path", sa.Text(), nullable=False), - sa.Column("scan_mode", sa.String(length=16), nullable=False), - sa.Column( - "started_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("total_files", sa.Integer(), nullable=False, server_default="0"), - sa.Column("imported", sa.Integer(), nullable=False, server_default="0"), - sa.Column("skipped", sa.Integer(), nullable=False, server_default="0"), - sa.Column("failed", sa.Integer(), nullable=False, server_default="0"), - sa.Column("status", sa.String(length=16), nullable=False, server_default="running"), - sa.PrimaryKeyConstraint("id", name="pk_import_batch"), - ) - op.create_index("ix_import_batch_status", "import_batch", ["status"]) - - # --- ImportTask --- - op.create_table( - "import_task", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("batch_id", sa.Integer(), nullable=False), - sa.Column("source_path", sa.Text(), nullable=False), - sa.Column("task_type", sa.String(length=16), nullable=False), - sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"), - sa.Column("result_image_id", sa.Integer(), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("size_bytes", sa.BigInteger(), nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.func.now(), - ), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint( - ["batch_id"], - ["import_batch.id"], - name="fk_import_task_batch_id_import_batch", - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["result_image_id"], - ["image_record.id"], - name="fk_import_task_result_image_id_image_record", - ondelete="SET NULL", - ), - sa.PrimaryKeyConstraint("id", name="pk_import_task"), - ) - op.create_index("ix_import_task_batch_id", "import_task", ["batch_id"]) - op.create_index("ix_import_task_status", "import_task", ["status"]) - op.create_index( - "ix_import_task_created_at_desc", - "import_task", - [sa.text("created_at DESC")], - ) - - # --- ImportSettings (single-row table) --- - op.create_table( - "import_settings", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("import_scan_path", sa.Text(), nullable=False, server_default="/import"), - sa.Column("min_width", sa.Integer(), nullable=False, server_default="0"), - sa.Column("min_height", sa.Integer(), nullable=False, server_default="0"), - sa.Column( - "skip_transparent", sa.Boolean(), nullable=False, server_default=sa.false() - ), - sa.Column( - "transparency_threshold", - sa.Float(), - nullable=False, - server_default="0.9", - ), - sa.Column( - "skip_single_color", sa.Boolean(), nullable=False, server_default=sa.false() - ), - sa.Column( - "single_color_threshold", - sa.Float(), - nullable=False, - server_default="0.95", - ), - sa.Column("single_color_tolerance", sa.Integer(), nullable=False, server_default="30"), - sa.PrimaryKeyConstraint("id", name="pk_import_settings"), - sa.CheckConstraint("id = 1", name="ck_import_settings_singleton"), - ) - # Seed the single row immediately so callers can always SELECT id=1. - op.execute("INSERT INTO import_settings (id) VALUES (1)") - - # --- ImageRecord additions --- - op.add_column( - "image_record", - sa.Column( - "integrity_status", - sa.String(length=24), - nullable=False, - server_default="unknown", - ), - ) - op.create_index( - "ix_image_record_integrity_status", - "image_record", - ["integrity_status"], - ) - - -def downgrade() -> None: - op.drop_index("ix_image_record_integrity_status", table_name="image_record") - op.drop_column("image_record", "integrity_status") - - op.drop_table("import_settings") - op.drop_index("ix_import_task_created_at_desc", table_name="import_task") - op.drop_index("ix_import_task_status", table_name="import_task") - op.drop_index("ix_import_task_batch_id", table_name="import_task") - op.drop_table("import_task") - op.drop_index("ix_import_batch_status", table_name="import_batch") - op.drop_table("import_batch") - - op.drop_constraint("ck_tag_fandom_requires_character", "tag", type_="check") - op.execute("DROP INDEX uq_tag_name_kind_fandom") - op.add_column("tag", sa.Column("namespace", sa.String(length=64), nullable=True)) - op.create_index("ix_tag_namespace", "tag", ["namespace"]) - op.create_index("ix_tag_name", "tag", ["name"], unique=False) - op.create_unique_constraint("uq_tag_name", "tag", ["name"]) - op.drop_constraint("fk_tag_fandom_id_tag", "tag", type_="foreignkey") - op.drop_column("tag", "fandom_id") - op.drop_column("tag", "kind") - sa.Enum(name="tag_kind").drop(op.get_bind(), checkfirst=True) diff --git a/alembic/versions/0003_fc2b_ml_pipeline.py b/alembic/versions/0003_fc2b_ml_pipeline.py deleted file mode 100644 index 584bffe..0000000 --- a/alembic/versions/0003_fc2b_ml_pipeline.py +++ /dev/null @@ -1,172 +0,0 @@ -"""fc2b: ML pipeline — allowlist, aliases, centroids, ml_settings - -Revision ID: 0003 -Revises: 0002 -Create Date: 2026-05-15 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from pgvector.sqlalchemy import Vector - -revision: str = "0003" -down_revision: Union[str, None] = "0002" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # 3.1 rename wd14_* -> tagger_* - op.alter_column("image_record", "wd14_predictions", new_column_name="tagger_predictions") - op.alter_column( - "image_record", "wd14_model_version", new_column_name="tagger_model_version" - ) - - # 3.2 tag_allowlist - op.create_table( - "tag_allowlist", - sa.Column("tag_id", sa.Integer(), nullable=False), - sa.Column( - "min_confidence", sa.Float(), nullable=False, server_default="0.95" - ), - sa.Column( - "added_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["tag_id"], ["tag.id"], name="fk_tag_allowlist_tag_id_tag", - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("tag_id", name="pk_tag_allowlist"), - sa.CheckConstraint( - "min_confidence > 0 AND min_confidence <= 1", - name="ck_tag_allowlist_confidence_range", - ), - ) - - # 3.3 tag_suggestion_rejection - op.create_table( - "tag_suggestion_rejection", - sa.Column("image_record_id", sa.Integer(), nullable=False), - sa.Column("tag_id", sa.Integer(), nullable=False), - sa.Column( - "rejected_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["image_record_id"], ["image_record.id"], - name="fk_tsr_image_record_id_image_record", ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["tag_id"], ["tag.id"], name="fk_tsr_tag_id_tag", ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint( - "image_record_id", "tag_id", name="pk_tag_suggestion_rejection" - ), - ) - op.create_index( - "ix_tag_suggestion_rejection_tag", "tag_suggestion_rejection", ["tag_id"] - ) - - # 3.4 tag_alias - op.create_table( - "tag_alias", - sa.Column("alias_string", sa.String(length=255), nullable=False), - sa.Column("alias_category", sa.String(length=32), nullable=False), - sa.Column("canonical_tag_id", sa.Integer(), nullable=False), - sa.Column( - "created_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["canonical_tag_id"], ["tag.id"], - name="fk_tag_alias_canonical_tag_id_tag", ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint( - "alias_string", "alias_category", name="pk_tag_alias" - ), - ) - op.create_index("ix_tag_alias_canonical", "tag_alias", ["canonical_tag_id"]) - - # 3.5 tag_reference_embedding (centroids) - op.create_table( - "tag_reference_embedding", - sa.Column("tag_id", sa.Integer(), nullable=False), - sa.Column("embedding", Vector(1152), nullable=False), - sa.Column("reference_count", sa.Integer(), nullable=False), - sa.Column("model_version", sa.String(length=128), nullable=False), - sa.Column( - "updated_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["tag_id"], ["tag.id"], - name="fk_tag_reference_embedding_tag_id_tag", ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint("tag_id", name="pk_tag_reference_embedding"), - ) - - # 3.6 ml_settings singleton - op.create_table( - "ml_settings", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column( - "suggestion_threshold_artist", sa.Float(), nullable=False, - server_default="0.30", - ), - sa.Column( - "suggestion_threshold_character", sa.Float(), nullable=False, - server_default="0.50", - ), - sa.Column( - "suggestion_threshold_copyright", sa.Float(), nullable=False, - server_default="0.50", - ), - sa.Column( - "suggestion_threshold_general", sa.Float(), nullable=False, - server_default="0.95", - ), - sa.Column( - "centroid_similarity_threshold", sa.Float(), nullable=False, - server_default="0.55", - ), - sa.Column( - "min_reference_images", sa.Integer(), nullable=False, - server_default="5", - ), - sa.Column( - "tagger_model_version", sa.String(length=128), nullable=False, - server_default="camie-tagger-v2", - ), - sa.Column( - "embedder_model_version", sa.String(length=128), nullable=False, - server_default="siglip-so400m-patch14-384", - ), - sa.Column( - "updated_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.PrimaryKeyConstraint("id", name="pk_ml_settings"), - sa.CheckConstraint("id = 1", name="ck_ml_settings_singleton"), - ) - op.execute("INSERT INTO ml_settings (id) VALUES (1)") - - -def downgrade() -> None: - op.drop_table("ml_settings") - op.drop_table("tag_reference_embedding") - op.drop_index("ix_tag_alias_canonical", table_name="tag_alias") - op.drop_table("tag_alias") - op.drop_index( - "ix_tag_suggestion_rejection_tag", table_name="tag_suggestion_rejection" - ) - op.drop_table("tag_suggestion_rejection") - op.drop_table("tag_allowlist") - op.alter_column( - "image_record", "tagger_model_version", new_column_name="wd14_model_version" - ) - op.alter_column( - "image_record", "tagger_predictions", new_column_name="wd14_predictions" - ) diff --git a/alembic/versions/0004_fc2c_i_tsm_system_rows.py b/alembic/versions/0004_fc2c_i_tsm_system_rows.py deleted file mode 100644 index e8bd920..0000000 --- a/alembic/versions/0004_fc2c_i_tsm_system_rows.py +++ /dev/null @@ -1,23 +0,0 @@ -"""fc2c-i: enable tsm_system_rows for scalable random sampling - -Revision ID: 0004 -Revises: 0003 -Create Date: 2026-05-15 - -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0004" -down_revision: Union[str, None] = "0003" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows") - - -def downgrade() -> None: - op.execute("DROP EXTENSION IF EXISTS tsm_system_rows") diff --git a/alembic/versions/0005_fc2c_iii_a_series_page.py b/alembic/versions/0005_fc2c_iii_a_series_page.py deleted file mode 100644 index ffe397e..0000000 --- a/alembic/versions/0005_fc2c_iii_a_series_page.py +++ /dev/null @@ -1,50 +0,0 @@ -"""fc2c-iii-a: series_page ordered membership - -Revision ID: 0005 -Revises: 0004 -Create Date: 2026-05-16 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0005" -down_revision: Union[str, None] = "0004" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "series_page", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("series_tag_id", sa.Integer(), nullable=False), - sa.Column("image_id", sa.Integer(), nullable=False), - sa.Column("page_number", sa.Integer(), nullable=False), - sa.Column( - "created_at", sa.DateTime(timezone=True), - nullable=False, server_default=sa.func.now(), - ), - sa.Column( - "updated_at", sa.DateTime(timezone=True), - nullable=False, server_default=sa.func.now(), - ), - sa.ForeignKeyConstraint( - ["series_tag_id"], ["tag.id"], ondelete="CASCADE" - ), - sa.ForeignKeyConstraint( - ["image_id"], ["image_record.id"], ondelete="CASCADE" - ), - sa.PrimaryKeyConstraint("id"), - sa.UniqueConstraint("image_id", name="uq_series_page_image"), - ) - op.create_index( - "ix_series_page_series_tag_id", "series_page", ["series_tag_id"] - ) - - -def downgrade() -> None: - op.drop_index("ix_series_page_series_tag_id", table_name="series_page") - op.drop_table("series_page") diff --git a/alembic/versions/0006_fc2d_phash_threshold.py b/alembic/versions/0006_fc2d_phash_threshold.py deleted file mode 100644 index 895ed46..0000000 --- a/alembic/versions/0006_fc2d_phash_threshold.py +++ /dev/null @@ -1,30 +0,0 @@ -"""fc2d: import_settings.phash_threshold - -Revision ID: 0006 -Revises: 0005 -Create Date: 2026-05-17 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0006" -down_revision: Union[str, None] = "0005" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "import_settings", - sa.Column( - "phash_threshold", sa.Integer(), - nullable=False, server_default="10", - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "phash_threshold") diff --git a/alembic/versions/0007_fc2d_post_metadata_fields.py b/alembic/versions/0007_fc2d_post_metadata_fields.py deleted file mode 100644 index 24e8ba9..0000000 --- a/alembic/versions/0007_fc2d_post_metadata_fields.py +++ /dev/null @@ -1,31 +0,0 @@ -"""fc2d-iv: post.description + post.attachment_count - -Revision ID: 0007 -Revises: 0006 -Create Date: 2026-05-18 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0007" -down_revision: Union[str, None] = "0006" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "post", sa.Column("description", sa.Text(), nullable=True) - ) - op.add_column( - "post", - sa.Column("attachment_count", sa.Integer(), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("post", "attachment_count") - op.drop_column("post", "description") diff --git a/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py b/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py deleted file mode 100644 index 4019e2d..0000000 --- a/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py +++ /dev/null @@ -1,52 +0,0 @@ -"""fc2d-vii-c: image_record.artist_id + backfill + drop artist tags - -Revision ID: 0008 -Revises: 0007 -Create Date: 2026-05-18 - -Internal forward-correctness migration (the big legacy-import migration -stays deferred). downgrade() does NOT recreate deleted artist tags; -downgrade is dev-only and the data is reconstructable by re-import. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -from backend.app.utils.artist_backfill import ( - BACKFILL_PRIMARY_SQL, - BACKFILL_PROVENANCE_SQL, - BACKFILL_TAG_SQL, - DELETE_ARTIST_TAGS_SQL, -) - -revision: str = "0008" -down_revision: Union[str, None] = "0007" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "image_record", - sa.Column("artist_id", sa.Integer(), nullable=True), - ) - op.create_foreign_key( - "fk_image_record_artist_id", "image_record", "artist", - ["artist_id"], ["id"], ondelete="SET NULL", - ) - op.create_index( - "ix_image_record_artist_id", "image_record", ["artist_id"], - ) - op.execute(BACKFILL_PRIMARY_SQL) - op.execute(BACKFILL_PROVENANCE_SQL) - op.execute(BACKFILL_TAG_SQL) - op.execute(DELETE_ARTIST_TAGS_SQL) - - -def downgrade() -> None: - op.drop_index("ix_image_record_artist_id", table_name="image_record") - op.drop_constraint( - "fk_image_record_artist_id", "image_record", type_="foreignkey" - ) - op.drop_column("image_record", "artist_id") diff --git a/alembic/versions/0009_fc2d_iii_post_attachment.py b/alembic/versions/0009_fc2d_iii_post_attachment.py deleted file mode 100644 index 4820fa4..0000000 --- a/alembic/versions/0009_fc2d_iii_post_attachment.py +++ /dev/null @@ -1,68 +0,0 @@ -"""fc2d-iii: post_attachment + import_batch.attachments - -Revision ID: 0009 -Revises: 0008 -Create Date: 2026-05-19 - -Internal forward-correctness migration (big legacy-import migration -stays deferred). No backfill — no attachments exist yet. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0009" -down_revision: Union[str, None] = "0008" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "post_attachment", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "post_id", sa.Integer(), - sa.ForeignKey("post.id", ondelete="SET NULL"), nullable=True, - ), - sa.Column( - "artist_id", sa.Integer(), - sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, - ), - sa.Column("sha256", sa.String(64), nullable=False), - sa.Column("path", sa.Text(), nullable=False), - sa.Column("original_filename", sa.Text(), nullable=False), - sa.Column("ext", sa.String(32), nullable=False), - sa.Column("mime", sa.String(128), nullable=True), - sa.Column("size_bytes", sa.BigInteger(), nullable=False), - sa.Column( - "captured_at", sa.DateTime(timezone=True), - server_default=sa.func.now(), nullable=False, - ), - ) - op.create_index( - "ix_post_attachment_sha256", "post_attachment", ["sha256"], - unique=True, - ) - op.create_index( - "ix_post_attachment_post_id", "post_attachment", ["post_id"], - ) - op.create_index( - "ix_post_attachment_artist_id", "post_attachment", ["artist_id"], - ) - op.add_column( - "import_batch", - sa.Column( - "attachments", sa.Integer(), nullable=False, - server_default="0", - ), - ) - - -def downgrade() -> None: - op.drop_column("import_batch", "attachments") - op.drop_index("ix_post_attachment_artist_id", table_name="post_attachment") - op.drop_index("ix_post_attachment_post_id", table_name="post_attachment") - op.drop_index("ix_post_attachment_sha256", table_name="post_attachment") - op.drop_table("post_attachment") diff --git a/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py b/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py deleted file mode 100644 index 10f502b..0000000 --- a/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py +++ /dev/null @@ -1,32 +0,0 @@ -"""fc3a: unique(source.artist_id, source.platform, source.url) - -Revision ID: 0010 -Revises: 0009 -Create Date: 2026-05-20 - -Enforces FC-3a's dedup invariant at the DB level. No backfill — no -existing rows are expected to collide; if they do the migration will -fail loudly (intended). -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0010" -down_revision: Union[str, None] = "0009" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_unique_constraint( - "uq_source_artist_platform_url", - "source", - ["artist_id", "platform", "url"], - ) - - -def downgrade() -> None: - op.drop_constraint( - "uq_source_artist_platform_url", "source", type_="unique" - ) diff --git a/alembic/versions/0011_fc3b_credential_schema_alignment.py b/alembic/versions/0011_fc3b_credential_schema_alignment.py deleted file mode 100644 index 00a31a5..0000000 --- a/alembic/versions/0011_fc3b_credential_schema_alignment.py +++ /dev/null @@ -1,41 +0,0 @@ -"""fc3b: rename credential.kind -> credential_type, drop status, add last_verified - -Revision ID: 0011 -Revises: 0010 -Create Date: 2026-05-20 - -Aligns the credential table with the GallerySubscriber wire-field names -so the existing browser extension can POST to FC unmodified. Greenfield — -no rows exist in production yet, so no data preservation logic is -needed; the rename uses ALTER COLUMN rather than copy-then-drop. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0011" -down_revision: Union[str, None] = "0010" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.alter_column("credential", "kind", new_column_name="credential_type") - op.drop_column("credential", "status") - op.add_column( - "credential", - sa.Column("last_verified", sa.DateTime(timezone=True), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("credential", "last_verified") - op.add_column( - "credential", - sa.Column( - "status", sa.String(length=32), nullable=False, - server_default="active", - ), - ) - op.alter_column("credential", "credential_type", new_column_name="kind") diff --git a/alembic/versions/0012_fc3b_app_setting.py b/alembic/versions/0012_fc3b_app_setting.py deleted file mode 100644 index 42c06eb..0000000 --- a/alembic/versions/0012_fc3b_app_setting.py +++ /dev/null @@ -1,36 +0,0 @@ -"""fc3b: app_setting key/value table - -Revision ID: 0012 -Revises: 0011 -Create Date: 2026-05-20 - -A simple key/value table for small app settings that don't fit -ImportSettings. Initially seeds only `extension_api_key` (done in -create_app on first boot — not in the migration, to keep it -deterministic and independent of randomness). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0012" -down_revision: Union[str, None] = "0011" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "app_setting", - sa.Column("key", sa.String(length=64), primary_key=True), - sa.Column("value", sa.Text(), nullable=False), - sa.Column( - "updated_at", sa.DateTime(timezone=True), - nullable=False, server_default=sa.func.now(), - ), - ) - - -def downgrade() -> None: - op.drop_table("app_setting") diff --git a/alembic/versions/0013_fc3c_download_event_metadata.py b/alembic/versions/0013_fc3c_download_event_metadata.py deleted file mode 100644 index c88b3f9..0000000 --- a/alembic/versions/0013_fc3c_download_event_metadata.py +++ /dev/null @@ -1,52 +0,0 @@ -"""fc3c: download_event.metadata + import_settings downloader fields - -Revision ID: 0013 -Revises: 0012 -Create Date: 2026-05-20 - -Additive only. download_event.metadata is the rich JSONB blob FC-3c -populates per run (run_stats, stdout/stderr, quarantined paths, import -summary). import_settings gains two operator-tunable downloader knobs: -download_rate_limit_seconds (gallery-dl extractor.sleep) and -download_validate_files (toggle the magic-byte validator). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -revision: str = "0013" -down_revision: Union[str, None] = "0012" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "download_event", - sa.Column( - "metadata", postgresql.JSONB, - nullable=False, server_default=sa.text("'{}'::jsonb"), - ), - ) - op.add_column( - "import_settings", - sa.Column( - "download_rate_limit_seconds", sa.Float(), - nullable=False, server_default="3.0", - ), - ) - op.add_column( - "import_settings", - sa.Column( - "download_validate_files", sa.Boolean(), - nullable=False, server_default=sa.true(), - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "download_validate_files") - op.drop_column("import_settings", "download_rate_limit_seconds") - op.drop_column("download_event", "metadata") diff --git a/alembic/versions/0014_fc3d_scheduling.py b/alembic/versions/0014_fc3d_scheduling.py deleted file mode 100644 index 955e956..0000000 --- a/alembic/versions/0014_fc3d_scheduling.py +++ /dev/null @@ -1,58 +0,0 @@ -"""fc3d: scheduling + source health columns - -Revision ID: 0014 -Revises: 0013 -Create Date: 2026-05-21 - -Additive only. source.consecutive_failures (default 0, DownloadService -finalize hook owns the writes). import_settings gains the three -scheduling knobs (global default interval, event retention, failure -warning threshold). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0014" -down_revision: Union[str, None] = "0013" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "source", - sa.Column( - "consecutive_failures", sa.Integer(), - nullable=False, server_default="0", - ), - ) - op.add_column( - "import_settings", - sa.Column( - "download_schedule_default_seconds", sa.Integer(), - nullable=False, server_default="28800", - ), - ) - op.add_column( - "import_settings", - sa.Column( - "download_event_retention_days", sa.Integer(), - nullable=False, server_default="90", - ), - ) - op.add_column( - "import_settings", - sa.Column( - "download_failure_warning_threshold", sa.Integer(), - nullable=False, server_default="5", - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "download_failure_warning_threshold") - op.drop_column("import_settings", "download_event_retention_days") - op.drop_column("import_settings", "download_schedule_default_seconds") - op.drop_column("source", "consecutive_failures") diff --git a/alembic/versions/0015_fc5_migration_run.py b/alembic/versions/0015_fc5_migration_run.py deleted file mode 100644 index 89d7d89..0000000 --- a/alembic/versions/0015_fc5_migration_run.py +++ /dev/null @@ -1,51 +0,0 @@ -"""fc5: migration_run table - -Revision ID: 0015 -Revises: 0014 -Create Date: 2026-05-22 - -Additive only. New table tracks each invocation of the FC-5 migration -tooling (backup, gs, ir, ml_queue, verify, rollback). kind/status are -plain String(32) — values validated at the API layer per the spec, not -a Postgres ENUM (so adding kinds later doesn't need a schema migration). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -revision: str = "0015" -down_revision: Union[str, None] = "0014" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "migration_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("kind", sa.String(32), nullable=False, index=True), - sa.Column("status", sa.String(32), nullable=False, index=True), - sa.Column( - "dry_run", sa.Boolean(), nullable=False, server_default=sa.false(), - ), - sa.Column( - "started_at", sa.DateTime(timezone=True), - nullable=False, server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column( - "counts", postgresql.JSONB, - nullable=False, server_default=sa.text("'{}'::jsonb"), - ), - sa.Column("error", sa.Text(), nullable=True), - sa.Column( - "metadata", postgresql.JSONB, - nullable=False, server_default=sa.text("'{}'::jsonb"), - ), - ) - - -def downgrade() -> None: - op.drop_table("migration_run") diff --git a/alembic/versions/0016_fc3i_task_run.py b/alembic/versions/0016_fc3i_task_run.py deleted file mode 100644 index 678b1ee..0000000 --- a/alembic/versions/0016_fc3i_task_run.py +++ /dev/null @@ -1,86 +0,0 @@ -"""fc3i: task_run table - -Revision ID: 0016 -Revises: 0015 -Create Date: 2026-05-24 - -Additive only. New table records every Celery task attempt via signal -handlers (backend.app.celery_signals). Status is plain String(16) not -Postgres ENUM (per feedback_check_existing_enums: ENUM columns hard- -fail at INSERT, String columns extend cleanly). - -Composite indexes anticipate the three dashboard panes: -- (queue, started_at desc) — per-lane recent activity -- (status, started_at desc) — recent failures pane -- (task_name, started_at desc) — drill-down by task - -Indexed columns get individual indexes via `index=True` on the model; -the composites below cover the multi-column lookups. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0016" -down_revision: Union[str, None] = "0015" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "task_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("celery_task_id", sa.String(length=64), nullable=False), - sa.Column("queue", sa.String(length=32), nullable=False), - sa.Column("task_name", sa.String(length=128), nullable=False), - sa.Column("target_id", sa.Integer(), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("duration_ms", sa.Integer(), nullable=True), - sa.Column( - "status", sa.String(length=16), nullable=False, - server_default="running", - ), - sa.Column("error_type", sa.String(length=128), nullable=True), - sa.Column("error_message", sa.Text(), nullable=True), - sa.Column("retry_count", sa.Integer(), nullable=True), - sa.Column("worker_hostname", sa.String(length=128), nullable=True), - sa.Column("args_summary", sa.String(length=255), nullable=True), - ) - - # Single-column indexes (matches Mapped[...].index=True on model). - op.create_index("ix_task_run_celery_task_id", "task_run", ["celery_task_id"]) - op.create_index("ix_task_run_queue", "task_run", ["queue"]) - op.create_index("ix_task_run_task_name", "task_run", ["task_name"]) - op.create_index("ix_task_run_started_at", "task_run", ["started_at"]) - op.create_index("ix_task_run_finished_at", "task_run", ["finished_at"]) - op.create_index("ix_task_run_status", "task_run", ["status"]) - - # Composite indexes for dashboard query patterns. - op.create_index( - "ix_task_run_queue_started", - "task_run", ["queue", sa.text("started_at DESC")], - ) - op.create_index( - "ix_task_run_status_started", - "task_run", ["status", sa.text("started_at DESC")], - ) - op.create_index( - "ix_task_run_name_started", - "task_run", ["task_name", sa.text("started_at DESC")], - ) - - -def downgrade() -> None: - op.drop_index("ix_task_run_name_started", table_name="task_run") - op.drop_index("ix_task_run_status_started", table_name="task_run") - op.drop_index("ix_task_run_queue_started", table_name="task_run") - op.drop_index("ix_task_run_status", table_name="task_run") - op.drop_index("ix_task_run_finished_at", table_name="task_run") - op.drop_index("ix_task_run_started_at", table_name="task_run") - op.drop_index("ix_task_run_task_name", table_name="task_run") - op.drop_index("ix_task_run_queue", table_name="task_run") - op.drop_index("ix_task_run_celery_task_id", table_name="task_run") - op.drop_table("task_run") diff --git a/alembic/versions/0017_fc3h_backup_run.py b/alembic/versions/0017_fc3h_backup_run.py deleted file mode 100644 index 5b6a839..0000000 --- a/alembic/versions/0017_fc3h_backup_run.py +++ /dev/null @@ -1,82 +0,0 @@ -"""fc3h: backup_run table - -Revision ID: 0017 -Revises: 0016 -Create Date: 2026-05-24 - -Additive. New table records every backup/restore attempt with artifact -metadata. Lifecycle tracking lives in task_run from FC-3i; this is -artifact-only (paths, sizes, tag, restore lineage). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0017" -down_revision: Union[str, None] = "0016" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "backup_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("kind", sa.String(length=16), nullable=False), - sa.Column( - "status", sa.String(length=16), nullable=False, - server_default="pending", - ), - sa.Column("tag", sa.String(length=64), nullable=True), - sa.Column("triggered_by", sa.String(length=32), nullable=False), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("sql_path", sa.Text(), nullable=True), - sa.Column("tar_path", sa.Text(), nullable=True), - sa.Column("size_bytes", sa.BigInteger(), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column( - "manifest", sa.JSON(), nullable=False, server_default="{}", - ), - sa.Column( - "restored_from_id", sa.Integer(), - sa.ForeignKey("backup_run.id", ondelete="SET NULL"), - nullable=True, - ), - ) - - # Single-column indexes (matches Mapped[...].index=True). - op.create_index("ix_backup_run_kind", "backup_run", ["kind"]) - op.create_index("ix_backup_run_status", "backup_run", ["status"]) - op.create_index("ix_backup_run_tag", "backup_run", ["tag"]) - op.create_index("ix_backup_run_started_at", "backup_run", ["started_at"]) - op.create_index("ix_backup_run_finished_at", "backup_run", ["finished_at"]) - - # Composite indexes for dashboard query patterns. - op.create_index( - "ix_backup_run_kind_started", - "backup_run", ["kind", sa.text("started_at DESC")], - ) - op.create_index( - "ix_backup_run_status_finished", - "backup_run", ["status", sa.text("finished_at DESC")], - ) - # Partial index: only tagged rows participate in retention-exempt query. - op.create_index( - "ix_backup_run_tag_partial", - "backup_run", ["tag"], - postgresql_where=sa.text("tag IS NOT NULL"), - ) - - -def downgrade() -> None: - op.drop_index("ix_backup_run_tag_partial", table_name="backup_run") - op.drop_index("ix_backup_run_status_finished", table_name="backup_run") - op.drop_index("ix_backup_run_kind_started", table_name="backup_run") - op.drop_index("ix_backup_run_finished_at", table_name="backup_run") - op.drop_index("ix_backup_run_started_at", table_name="backup_run") - op.drop_index("ix_backup_run_tag", table_name="backup_run") - op.drop_index("ix_backup_run_status", table_name="backup_run") - op.drop_index("ix_backup_run_kind", table_name="backup_run") - op.drop_table("backup_run") diff --git a/alembic/versions/0018_fc3h_backup_settings.py b/alembic/versions/0018_fc3h_backup_settings.py deleted file mode 100644 index 517c8f1..0000000 --- a/alembic/versions/0018_fc3h_backup_settings.py +++ /dev/null @@ -1,62 +0,0 @@ -"""fc3h: backup_* knobs on import_settings - -Revision ID: 0018 -Revises: 0017 -Create Date: 2026-05-24 - -Adds four columns to the singleton import_settings row: - - backup_db_nightly_enabled (default False — opt-in) - - backup_db_nightly_hour_utc (default 3) - - backup_db_keep_last_n (default 14) - - backup_images_keep_last_n (default 3) - -server_default ensures the singleton row is backfilled in place -without an UPDATE statement. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0018" -down_revision: Union[str, None] = "0017" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "import_settings", - sa.Column( - "backup_db_nightly_enabled", sa.Boolean(), - nullable=False, server_default=sa.false(), - ), - ) - op.add_column( - "import_settings", - sa.Column( - "backup_db_nightly_hour_utc", sa.Integer(), - nullable=False, server_default="3", - ), - ) - op.add_column( - "import_settings", - sa.Column( - "backup_db_keep_last_n", sa.Integer(), - nullable=False, server_default="14", - ), - ) - op.add_column( - "import_settings", - sa.Column( - "backup_images_keep_last_n", sa.Integer(), - nullable=False, server_default="3", - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "backup_images_keep_last_n") - op.drop_column("import_settings", "backup_db_keep_last_n") - op.drop_column("import_settings", "backup_db_nightly_hour_utc") - op.drop_column("import_settings", "backup_db_nightly_enabled") diff --git a/alembic/versions/0019_import_batch_refreshed.py b/alembic/versions/0019_import_batch_refreshed.py deleted file mode 100644 index 1770daa..0000000 --- a/alembic/versions/0019_import_batch_refreshed.py +++ /dev/null @@ -1,38 +0,0 @@ -"""import_batch.refreshed counter for deep-scan sidecar re-application - -Revision ID: 0019 -Revises: 0018 -Create Date: 2026-05-25 - -Adds a `refreshed` counter to `import_batch`, mirroring the existing -`imported`/`skipped`/`failed`/`attachments` columns. Deep scan now -re-applies sidecar metadata to already-imported files (the IR feature -that didn't make the FC port the first time); a "refreshed" outcome -increments this counter so the UI can surface "X new, Y refreshed" -instead of the misleading "Scan complete — no new files" message. - -server_default=0 backfills existing rows in place — no UPDATE needed. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0019" -down_revision: Union[str, None] = "0018" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "import_batch", - sa.Column( - "refreshed", sa.Integer(), - nullable=False, server_default=sa.text("0"), - ), - ) - - -def downgrade() -> None: - op.drop_column("import_batch", "refreshed") diff --git a/alembic/versions/0020_library_audit_run.py b/alembic/versions/0020_library_audit_run.py deleted file mode 100644 index 07a8b87..0000000 --- a/alembic/versions/0020_library_audit_run.py +++ /dev/null @@ -1,65 +0,0 @@ -"""fc-cleanup: library_audit_run table for async transparency/single_color audits - -Revision ID: 0020 -Revises: 0019 -Create Date: 2026-05-26 - -The table backs the async audit lifecycle: rule + params snapshot, status -state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and -the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs -per row by the scan task (oversize = rule too aggressive, operator narrows -before re-running). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -revision: str = "0020" -down_revision: Union[str, None] = "0019" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "library_audit_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("rule", sa.String(32), nullable=False), - sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column( - "status", sa.String(16), - nullable=False, server_default="running", - ), - sa.Column( - "started_at", sa.DateTime(timezone=True), - nullable=False, server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column( - "scanned_count", sa.Integer(), - nullable=False, server_default="0", - ), - sa.Column( - "matched_count", sa.Integer(), - nullable=False, server_default="0", - ), - sa.Column( - "matched_ids", postgresql.JSONB(astext_type=sa.Text()), - nullable=False, server_default=sa.text("'[]'::jsonb"), - ), - sa.Column("error", sa.Text(), nullable=True), - ) - op.create_index( - "ix_library_audit_run_rule", "library_audit_run", ["rule"], - ) - op.create_index( - "ix_library_audit_run_status", "library_audit_run", ["status"], - ) - - -def downgrade() -> None: - op.drop_index("ix_library_audit_run_status", table_name="library_audit_run") - op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run") - op.drop_table("library_audit_run") diff --git a/alembic/versions/0021_image_provenance_unique.py b/alembic/versions/0021_image_provenance_unique.py deleted file mode 100644 index b9be941..0000000 --- a/alembic/versions/0021_image_provenance_unique.py +++ /dev/null @@ -1,54 +0,0 @@ -"""provenance-race: dedupe + UNIQUE(image_record_id, post_id) on image_provenance - -Revision ID: 0021 -Revises: 0020 -Create Date: 2026-05-26 - -Closes the race in Importer._apply_sidecar's existence-check + INSERT pattern. -Two workers writing for the same (image, post) pair both saw no existing row -and both inserted, leaving duplicates that then broke .scalar_one_or_none() -on every subsequent deep-scan rederive against those images -(MultipleResultsFound). Most plausibly seeded when the 5-min recovery sweep -re-enqueued a still-running long-import task and the second worker collided -with the first inside _apply_sidecar. - -Migration steps: - 1. DELETE all but min(id) per (image_record_id, post_id) pair. Operator's - DB had 2 affected pairs at write-time; harmless no-op if zero. - 2. Add UNIQUE constraint so the importer's new savepoint+IntegrityError - recovery path can trip on collision and re-select, mirroring - uq_source_artist_platform_url and uq_post_source_external_id. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0021" -down_revision: Union[str, None] = "0020" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute( - """ - DELETE FROM image_provenance ip1 - USING image_provenance ip2 - WHERE ip1.image_record_id = ip2.image_record_id - AND ip1.post_id = ip2.post_id - AND ip1.id > ip2.id - """ - ) - op.create_unique_constraint( - "uq_image_provenance_image_post", - "image_provenance", - ["image_record_id", "post_id"], - ) - - -def downgrade() -> None: - op.drop_constraint( - "uq_image_provenance_image_post", - "image_provenance", - type_="unique", - ) diff --git a/alembic/versions/0022_source_per_artist_platform.py b/alembic/versions/0022_source_per_artist_platform.py deleted file mode 100644 index d3e75dd..0000000 --- a/alembic/versions/0022_source_per_artist_platform.py +++ /dev/null @@ -1,223 +0,0 @@ -"""source-collapse: one Source per (artist, platform) — consolidate junk per-post Sources - -Revision ID: 0022 -Revises: 0021 -Create Date: 2026-05-26 - -Closes the operator-flagged 2026-05-26 issue where the filesystem importer -called _find_or_create_source(url=sd.post_url), creating one Source row per -imported post URL. Operator's Atole artist had 406 Source rows where there -should have been 1 (the /cw/Atole subscription Source). - -Source represents a subscription feed (one per artist+platform — the -gallery-dl URL polled by the FC-3 downloader). Posts hang off it. The -filesystem importer was misusing Source as a per-post key. - -Migration steps per (artist_id, platform) group with >1 Source: - 1. Pick canonical — prefer a URL NOT matching '/posts/$' (real - campaign URL like /cw/Atole); else min(id). - 2. PRE-merge any Posts under non-canonical sources whose - external_post_id ALREADY exists under the canonical source. (Same - gallery-dl post imported via two different sidecar paths can plant - two Post rows with identical external_post_id under different - Sources for the same artist.) Repoint ImageProvenance + - ImageRecord.primary_post_id to the canonical-side Post, dedupe - ImageProvenance against alembic 0021's uq, then delete the - non-canonical-side Post. This MUST happen before step 3 — Postgres - fires uq_post_source_external_id row-by-row during the bulk UPDATE - and the merge-after-reparent ordering 500s on first collision - (operator-hit during v26.05.26.1 deploy, 2026-05-26). - 3. Reparent remaining Posts onto canonical (no collisions possible now). - 4. Reparent ImageProvenance.source_id off the non-canonical sources. - 5. Delete the orphan Source rows. - 6. If the canonical Source's URL still looks like a per-post URL (no - campaign URL existed among candidates), rewrite it to - 'sidecar::' so the artist detail page shows - something readable. -""" -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import text - -revision: str = "0022" -down_revision: Union[str, None] = "0021" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_POST_URL_RE = r"/posts/[^/]+$" - - -def upgrade() -> None: - conn = op.get_bind() - - # Find (artist_id, platform) groups with > 1 Source row. - groups = conn.execute(text(""" - SELECT artist_id, platform - FROM source - GROUP BY artist_id, platform - HAVING COUNT(*) > 1 - """)).fetchall() - - for artist_id, platform in groups: - rows = conn.execute( - text(""" - SELECT id, url FROM source - WHERE artist_id = :a AND platform = :p - ORDER BY id ASC - """), - {"a": artist_id, "p": platform}, - ).fetchall() - - # Canonical: first row whose URL doesn't look like a per-post URL; - # else min(id). - canonical_id = None - for sid, url in rows: - if not _matches_post_url(url): - canonical_id = sid - break - if canonical_id is None: - canonical_id = rows[0][0] - - other_ids = [sid for sid, _ in rows if sid != canonical_id] - if not other_ids: - continue - - # STEP 2: PRE-merge ALL Posts with duplicate external_post_id - # across the entire (canonical + others) group, BEFORE the bulk - # reparent. Two cases must both be handled: - # (A) canonical has Post X with epid=N; an "other" source has - # Post Y with epid=N → after bulk UPDATE, (canonical, N) - # collides with itself. - # (B) two different "other" sources each have a Post with - # epid=N; canonical has none → after bulk UPDATE, both - # are repointed to (canonical, N) and the second collides. - # The earlier version of this migration only handled (A); the - # operator's deploy 2026-05-26 tripped (B) at line 139. - # Fix: group ALL Posts in the (artist, platform) by epid; for - # any group with count>1, pick the keep (prefer one already - # under canonical; else lowest id) and merge the rest into it. - all_posts = conn.execute( - text(""" - SELECT external_post_id, id, source_id - FROM post - WHERE source_id = :canonical OR source_id = ANY(:others) - ORDER BY external_post_id, id - """), - {"canonical": canonical_id, "others": other_ids}, - ).fetchall() - by_epid: dict = {} - for epid, post_id, src_id in all_posts: - by_epid.setdefault(epid, []).append((post_id, src_id)) - for _epid, posts in by_epid.items(): - if len(posts) <= 1: - continue - # Prefer a Post already under canonical as the keep. - canonical_posts = [p for p in posts if p[1] == canonical_id] - if canonical_posts: - keep_id = canonical_posts[0][0] - else: - keep_id = posts[0][0] # already sorted by id ASC - drop_ids = [p[0] for p in posts if p[0] != keep_id] - for drop_id in drop_ids: - # Pre-delete image_provenance rows under drop_ whose - # image_record_id ALREADY has a provenance under keep — - # the UPDATE below would otherwise repoint them and - # trip uq_image_provenance_image_post (alembic 0021) - # row-by-row before any after-the-fact dedupe could - # run. Operator's v26.05.26.3 deploy 2026-05-26 tripped - # this at line 123. - conn.execute( - text(""" - DELETE FROM image_provenance - WHERE post_id = :drop_ - AND image_record_id IN ( - SELECT image_record_id FROM image_provenance - WHERE post_id = :keep - ) - """), - {"keep": keep_id, "drop_": drop_id}, - ) - # Now safe to repoint the survivors. - conn.execute( - text(""" - UPDATE image_provenance SET post_id = :keep - WHERE post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text(""" - UPDATE image_record SET primary_post_id = :keep - WHERE primary_post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text("DELETE FROM post WHERE id = :drop_"), - {"drop_": drop_id}, - ) - - # STEP 3: Bulk reparent the remaining Posts off the other - # Sources. After step 2, no collisions on - # (canonical, external_post_id) are possible. - conn.execute( - text(""" - UPDATE post SET source_id = :canonical - WHERE source_id = ANY(:others) - """), - {"canonical": canonical_id, "others": other_ids}, - ) - - # STEP 4: Reparent ImageProvenance.source_id (denormalized FK). - # No UNIQUE on source_id; safe bulk update. - conn.execute( - text(""" - UPDATE image_provenance SET source_id = :canonical - WHERE source_id = ANY(:others) - """), - {"canonical": canonical_id, "others": other_ids}, - ) - - # STEP 5: Drop the orphan Sources. - conn.execute( - text("DELETE FROM source WHERE id = ANY(:others)"), - {"others": other_ids}, - ) - - # If the canonical's URL still looks per-post (no campaign URL - # existed among the candidates), rewrite to a synthetic anchor so - # the artist detail page renders something readable. - canonical_url = conn.execute( - text("SELECT url FROM source WHERE id = :id"), - {"id": canonical_id}, - ).scalar_one() - if _matches_post_url(canonical_url): - slug = conn.execute( - text("SELECT slug FROM artist WHERE id = :id"), - {"id": artist_id}, - ).scalar_one() - conn.execute( - text(""" - UPDATE source - SET url = :new_url, enabled = false - WHERE id = :id - """), - { - "id": canonical_id, - "new_url": f"sidecar:{platform}:{slug}", - }, - ) - - -def downgrade() -> None: - # Lossy migration — orphan Sources deleted, Posts reparented, Posts - # merged. No safe downgrade. If you need to roll back the schema - # invariant, fork from 0021 and re-run filesystem imports. - pass - - -def _matches_post_url(url: str) -> bool: - """True if url ends with /posts/ (gallery-dl-style per-post URL).""" - import re - return bool(re.search(_POST_URL_RE, url or "")) diff --git a/alembic/versions/0023_drop_meta_rating_tag_kinds.py b/alembic/versions/0023_drop_meta_rating_tag_kinds.py deleted file mode 100644 index fc65ea1..0000000 --- a/alembic/versions/0023_drop_meta_rating_tag_kinds.py +++ /dev/null @@ -1,99 +0,0 @@ -"""drop meta + rating tag kinds — operator-retired 2026-05-26 - -Revision ID: 0023 -Revises: 0022 -Create Date: 2026-05-26 - -Operator decided meta + rating aren't valid tag kinds for FC. Per-row -behavior: DELETE existing rows (operator chose "clean break" over -"convert to general"). All cascading FKs (image_tag, tag_alias, -tag_allowlist, tag_reference_embedding, tag_suggestion_rejection, -series_page) use ondelete="CASCADE" so a single DELETE on tag cleans -the related rows in one go. - -After the data cleanup, recreate the tag_kind ENUM without 'meta' / -'rating' (Postgres has no `ALTER TYPE ... DROP VALUE`; standard -rename-create-cast-drop dance). The server default 'general' is -dropped before the type swap and restored after. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0023" -down_revision: Union[str, None] = "0022" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # 1. Delete tags of the retired kinds. CASCADE handles related tables. - op.execute("DELETE FROM tag WHERE kind IN ('meta', 'rating')") - - # 2. Drop the CHECK constraint that references the enum's literal - # values. Postgres can't resolve `kind = 'character'` across the - # type swap below — the literal would bind to the new tag_kind - # but the column is on tag_kind_old, producing - # "operator does not exist: tag_kind = tag_kind_old". - # (Operator-hit during the v26.05.26.5 deploy attempt; ck was - # originally added by alembic 0002.) Recreated post-swap. - op.drop_constraint( - "ck_tag_fandom_requires_character", "tag", type_="check" - ) - - # 3. Drop the server default — ALTER COLUMN TYPE can't carry it - # across the type swap below. - op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT") - - # 4. Recreate the tag_kind enum without meta/rating. - op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old") - op.execute( - "CREATE TYPE tag_kind AS ENUM (" - "'artist', 'character', 'fandom', 'general', " - "'series', 'archive', 'post'" - ")" - ) - op.execute( - "ALTER TABLE tag " - "ALTER COLUMN kind TYPE tag_kind " - "USING kind::text::tag_kind" - ) - op.execute("DROP TYPE tag_kind_old") - - # 5. Restore the server default. - op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'") - - # 6. Restore the CHECK constraint (now bound to the new tag_kind). - op.create_check_constraint( - "ck_tag_fandom_requires_character", - "tag", - "(fandom_id IS NULL) OR (kind = 'character')", - ) - - -def downgrade() -> None: - # Add the values back to the enum so old code can boot. The deleted - # tag rows are gone permanently — no safe restore. - op.drop_constraint( - "ck_tag_fandom_requires_character", "tag", type_="check" - ) - op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT") - op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old") - op.execute( - "CREATE TYPE tag_kind AS ENUM (" - "'artist', 'character', 'fandom', 'general', " - "'series', 'archive', 'post', 'meta', 'rating'" - ")" - ) - op.execute( - "ALTER TABLE tag " - "ALTER COLUMN kind TYPE tag_kind " - "USING kind::text::tag_kind" - ) - op.execute("DROP TYPE tag_kind_old") - op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'") - op.create_check_constraint( - "ck_tag_fandom_requires_character", - "tag", - "(fandom_id IS NULL) OR (kind = 'character')", - ) diff --git a/alembic/versions/0024_backfill_post_title_from_description.py b/alembic/versions/0024_backfill_post_title_from_description.py deleted file mode 100644 index 2b1385c..0000000 --- a/alembic/versions/0024_backfill_post_title_from_description.py +++ /dev/null @@ -1,80 +0,0 @@ -"""backfill post.post_title from description first-line — 2026-05-27 - -Revision ID: 0024 -Revises: 0023 -Create Date: 2026-05-27 - -SubscribeStar gallery-dl always writes `title: ""` and embeds the leading -sentence inside `content` HTML. FC's sidecar parser was leaving -post_title NULL for every SubscribeStar post since FC-3 shipped. The -parser fix (sidecar._first_line_text fallback) now synthesizes a title -at parse time; this migration applies the same logic retroactively to -existing rows. - -Operator-flagged 2026-05-27 after inspecting -/mnt/Data/Patreon/Cheunart/subscribestar/ sidecars. - -Idempotent: only touches rows where post_title IS NULL or empty AND -description IS NOT NULL. Re-running the migration is a no-op. -""" -from __future__ import annotations - -import re -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import text - -revision: str = "0024" -down_revision: Union[str, None] = "0023" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -_TAG_RE = re.compile(r"<[^>]+>") -_WS_RE = re.compile(r"\s+") - - -def _first_line_text(body: str, limit: int = 120) -> str | None: - """Mirror of sidecar._first_line_text. Kept inline so the migration - doesn't carry a runtime import dependency from app code that may - have moved by the time the migration is replayed years from now.""" - if not body: - return None - text_ = _TAG_RE.sub(" ", body) - text_ = text_.replace("\xa0", " ") - for line in text_.splitlines(): - line = _WS_RE.sub(" ", line).strip() - if line: - if len(line) > limit: - return line[: limit - 1].rstrip() + "…" - return line - return None - - -def upgrade() -> None: - bind = op.get_bind() - rows = bind.execute( - text( - "SELECT id, description FROM post " - "WHERE (post_title IS NULL OR post_title = '') " - "AND description IS NOT NULL AND description <> ''" - ) - ).fetchall() - updated = 0 - for row in rows: - derived = _first_line_text(row.description) - if not derived: - continue - bind.execute( - text("UPDATE post SET post_title = :t WHERE id = :id"), - {"t": derived, "id": row.id}, - ) - updated += 1 - print(f"0024: backfilled post_title on {updated} row(s)") - - -def downgrade() -> None: - # No safe restore — we can't tell which post_titles were derived vs - # genuinely present. Leave the column alone on rollback. - pass diff --git a/alembic/versions/0025_fix_subscribestar_post_ids.py b/alembic/versions/0025_fix_subscribestar_post_ids.py deleted file mode 100644 index b44f430..0000000 --- a/alembic/versions/0025_fix_subscribestar_post_ids.py +++ /dev/null @@ -1,288 +0,0 @@ -"""sidecar-audit followup: correct external_post_id + post_url across all platforms - -Revision ID: 0025 -Revises: 0024 -Create Date: 2026-05-27 - -Closes the operator-flagged 2026-05-27 sidecar audit findings. Three -data-correctness bugs across non-Patreon platforms had been silently -corrupting Posts since FC-3 shipped; the parser fix (sidecar.py, same -commit) addresses new imports. This migration cleans up existing rows. - -Per-platform actions: - - subscribestar — gallery-dl wrote the per-attachment id in `id` and - the actual post id in `post_id`. FC's parser picked `id`, so every - multi-image SubscribeStar post was fragmented into N Post rows. - 1. For each SubscribeStar Post, read its sidecar (via the related - ImageRecord's on-disk path), pull `post_id`, overwrite - external_post_id and post_url. - 2. Merge groups of Posts under one source that now share an - external_post_id (fragments of the same actual post). Same - ImageProvenance pre-delete + repoint dance as alembic 0022. - - hentaifoundry — sidecars have NO `url` field; `src` is the image - URL. FC's parser stored post_url=NULL. Read each HF Post's sidecar - for `user` + `index`, derive the canonical /pictures/user// - permalink. external_post_id (= `index`) was already correct. - - discord — gallery-dl wrote the CDN attachment URL in `url`. FC's - parser stored that as post_url. Read each Discord Post's sidecar - for the server/channel/message triple, derive the proper - discord.com/channels/.../ permalink. external_post_id (= - `message_id`) was already correct. - - pixiv — pure-SQL backfill: replace any `i.pximg.net`-style URL on - Post.post_url with the derived `/artworks/` permalink. Pixiv - external_post_id (= `id`) was already correct; no sidecar IO - needed. - -Idempotent: re-running on already-corrected data is a no-op (skips -rows whose derived value matches what's already stored). - -Posts whose related ImageRecord paths don't resolve on disk (orphaned -filesystem state) are skipped with a count in the migration output — -those will be picked up by a future deep-scan. -""" -from __future__ import annotations - -import json -import re -from pathlib import Path -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import text - -revision: str = "0025" -down_revision: Union[str, None] = "0024" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -# Mirror of sidecar._NUMBERING_PREFIX. Kept inline so the migration is -# self-contained (the operator's banked rule: -# reference_postgres_enum_swap_drop_checks.md says migrations shouldn't -# import from runtime app code). -_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$") - - -def _find_sidecar(media_path: Path) -> Path | None: - """gallery-dl writes the sidecar under the unprefixed stem - (`HOLLOW-ICHIGO.json`) while the media file gets a NN_ ordering - prefix (`01_HOLLOW-ICHIGO.png`). Try in order: - 1. .json next to the media - 2. .json next to the media (full-name variant) - 3. strip the NN_ prefix from the stem, then .json - """ - if not media_path: - return None - cand = media_path.with_suffix(".json") - if cand.is_file(): - return cand - cand = media_path.parent / f"{media_path.name}.json" - if cand.is_file(): - return cand - m = _NUMBERING_PREFIX.match(media_path.stem) - if m: - cand = media_path.parent / f"{m.group(1)}.json" - if cand.is_file(): - return cand - return None - - -def _str_id(v) -> str | None: - """str() a JSON scalar id; reject bool (JSON booleans are ints in - Python's eyes but they aren't valid sidecar ids).""" - if isinstance(v, bool): - return None - if isinstance(v, (str, int)) and str(v).strip(): - return str(v).strip() - return None - - -def _str_field(v) -> str | None: - if isinstance(v, str) and v.strip(): - return v.strip() - return None - - -def upgrade() -> None: - conn = op.get_bind() - - # ── PART 1: Per-platform corrections requiring filesystem IO ───── - # SubscribeStar, HentaiFoundry, Discord all need fields from the - # sidecar to construct the right post_url. We walk each Post's - # related ImageRecord.path to find the sidecar, read it, derive, - # and update. - targets = conn.execute(text(""" - SELECT p.id, p.external_post_id, p.post_url, s.platform - FROM post p - JOIN source s ON s.id = p.source_id - WHERE s.platform IN ('subscribestar', 'hentaifoundry', 'discord') - """)).fetchall() - - stats: dict[str, dict[str, int]] = { - plat: {"read": 0, "updated": 0, "no_sidecar": 0} - for plat in ("subscribestar", "hentaifoundry", "discord") - } - for post_row in targets: - plat = post_row.platform - path = _first_attachment_path(conn, post_row.id) - if not path: - stats[plat]["no_sidecar"] += 1 - continue - sidecar = _find_sidecar(Path(path)) - if sidecar is None: - stats[plat]["no_sidecar"] += 1 - continue - try: - data = json.loads(sidecar.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - stats[plat]["no_sidecar"] += 1 - continue - stats[plat]["read"] += 1 - - new_epid = post_row.external_post_id - new_url = None - if plat == "subscribestar": - pid = _str_id(data.get("post_id")) - if pid: - new_epid = pid - new_url = f"https://www.subscribestar.com/posts/{pid}" - elif plat == "hentaifoundry": - user = _str_field(data.get("user")) or _str_field(data.get("artist")) - idx = _str_id(data.get("index")) - if user and idx: - new_url = f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" - elif plat == "discord": - sid = _str_id(data.get("server_id")) - cid = _str_id(data.get("channel_id")) - mid = _str_id(data.get("message_id")) - if sid and cid and mid: - new_url = f"https://discord.com/channels/{sid}/{cid}/{mid}" - - # Idempotent: skip if nothing changed. - if new_epid == post_row.external_post_id and new_url == post_row.post_url: - continue - conn.execute( - text(""" - UPDATE post - SET external_post_id = :epid, post_url = :url - WHERE id = :id - """), - {"epid": new_epid, "url": new_url, "id": post_row.id}, - ) - stats[plat]["updated"] += 1 - - for plat, s in stats.items(): - print( - f"0025: {plat} — read {s['read']} sidecars, " - f"updated {s['updated']} Posts, " - f"{s['no_sidecar']} Posts had no resolvable sidecar" - ) - - # ── PART 2: Merge SubscribeStar fragments now sharing epid ─────── - # After Part 1, each group of Posts under one source with the SAME - # new external_post_id is a fragment-set of the same actual post. - # Merge to one canonical row. Pre-handle the same ImageProvenance - # collision pattern as alembic 0022 (uq_image_provenance_image_post). - fragment_groups = conn.execute(text(""" - SELECT p.source_id, p.external_post_id, - ARRAY_AGG(p.id ORDER BY p.id ASC) AS post_ids - FROM post p - JOIN source s ON s.id = p.source_id - WHERE s.platform = 'subscribestar' - AND p.external_post_id IS NOT NULL - GROUP BY p.source_id, p.external_post_id - HAVING COUNT(*) > 1 - """)).fetchall() - - merged = 0 - for grp in fragment_groups: - post_ids = list(grp.post_ids) - keep_id, *drop_ids = post_ids - for drop_id in drop_ids: - # Pre-DELETE colliding ImageProvenance under drop_ that - # already exist under keep (alembic 0022 banked the pattern). - conn.execute( - text(""" - DELETE FROM image_provenance - WHERE post_id = :drop_ - AND image_record_id IN ( - SELECT image_record_id FROM image_provenance - WHERE post_id = :keep - ) - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text(""" - UPDATE image_provenance SET post_id = :keep - WHERE post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text(""" - UPDATE image_record SET primary_post_id = :keep - WHERE primary_post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text(""" - UPDATE post_attachment SET post_id = :keep - WHERE post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text("DELETE FROM post WHERE id = :drop_"), - {"drop_": drop_id}, - ) - merged += 1 - print(f"0025: subscribestar — merged {merged} duplicate Post fragments") - - # ── PART 3: Pixiv post_url backfill (pure SQL) ─────────────────── - # Pixiv's external_post_id is already correct (gallery-dl's `id` is - # the post id). Only post_url needs derivation: replace anything - # under i.pximg.net (the file URL) with the /artworks/ permalink. - pixiv_updated = conn.execute(text(""" - UPDATE post p - SET post_url = 'https://www.pixiv.net/artworks/' || p.external_post_id - FROM source s - WHERE p.source_id = s.id - AND s.platform = 'pixiv' - AND p.external_post_id IS NOT NULL - AND (p.post_url IS NULL - OR p.post_url LIKE 'https://i.pximg.net/%' - OR p.post_url LIKE 'http://i.pximg.net/%') - """)).rowcount - print(f"0025: pixiv — backfilled post_url on {pixiv_updated} Posts") - - -def _first_attachment_path(conn, post_id: int) -> str | None: - """Return any ImageRecord.path attached to this post (via - ImageProvenance). Lowest-id row keeps the migration deterministic - so re-running on the same DB picks the same sidecar.""" - row = conn.execute( - text(""" - SELECT ir.path - FROM image_provenance ip - JOIN image_record ir ON ir.id = ip.image_record_id - WHERE ip.post_id = :pid - ORDER BY ip.id ASC - LIMIT 1 - """), - {"pid": post_id}, - ).first() - return row[0] if row else None - - -def downgrade() -> None: - # Lossy: external_post_id values were overwritten with the correct - # post_id; original per-attachment ids weren't preserved. Post-merge - # also deleted drop rows. No safe restore. To roll back the schema - # invariant, fork from 0024 and re-run sidecar imports. - pass diff --git a/alembic/versions/0026_import_task_recovery_count_refetched.py b/alembic/versions/0026_import_task_recovery_count_refetched.py deleted file mode 100644 index ccbc3da..0000000 --- a/alembic/versions/0026_import_task_recovery_count_refetched.py +++ /dev/null @@ -1,53 +0,0 @@ -"""import_task.recovery_count + refetched — poison-pill circuit breaker - -Revision ID: 0026 -Revises: 0025 -Create Date: 2026-05-28 - -Backs the import-task resilience work (operator-flagged 2026-05-28): - -- recovery_count: how many times recover_interrupted_tasks has - re-queued this row from a stuck 'processing' state. A row that - hard-crashes the worker (OOM / segfault on a corrupt or oversized - input) leaves no terminal flip, so the sweep re-queues it — and - without a cap it would loop forever, re-crashing the worker each - time. After MAX_RECOVERY_ATTEMPTS the sweep marks it 'failed' with a - diagnostic instead. - -- refetched: whether a one-shot re-download has already been attempted - for this task's file. Bounds the Layer-2 re-fetch remediation to a - single attempt so source-side corruption doesn't loop. - -Both default to 0 / false; additive, no backfill needed. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0026" -down_revision: Union[str, None] = "0025" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "import_task", - sa.Column( - "recovery_count", sa.Integer(), nullable=False, - server_default="0", - ), - ) - op.add_column( - "import_task", - sa.Column( - "refetched", sa.Boolean(), nullable=False, - server_default=sa.false(), - ), - ) - - -def downgrade() -> None: - op.drop_column("import_task", "refetched") - op.drop_column("import_task", "recovery_count") diff --git a/alembic/versions/0027_drop_migration_run.py b/alembic/versions/0027_drop_migration_run.py deleted file mode 100644 index 454481b..0000000 --- a/alembic/versions/0027_drop_migration_run.py +++ /dev/null @@ -1,50 +0,0 @@ -"""drop migration_run — one-and-done GS/IR migration tooling removed - -Revision ID: 0027 -Revises: 0026 -Create Date: 2026-05-29 - -The GS/IR migration tooling (services/migrators, /api/migrate, the -run_migration task, LegacyMigrationCard, and the MigrationRun model) was -removed after the migration cutover completed. This drops its now-orphaned -run-log table. Downgrade recreates the table (mirrors the old model) so the -migration is reversible. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects.postgresql import JSONB - -revision: str = "0027" -down_revision: Union[str, None] = "0026" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_table("migration_run") - - -def downgrade() -> None: - op.create_table( - "migration_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("kind", sa.String(length=32), nullable=False), - sa.Column("status", sa.String(length=32), nullable=False), - sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.false()), - sa.Column( - "started_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column( - "counts", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"), - ), - sa.Column("error", sa.Text(), nullable=True), - sa.Column( - "metadata", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"), - ), - ) - op.create_index("ix_migration_run_kind", "migration_run", ["kind"]) - op.create_index("ix_migration_run_status", "migration_run", ["status"]) diff --git a/alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py b/alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py deleted file mode 100644 index 5ec9267..0000000 --- a/alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py +++ /dev/null @@ -1,190 +0,0 @@ -"""collapse-sidecar-synthetic: repoint Posts/ImageProvenance/DownloadEvents -from `sidecar::` synthetic Source anchors onto the real -Source for the same (artist, platform) when one exists, then delete the -synthetic. - -Revision ID: 0028 -Revises: 0027 -Create Date: 2026-05-31 - -Background: alembic 0022 (2026-05-26) consolidated the old per-post-URL -Source rows into one canonical Source per (artist, platform). When NO -real campaign URL was salvageable among the candidates, it rewrote the -canonical row to url='sidecar::' enabled=false as a -disabled anchor for any Posts already attached. - -That was fine while it was the only Source for that artist+platform. -But: the unique constraint on Source is (artist_id, platform, url), not -(artist_id, platform). When the operator later added the real -subscription via the UI / extension / etc., a SECOND row landed — -the real one — with id > the synthetic. Both coexisted. - -Two follow-on problems surfaced 2026-05-31: - - 1. The Subscriptions UI listed both rows. The synthetic was disabled - so the scheduler never polled it, but it looked like a phantom - subscription. (Fixed in same commit by SourceService.list filter.) - 2. importer._source_for_sidecar picked Source by `ORDER BY id ASC - LIMIT 1`, so EVERY gallery-dl download since the real Source was - added attached its Post to the SYNTHETIC anchor, not the real - Source. (Fixed in same commit by preferring non-sidecar URLs.) - -This migration is the data half of the cleanup: for every (artist, -platform) with both a synthetic AND a real Source, repoint the -synthetic's children (Posts, ImageProvenance, DownloadEvents) onto the -real Source and delete the synthetic. Reuses the same epid/provenance -collision dance from alembic 0022 because the same uniqueness -constraints fire row-by-row during bulk UPDATEs. - -Lone synthetic anchors — those where no real Source for the same -(artist, platform) exists (e.g., filesystem-imported artist with no -subscription added) — are LEFT INTACT. They anchor real imported -content; deleting them would CASCADE-delete the Posts the operator -imported. The SourceService.list filter hides them from the UI; the -operator can delete them by hand if they want the underlying imports -gone. -""" -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import text - -revision: str = "0028" -down_revision: Union[str, None] = "0027" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - conn = op.get_bind() - - # Find (artist_id, platform) groups where BOTH a sidecar synthetic - # and at least one real Source exist. - groups = conn.execute(text(""" - SELECT artist_id, platform - FROM source - GROUP BY artist_id, platform - HAVING bool_or(url LIKE 'sidecar:%') - AND bool_or(url NOT LIKE 'sidecar:%') - """)).fetchall() - - for artist_id, platform in groups: - rows = conn.execute( - text(""" - SELECT id, url FROM source - WHERE artist_id = :a AND platform = :p - ORDER BY id ASC - """), - {"a": artist_id, "p": platform}, - ).fetchall() - - synthetic_ids = [sid for sid, url in rows if url.startswith("sidecar:")] - real_rows = [(sid, url) for sid, url in rows if not url.startswith("sidecar:")] - if not synthetic_ids or not real_rows: - continue # belt+suspenders; the GROUP BY already filtered - - # Canonical real: lowest-id non-sidecar Source. - canonical_id = real_rows[0][0] - - # STEP A: PRE-merge Post collisions on (canonical, external_post_id). - # Mirror alembic 0022's pre-merge logic — when synth has Post X - # epid=N and real has Post Y epid=N, the bulk UPDATE below would - # trip uq_post_source_external_id row-by-row. Group all Posts - # under (canonical + synthetics) by epid; for any group >1, - # pick a keep (prefer one already under canonical, else lowest - # id) and merge the rest into it. - all_posts = conn.execute( - text(""" - SELECT external_post_id, id, source_id - FROM post - WHERE source_id = :canonical OR source_id = ANY(:synths) - ORDER BY external_post_id, id - """), - {"canonical": canonical_id, "synths": synthetic_ids}, - ).fetchall() - by_epid: dict = {} - for epid, post_id, src_id in all_posts: - by_epid.setdefault(epid, []).append((post_id, src_id)) - for _epid, posts in by_epid.items(): - if len(posts) <= 1: - continue - canonical_side = [p for p in posts if p[1] == canonical_id] - keep_id = canonical_side[0][0] if canonical_side else posts[0][0] - drop_ids = [p[0] for p in posts if p[0] != keep_id] - for drop_id in drop_ids: - # Pre-delete image_provenance rows under drop_ whose - # image_record_id already has provenance under keep — - # avoids tripping uq_image_provenance_image_post (0021) - # row-by-row during the repoint UPDATE. - conn.execute( - text(""" - DELETE FROM image_provenance - WHERE post_id = :drop_ - AND image_record_id IN ( - SELECT image_record_id FROM image_provenance - WHERE post_id = :keep - ) - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text(""" - UPDATE image_provenance SET post_id = :keep - WHERE post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text(""" - UPDATE image_record SET primary_post_id = :keep - WHERE primary_post_id = :drop_ - """), - {"keep": keep_id, "drop_": drop_id}, - ) - conn.execute( - text("DELETE FROM post WHERE id = :drop_"), - {"drop_": drop_id}, - ) - - # STEP B: Bulk reparent the remaining Posts off the synthetics. - conn.execute( - text(""" - UPDATE post SET source_id = :canonical - WHERE source_id = ANY(:synths) - """), - {"canonical": canonical_id, "synths": synthetic_ids}, - ) - - # STEP C: Reparent ImageProvenance.source_id (denormalized FK; - # no UNIQUE on source_id, safe bulk). - conn.execute( - text(""" - UPDATE image_provenance SET source_id = :canonical - WHERE source_id = ANY(:synths) - """), - {"canonical": canonical_id, "synths": synthetic_ids}, - ) - - # STEP D: Reparent any DownloadEvent.source_id. Synthetics are - # enabled=false so the scheduler never created events for them; - # this is belt+suspenders for any rows planted by manual force - # or older code paths. - conn.execute( - text(""" - UPDATE download_event SET source_id = :canonical - WHERE source_id = ANY(:synths) - """), - {"canonical": canonical_id, "synths": synthetic_ids}, - ) - - # STEP E: Drop the now-empty synthetics. - conn.execute( - text("DELETE FROM source WHERE id = ANY(:synths)"), - {"synths": synthetic_ids}, - ) - - -def downgrade() -> None: - # Lossy migration — synthetic Sources deleted, Posts repointed and - # potentially merged. No safe downgrade. - pass diff --git a/alembic/versions/0029_drop_artist_copyright_ml_thresholds.py b/alembic/versions/0029_drop_artist_copyright_ml_thresholds.py deleted file mode 100644 index e6c044b..0000000 --- a/alembic/versions/0029_drop_artist_copyright_ml_thresholds.py +++ /dev/null @@ -1,71 +0,0 @@ -"""drop artist + copyright ml thresholds; lower general default to 0.50 - -Revision ID: 0029 -Revises: 0028 -Create Date: 2026-06-01 - -Operator-flagged 2026-06-01: the view modal's Suggestions panel hides -most general-category predictions because the default threshold is -0.95. Lowering the default to 0.50 (matches character) so general -suggestions surface more aggressively; the value remains tunable in -Settings → ML. - -Same change retires two ML suggestion categories whose Tag.kind -surfaces are unused: - -- `artist`: retired in FC-2d-vii-c — artist identity is acquisition- - derived (image_record.artist_id), never ML-inferred. The threshold - column was a leftover from before that retirement. -- `copyright`: retired 2026-06-01 — the app uses `fandom` for the - franchise/copyright concept (per TagsView.vue's doc comment); no - Tag rows of kind=copyright exist, and the threshold column never - fed anything user-visible. - -Both columns are dropped from ml_settings; the existing row's -suggestion_threshold_general value is bumped from 0.95 to 0.50 iff -it's still at the old default, so deployed installs pick up the new -UX without overriding any operator tuning. -""" -from typing import Sequence, Union - -from alembic import op -from sqlalchemy import text - -revision: str = "0029" -down_revision: Union[str, None] = "0028" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Bump the general threshold for installs still at the old default. - op.execute(text( - "UPDATE ml_settings " - "SET suggestion_threshold_general = 0.50 " - "WHERE id = 1 AND suggestion_threshold_general = 0.95" - )) - op.drop_column("ml_settings", "suggestion_threshold_artist") - op.drop_column("ml_settings", "suggestion_threshold_copyright") - - -def downgrade() -> None: - # Restore the columns with their prior defaults. The bump from - # 0.95 → 0.50 isn't reversible without remembering whether the - # operator had explicitly set 0.95 (unlikely — that was just the - # default) so we leave the current general value as-is. - from sqlalchemy import Column, Float - - op.add_column( - "ml_settings", - Column( - "suggestion_threshold_artist", - Float, nullable=False, server_default="0.30", - ), - ) - op.add_column( - "ml_settings", - Column( - "suggestion_threshold_copyright", - Float, nullable=False, server_default="0.50", - ), - ) diff --git a/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py b/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py deleted file mode 100644 index c37e499..0000000 --- a/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py +++ /dev/null @@ -1,145 +0,0 @@ -"""nullable post.source_id + denormalized post.artist_id; retire sidecar synthetics - -Revision ID: 0030 -Revises: 0029 -Create Date: 2026-06-01 - -Operator-asked 2026-06-01 after the Dymkens orphan investigation: the -sidecar synthetic Source pattern (`sidecar::` rows -with enabled=false) was technically correct but misled the operator -into thinking they had phantom subscriptions. The synthetics existed -solely to satisfy `Post.source_id NOT NULL` for filesystem-imported -content with no real subscription. - -This migration makes the data model honest: - -1. **Post gets a denormalized `artist_id` column** so artist filters - work without traversing `Post → Source.artist_id`. Backfilled from - the existing Source linkage, then NOT NULL'd. -2. **`Post.source_id` becomes nullable**, FK ondelete `CASCADE` → `SET - NULL`. Deleting a Source detaches its Posts instead of destroying - imported content (semantically: subscription ends, archive stays). -3. **`ImageProvenance.source_id` becomes nullable** with the same FK - semantic change. -4. **Sidecar synthetic Sources are deleted** — first NULL out the - FKs from Post + ImageProvenance pointing at them (so the implicit - CASCADE doesn't fire), then delete. DownloadEvent FK is unchanged - (still CASCADE'd, NOT NULL'd) — synthetics have `enabled=false` - so no events exist for them. - -Uniqueness handling: the existing `uq_post_source_external_id` -(source_id, external_post_id) keeps working for source-bound Posts -(Postgres treats NULL != NULL so NULL-source rows aren't deduped by -it). A second partial unique index covers the NULL-source case on -(artist_id, external_post_id) so filesystem-imported posts still -dedupe within an artist. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy import text - -revision: str = "0030" -down_revision: Union[str, None] = "0029" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - conn = op.get_bind() - - # Step 1: add Post.artist_id, initially nullable for backfill. - # FK naming follows the Base.metadata naming_convention - # (fk___) — alembic 0001 set this up. - op.add_column( - "post", - sa.Column("artist_id", sa.Integer, nullable=True), - ) - op.create_foreign_key( - "fk_post_artist_id_artist", "post", "artist", - ["artist_id"], ["id"], ondelete="CASCADE", - ) - - # Step 2: backfill from Source.artist_id (every existing Post has a - # Source today, so every row gets populated). - conn.execute(text(""" - UPDATE post p - SET artist_id = s.artist_id - FROM source s - WHERE p.source_id = s.id AND p.artist_id IS NULL - """)) - - # Sanity: count any remaining NULLs. Should be zero pre-this-migration. - remaining = conn.execute(text( - "SELECT COUNT(*) FROM post WHERE artist_id IS NULL" - )).scalar_one() - if remaining: - raise RuntimeError( - f"alembic 0030: {remaining} post rows have no resolvable " - f"artist_id after backfill. Investigate before continuing." - ) - - # Step 3: enforce NOT NULL + add index for artist-filter queries. - op.alter_column("post", "artist_id", nullable=False) - op.create_index("ix_post_artist_id", "post", ["artist_id"]) - - # Step 4: relax post.source_id + flip FK to SET NULL. The original FK - # name from alembic 0001 is `fk_post_source_id_source` per the - # NAMING_CONVENTION in models/base.py. - op.alter_column("post", "source_id", nullable=True) - op.drop_constraint("fk_post_source_id_source", "post", type_="foreignkey") - op.create_foreign_key( - "fk_post_source_id_source", "post", "source", - ["source_id"], ["id"], ondelete="SET NULL", - ) - - # Step 5: relax image_provenance.source_id + flip FK to SET NULL. - op.alter_column("image_provenance", "source_id", nullable=True) - op.drop_constraint( - "fk_image_provenance_source_id_source", "image_provenance", - type_="foreignkey", - ) - op.create_foreign_key( - "fk_image_provenance_source_id_source", "image_provenance", "source", - ["source_id"], ["id"], ondelete="SET NULL", - ) - - # Step 6: partial unique index on (artist_id, external_post_id) for - # NULL-source Posts. The existing uq_post_source_external_id keeps - # guarding source-bound rows; NULL-source rows now dedupe within - # an artist. - op.execute( - "CREATE UNIQUE INDEX uq_post_artist_external_id_null_source " - "ON post (artist_id, external_post_id) " - "WHERE source_id IS NULL" - ) - - # Step 7: retire sidecar synthetic Sources. NULL out the references - # FIRST (the new FK is SET NULL so CASCADE wouldn't fire anyway, but - # being explicit makes the intent clear). Then delete the synthetic - # source rows. Any DownloadEvent rows under synthetics CASCADE-die - # with the source — synthetics have enabled=false so there shouldn't - # be any in practice. - conn.execute(text(""" - UPDATE post - SET source_id = NULL - WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%') - """)) - conn.execute(text(""" - UPDATE image_provenance - SET source_id = NULL - WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%') - """)) - deleted = conn.execute(text( - "DELETE FROM source WHERE url LIKE 'sidecar:%' RETURNING id" - )).rowcount - print(f"alembic 0030: deleted {deleted} sidecar synthetic source rows") - - -def downgrade() -> None: - # Lossy migration — the deleted sidecar synthetics can't be - # restored from the orphan post.source_id / image_provenance.source_id - # values, and the partial unique index encodes a constraint that - # NULL-source Posts may now exist. No safe downgrade. - pass diff --git a/alembic/versions/0031_source_backfill_runs_remaining.py b/alembic/versions/0031_source_backfill_runs_remaining.py deleted file mode 100644 index ed140fc..0000000 --- a/alembic/versions/0031_source_backfill_runs_remaining.py +++ /dev/null @@ -1,45 +0,0 @@ -"""source.backfill_runs_remaining: sticky deep-scan mode - -Revision ID: 0031 -Revises: 0030 -Create Date: 2026-06-01 - -Tick vs backfill mode for subscription downloads. When -`backfill_runs_remaining > 0`, the next N download runs use -`skip: True` + 30-min timeout (walk full history). When 0, runs use -`skip: "exit:20"` + 14.5-min timeout (catch-up mode, exits early once -20 contiguous archived items are seen). - -Operator-flagged 2026-06-01 (Knuxy run #38887): a creator with ~550 -archived posts saturates the 870s catch-up timeout even when there is -no new content, because gallery-dl's default `skip: True` keeps walking. -Tick mode short-circuits that; backfill mode is the explicit opt-in for -deep history scans. - -Default 0 (all existing subscriptions start in tick mode). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0031" -down_revision: Union[str, None] = "0030" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "source", - sa.Column( - "backfill_runs_remaining", - sa.Integer, - nullable=False, - server_default="0", - ), - ) - - -def downgrade() -> None: - op.drop_column("source", "backfill_runs_remaining") diff --git a/alembic/versions/0032_source_error_type.py b/alembic/versions/0032_source_error_type.py deleted file mode 100644 index e264b35..0000000 --- a/alembic/versions/0032_source_error_type.py +++ /dev/null @@ -1,41 +0,0 @@ -"""source.error_type: surface ErrorType taxonomy in FailingSourcesCard - -Revision ID: 0032 -Revises: 0031 -Create Date: 2026-06-02 - -Audit 2026-06-02: the backend computes 13 ErrorType categories (auth_error, -rate_limited, not_found, access_denied, validation_failed, etc.) and -stamps each one on DownloadEvent.metadata, but the Source row only carried -the free-text last_error. Operators couldn't bulk-triage failing sources -("all auth_error → rotate cookies, all rate_limited → just wait") without -opening Logs per row. - -This column receives the last error_type from _update_source_health -and gets cleared on a successful run. Nullable + indexed so the failing- -sources rollup can filter/group cheaply. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0032" -down_revision: Union[str, None] = "0031" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "source", - sa.Column("error_type", sa.String(length=32), nullable=True), - ) - op.create_index( - "ix_source_error_type", "source", ["error_type"], - ) - - -def downgrade() -> None: - op.drop_index("ix_source_error_type", table_name="source") - op.drop_column("source", "error_type") diff --git a/alembic/versions/0033_suggestion_threshold_default_070.py b/alembic/versions/0033_suggestion_threshold_default_070.py deleted file mode 100644 index 652cf44..0000000 --- a/alembic/versions/0033_suggestion_threshold_default_070.py +++ /dev/null @@ -1,48 +0,0 @@ -"""suggestion_threshold default 0.50 → 0.70 - -Revision ID: 0033 -Revises: 0032 -Create Date: 2026-06-02 - -Operator-flagged 2026-06-02 — the 0.50 default (set on 2026-06-01) is -too noisy in practice; raise to 0.70 for both suggestion categories. - -Only conditionally updates singletons whose current value is still the -2026-06-01 default (0.50). Operators who deliberately tuned their row -to some other value (0.55, 0.65, 0.80, etc. via the Settings UI) keep -their pick — the migration only catches the unchanged-default case. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0033" -down_revision: Union[str, None] = "0032" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute( - "UPDATE ml_settings " - "SET suggestion_threshold_character = 0.70 " - "WHERE id = 1 AND suggestion_threshold_character = 0.50" - ) - op.execute( - "UPDATE ml_settings " - "SET suggestion_threshold_general = 0.70 " - "WHERE id = 1 AND suggestion_threshold_general = 0.50" - ) - - -def downgrade() -> None: - op.execute( - "UPDATE ml_settings " - "SET suggestion_threshold_character = 0.50 " - "WHERE id = 1 AND suggestion_threshold_character = 0.70" - ) - op.execute( - "UPDATE ml_settings " - "SET suggestion_threshold_general = 0.50 " - "WHERE id = 1 AND suggestion_threshold_general = 0.70" - ) diff --git a/alembic/versions/0034_artist_visit.py b/alembic/versions/0034_artist_visit.py deleted file mode 100644 index a2234a6..0000000 --- a/alembic/versions/0034_artist_visit.py +++ /dev/null @@ -1,53 +0,0 @@ -"""artist_visit: per-artist last-viewed timestamp for the "+N new" badge - -Revision ID: 0034 -Revises: 0033 -Create Date: 2026-06-03 - -Powers the artists-directory "+N new since last visit" badge + ArtistView -banner. Single row per artist (no user_id yet — rule #47 multi-user ACL -is aspirational; widens to (user_id, artist_id) PK when User lands). - -Seed every existing artist with `last_viewed_at = NOW()` so the badge -starts at 0 across the board — no noisy "you have 5000 unseen images" -on first deploy. New artists auto-get a row via -`ArtistService.find_or_create`. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0034" -down_revision: Union[str, None] = "0033" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "artist_visit", - sa.Column( - "artist_id", - sa.Integer, - sa.ForeignKey("artist.id", ondelete="CASCADE"), - primary_key=True, - ), - sa.Column( - "last_viewed_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - ) - # Seed: every existing artist starts "fully caught up". Without this, - # every operator with N artists would see N badges (worth of every - # image ever imported) on first deploy. - op.execute( - "INSERT INTO artist_visit (artist_id, last_viewed_at) " - "SELECT id, NOW() FROM artist" - ) - - -def downgrade() -> None: - op.drop_table("artist_visit") diff --git a/alembic/versions/0035_image_record_effective_date.py b/alembic/versions/0035_image_record_effective_date.py deleted file mode 100644 index 586cf51..0000000 --- a/alembic/versions/0035_image_record_effective_date.py +++ /dev/null @@ -1,70 +0,0 @@ -"""image_record.effective_date: materialized gallery sort key + index - -Revision ID: 0035 -Revises: 0034 -Create Date: 2026-06-04 - -The gallery ordered/cursored on COALESCE(post.post_date, -image_record.created_at) across the Post outer join. That expression spans -two tables, so no index can serve it — every /scroll sorted a large slice -of the library, and the frontend fired ten of them serially per initial -load. Materialize the value into image_record.effective_date and index -(effective_date DESC, id DESC) so the cursor scroll is an index range scan. - -Backfill = COALESCE(primary post's post_date, created_at) so existing rows -keep their exact ordering. New rows get the created_at-equivalent server -default; services/importer.py overrides it with the post's date when a -primary post with a date is linked. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0035" -down_revision: Union[str, None] = "0034" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Add nullable first so the backfill can populate before NOT NULL. - op.add_column( - "image_record", - sa.Column("effective_date", sa.DateTime(timezone=True), nullable=True), - ) - # Pure set-based UPDATEs (no per-row params) — immune to the 65535 - # bind-parameter ceiling regardless of library size. - op.execute( - """ - UPDATE image_record AS ir - SET effective_date = COALESCE(p.post_date, ir.created_at) - FROM post AS p - WHERE ir.primary_post_id = p.id - """ - ) - op.execute( - """ - UPDATE image_record - SET effective_date = created_at - WHERE effective_date IS NULL - """ - ) - op.alter_column( - "image_record", - "effective_date", - nullable=False, - server_default=sa.text("now()"), - ) - # DESC/DESC matches the gallery's ORDER BY effective_date DESC, id DESC - # so the scroll is a forward index scan; raw SQL because alembic's - # column list doesn't express per-column DESC cleanly. - op.execute( - "CREATE INDEX ix_image_record_effective_date " - "ON image_record (effective_date DESC, id DESC)" - ) - - -def downgrade() -> None: - op.drop_index("ix_image_record_effective_date", table_name="image_record") - op.drop_column("image_record", "effective_date") diff --git a/alembic/versions/0036_siglip_embedding_hnsw_index.py b/alembic/versions/0036_siglip_embedding_hnsw_index.py deleted file mode 100644 index a8c1251..0000000 --- a/alembic/versions/0036_siglip_embedding_hnsw_index.py +++ /dev/null @@ -1,41 +0,0 @@ -"""image_record.siglip_embedding: HNSW cosine index for "more like this" - -Revision ID: 0036 -Revises: 0035 -Create Date: 2026-06-04 - -Gallery Phase 3 (visual similarity search) ranks images by -`siglip_embedding.cosine_distance(source_embedding)`. Without an index that's -a sequential scan computing a 1152-dim distance for every row — fine at small -scale, but it grows linearly with the library. Add an HNSW index with -`vector_cosine_ops` so the top-N nearest search is sub-50ms ANN. - -1152 dims is under pgvector's 2000-dim HNSW limit, so HNSW (no training, -better recall than IVFFlat) is the right choice. ONE-TIME COST: building the -index over the existing embeddings (~57k vectors on the operator's library) -locks image_record for ~30-60s during this migration on deploy — acceptable -for a single-operator homelab. NULL embeddings (videos / not-yet-embedded -rows) are simply not indexed. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0036" -down_revision: Union[str, None] = "0035" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Raw SQL: alembic's create_index doesn't express the `USING hnsw (... - # vector_cosine_ops)` access-method + opclass cleanly. Must match the - # query's cosine_distance operator class to be usable by the planner. - op.execute( - "CREATE INDEX ix_image_record_siglip_hnsw " - "ON image_record USING hnsw (siglip_embedding vector_cosine_ops)" - ) - - -def downgrade() -> None: - op.drop_index("ix_image_record_siglip_hnsw", table_name="image_record") diff --git a/alembic/versions/0037_patreon_seen_media.py b/alembic/versions/0037_patreon_seen_media.py deleted file mode 100644 index 255484e..0000000 --- a/alembic/versions/0037_patreon_seen_media.py +++ /dev/null @@ -1,53 +0,0 @@ -"""patreon_seen_media: per-source ledger of already-ingested Patreon media - -Revision ID: 0037 -Revises: 0036 -Create Date: 2026-06-05 - -Native Patreon ingester (build step 2a). Replaces gallery-dl's -archive.sqlite3 with our own queryable table. The downloader upserts one -row per (source, media) so routine walks skip media we've already -processed; a future "recovery" mode bypasses the ledger to re-walk. - -`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form -``video::`` — hence String(128). The unique -constraint on (source_id, filehash) is the dedup upsert key. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0037" -down_revision: Union[str, None] = "0036" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "patreon_seen_media", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "source_id", - sa.Integer, - sa.ForeignKey("source.id", ondelete="CASCADE"), - nullable=False, - index=True, - ), - sa.Column("filehash", sa.String(128), nullable=False), - sa.Column("post_id", sa.String(64), nullable=True), - sa.Column( - "seen_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.UniqueConstraint( - "source_id", "filehash", name="uq_patreon_seen_media_source_id" - ), - ) - - -def downgrade() -> None: - op.drop_table("patreon_seen_media") diff --git a/alembic/versions/0038_patreon_failed_media.py b/alembic/versions/0038_patreon_failed_media.py deleted file mode 100644 index e907ae1..0000000 --- a/alembic/versions/0038_patreon_failed_media.py +++ /dev/null @@ -1,58 +0,0 @@ -"""patreon_failed_media: per-source dead-letter ledger for failing Patreon media - -Revision ID: 0038 -Revises: 0037 -Create Date: 2026-06-06 - -Plan #705 (#7). Media that keeps failing to download/validate (404'd CDN, -deleted post, geo-blocked Mux, persistently-corrupt bytes) gets recorded here -with an attempt counter; once it crosses the dead-letter threshold the ingester -skips it on routine walks (recovery still re-attempts). A clean download clears -the row. UNIQUE (source_id, filehash) is the upsert key (same media key the -seen-ledger uses). -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0038" -down_revision: Union[str, None] = "0037" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "patreon_failed_media", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "source_id", - sa.Integer, - sa.ForeignKey("source.id", ondelete="CASCADE"), - nullable=False, - index=True, - ), - sa.Column("filehash", sa.String(128), nullable=False), - sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), - sa.Column("last_error", sa.Text, nullable=True), - sa.Column( - "first_failed_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.Column( - "last_failed_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.UniqueConstraint( - "source_id", "filehash", name="uq_patreon_failed_media_source_id" - ), - ) - - -def downgrade() -> None: - op.drop_table("patreon_failed_media") diff --git a/alembic/versions/0039_library_audit_resume.py b/alembic/versions/0039_library_audit_resume.py deleted file mode 100644 index 6cfb8f9..0000000 --- a/alembic/versions/0039_library_audit_resume.py +++ /dev/null @@ -1,40 +0,0 @@ -"""library_audit_run: resume cursor + progress timestamp for chunked scans - -Revision ID: 0039 -Revises: 0038 -Create Date: 2026-06-07 - -scan_library_for_rule used to run one 2h pass that timed out on large libraries -and monopolized the concurrency-1 maintenance queue (operator-flagged). It now -runs short time-boxed chunks that re-enqueue: `resume_after_id` persists the -keyset cursor so the next chunk continues where it left off, and -`last_progress_at` lets the recovery sweep tell a progressing multi-chunk audit -from a genuinely stuck one. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0039" -down_revision: Union[str, None] = "0038" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "library_audit_run", - sa.Column( - "resume_after_id", sa.Integer, nullable=False, server_default="0" - ), - ) - op.add_column( - "library_audit_run", - sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("library_audit_run", "last_progress_at") - op.drop_column("library_audit_run", "resume_after_id") diff --git a/alembic/versions/0040_series_chapters.py b/alembic/versions/0040_series_chapters.py deleted file mode 100644 index a0808df..0000000 --- a/alembic/versions/0040_series_chapters.py +++ /dev/null @@ -1,108 +0,0 @@ -"""series chapters: chapter layer over series_page (FC-6.1) - -Revision ID: 0040 -Revises: 0039 -Create Date: 2026-06-07 - -A series (Tag kind='series') gains an ordered chapter layer. Reading order -becomes (series_chapter.chapter_number, series_page.page_number). Every existing -series is backfilled into a single auto-chapter (chapter_number=1) holding its -current flat pages, so no data is lost and the old flat ordering is preserved. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0040" -down_revision: Union[str, None] = "0039" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "series_chapter", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "series_tag_id", - sa.Integer, - sa.ForeignKey("tag.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("chapter_number", sa.Integer, nullable=False), - sa.Column("title", sa.Text, nullable=True), - sa.Column( - "is_placeholder", sa.Boolean, nullable=False, server_default="false" - ), - sa.Column("stated_page_start", sa.Integer, nullable=True), - sa.Column("stated_page_end", sa.Integer, nullable=True), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("now()"), - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("now()"), - ), - ) - op.create_index( - "ix_series_chapter_series_tag_id", "series_chapter", ["series_tag_id"] - ) - - # New columns on series_page; chapter_id starts nullable so we can backfill. - op.add_column( - "series_page", sa.Column("chapter_id", sa.Integer, nullable=True) - ) - op.add_column( - "series_page", sa.Column("stated_page", sa.Integer, nullable=True) - ) - - conn = op.get_bind() - # One auto-chapter per existing series (any series_tag_id present in pages). - conn.execute( - sa.text( - "INSERT INTO series_chapter " - "(series_tag_id, chapter_number, is_placeholder, created_at, updated_at) " - "SELECT DISTINCT series_tag_id, 1, false, now(), now() " - "FROM series_page" - ) - ) - # Point every existing page at its series' auto-chapter. - conn.execute( - sa.text( - "UPDATE series_page sp " - "SET chapter_id = sc.id " - "FROM series_chapter sc " - "WHERE sc.series_tag_id = sp.series_tag_id" - ) - ) - - # Now lock chapter_id down: NOT NULL + FK (cascade) + index. - op.alter_column("series_page", "chapter_id", nullable=False) - op.create_foreign_key( - "fk_series_page_chapter_id", - "series_page", - "series_chapter", - ["chapter_id"], - ["id"], - ondelete="CASCADE", - ) - op.create_index( - "ix_series_page_chapter_id", "series_page", ["chapter_id"] - ) - - -def downgrade() -> None: - op.drop_index("ix_series_page_chapter_id", table_name="series_page") - op.drop_constraint( - "fk_series_page_chapter_id", "series_page", type_="foreignkey" - ) - op.drop_column("series_page", "stated_page") - op.drop_column("series_page", "chapter_id") - op.drop_index("ix_series_chapter_series_tag_id", table_name="series_chapter") - op.drop_table("series_chapter") diff --git a/alembic/versions/0041_series_suggestions.py b/alembic/versions/0041_series_suggestions.py deleted file mode 100644 index 51b690a..0000000 --- a/alembic/versions/0041_series_suggestions.py +++ /dev/null @@ -1,98 +0,0 @@ -"""series suggestions: assisted-continuation matcher (FC-6.3) - -Revision ID: 0041 -Revises: 0040 -Create Date: 2026-06-07 - -A confirm-only queue of "this post may continue this series" hints, plus two -import_settings knobs (enable + score threshold) for the matcher. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0041" -down_revision: Union[str, None] = "0040" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "series_suggestion", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "post_id", - sa.Integer, - sa.ForeignKey("post.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column( - "series_tag_id", - sa.Integer, - sa.ForeignKey("tag.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("score", sa.Float, nullable=False), - sa.Column("signals", sa.JSON, nullable=True), - sa.Column( - "status", sa.String(16), nullable=False, server_default="pending" - ), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("now()"), - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("now()"), - ), - sa.UniqueConstraint( - "post_id", "series_tag_id", name="uq_series_suggestion_post_series" - ), - ) - op.create_index( - "ix_series_suggestion_post_id", "series_suggestion", ["post_id"] - ) - op.create_index( - "ix_series_suggestion_series_tag_id", - "series_suggestion", - ["series_tag_id"], - ) - op.create_index( - "ix_series_suggestion_status", "series_suggestion", ["status"] - ) - - op.add_column( - "import_settings", - sa.Column( - "series_suggest_enabled", - sa.Boolean, - nullable=False, - server_default=sa.true(), - ), - ) - op.add_column( - "import_settings", - sa.Column( - "series_suggest_threshold", - sa.Float, - nullable=False, - server_default="0.5", - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "series_suggest_threshold") - op.drop_column("import_settings", "series_suggest_enabled") - op.drop_index("ix_series_suggestion_status", table_name="series_suggestion") - op.drop_index( - "ix_series_suggestion_series_tag_id", table_name="series_suggestion" - ) - op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion") - op.drop_table("series_suggestion") diff --git a/alembic/versions/0042_series_chapter_stated_part.py b/alembic/versions/0042_series_chapter_stated_part.py deleted file mode 100644 index f898e56..0000000 --- a/alembic/versions/0042_series_chapter_stated_part.py +++ /dev/null @@ -1,32 +0,0 @@ -"""series chapter stated_part: operator-facing Part N label (FC-6.4) - -Revision ID: 0042 -Revises: 0041 -Create Date: 2026-06-07 - -A chapter's positional chapter_number is auto-managed (rewritten 1..N on -reorder/delete), so it can't double as the installment number the operator wants -to type (e.g. a series authored from a post that is Part 2). Add a nullable -stated_part alongside it — the same split as series_page.page_number (order) vs -series_page.stated_page (printed number). Nullable; the UI falls back to -chapter_number when unset. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0042" -down_revision: Union[str, None] = "0041" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "series_chapter", sa.Column("stated_part", sa.Integer, nullable=True) - ) - - -def downgrade() -> None: - op.drop_column("series_chapter", "stated_part") diff --git a/alembic/versions/0043_post_attachment_per_post_unique.py b/alembic/versions/0043_post_attachment_per_post_unique.py deleted file mode 100644 index e8e38ce..0000000 --- a/alembic/versions/0043_post_attachment_per_post_unique.py +++ /dev/null @@ -1,62 +0,0 @@ -"""post_attachment: per-post sha uniqueness (empty-post flood fix) - -Revision ID: 0043 -Revises: 0042 -Create Date: 2026-06-08 - -PostAttachment.sha256 was GLOBALLY unique, so a non-art file the creator attaches -to many posts (a standard pdf/zip/link-card) only ever got ONE row — on the first -post — leaving every later post a bare shell (no image, no attachment). The native -Patreon backfill of Anduo surfaced 1589 such shells (operator-flagged 2026-06-08). - -Switch to PER-POST uniqueness: the on-disk blob stays sha-deduped, but each post -gets its own row. Replace the unique sha256 index with a plain lookup index plus -two partial uniques — (post_id, sha256) for real posts and (sha256) for the -NULL-post filesystem case (still one row per file there). - -Existing data has ≤1 row per sha (the old global unique), so the new partial -uniques can't be violated on upgrade — no data backfill needed here. The bare-post -shells themselves are removed by the separate prune-empty-posts cleanup tool. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0043" -down_revision: Union[str, None] = "0042" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Drop the global unique index; recreate it as a plain (non-unique) lookup - # index so sha-based reads keep their index (matches the model's index=True). - op.drop_index("ix_post_attachment_sha256", table_name="post_attachment") - op.create_index( - "ix_post_attachment_sha256", "post_attachment", ["sha256"], - ) - op.create_index( - "uq_post_attachment_post_sha", "post_attachment", - ["post_id", "sha256"], unique=True, - postgresql_where=sa.text("post_id IS NOT NULL"), - ) - op.create_index( - "uq_post_attachment_null_post_sha", "post_attachment", - ["sha256"], unique=True, - postgresql_where=sa.text("post_id IS NULL"), - ) - - -def downgrade() -> None: - op.drop_index( - "uq_post_attachment_null_post_sha", table_name="post_attachment" - ) - op.drop_index( - "uq_post_attachment_post_sha", table_name="post_attachment" - ) - op.drop_index("ix_post_attachment_sha256", table_name="post_attachment") - op.create_index( - "ix_post_attachment_sha256", "post_attachment", ["sha256"], - unique=True, - ) diff --git a/alembic/versions/0044_ml_settings_tagger_store_floor.py b/alembic/versions/0044_ml_settings_tagger_store_floor.py deleted file mode 100644 index e019e36..0000000 --- a/alembic/versions/0044_ml_settings_tagger_store_floor.py +++ /dev/null @@ -1,37 +0,0 @@ -"""ml_settings.tagger_store_floor - -The ingest confidence floor below which tagger predictions are not stored, -promoted from the TAGGER_STORE_FLOOR env var to a DB-backed, UI-tunable -setting. Default 0.70 (was an env default of 0.05): the suggestion path -already filters at 0.70 and the centroid/learned path covers low-confidence -preferred tags, so the sub-0.70 tail was redundant weight — it had grown -image_record's TOAST to ~100 GB. See plan-task #764. - -Revision ID: 0044 -Revises: 0043 -Create Date: 2026-06-10 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0044" -down_revision: Union[str, None] = "0043" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "tagger_store_floor", sa.Float(), - nullable=False, server_default="0.7", - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "tagger_store_floor") diff --git a/alembic/versions/0045_image_prediction_table.py b/alembic/versions/0045_image_prediction_table.py deleted file mode 100644 index df11de2..0000000 --- a/alembic/versions/0045_image_prediction_table.py +++ /dev/null @@ -1,69 +0,0 @@ -"""image_prediction table (DDL only — backfill runs as a background task) - -Normalizes the per-image tagger predictions out of the JSON blob into a -queryable table (#768). This migration creates ONLY the table + indexes — it -is pure DDL and commits instantly, so web boots immediately. - -The data backfill from the existing image_record.tagger_predictions JSON is -deliberately NOT done here. Doing it inline made the whole migration one -transaction over the ~100 GB TOAST: nothing committed until the very end, it -was invisible/unmonitorable mid-run, and an early MATERIALIZED-CTE form spilled -the full 100 GB to temp. Instead the backfill is the -backend.app.tasks.admin.backfill_image_predictions_task — batched by id window, -committed per chunk (visible progress + resumable), idempotent -(ON CONFLICT DO NOTHING). Trigger it from Settings → Maintenance once web is up. - -The old image_record.tagger_predictions column is left in place (vestigial) and -dropped in a follow-up once the backfill + code cutover are verified — dropping -it needs an ACCESS EXCLUSIVE lock on the hot image_record table (the 0044 lock -class), so it's deferred to a quiesced-worker window. - -Revision ID: 0045 -Revises: 0044 -Create Date: 2026-06-10 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0045" -down_revision: Union[str, None] = "0044" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "image_prediction", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "image_record_id", sa.Integer(), - sa.ForeignKey("image_record.id", ondelete="CASCADE"), - nullable=False, - ), - sa.Column("raw_name", sa.String(length=255), nullable=False), - sa.Column("category", sa.String(length=64), nullable=False), - sa.Column("score", sa.Float(), nullable=False), - sa.UniqueConstraint( - "image_record_id", "raw_name", name="image_raw_name", - ), - ) - op.create_index( - "ix_image_prediction_image", "image_prediction", ["image_record_id"], - ) - op.create_index( - "ix_image_prediction_name_score", "image_prediction", - ["raw_name", "score"], - ) - # No data backfill here — see the module docstring. The one-time copy from - # image_record.tagger_predictions runs as backfill_image_predictions_task - # (batched, resumable, idempotent), kept out of this transaction so web boots - # without waiting on a ~100 GB pass. - - -def downgrade() -> None: - op.drop_index("ix_image_prediction_name_score", "image_prediction") - op.drop_index("ix_image_prediction_image", "image_prediction") - op.drop_table("image_prediction") diff --git a/alembic/versions/0046_drop_tagger_predictions.py b/alembic/versions/0046_drop_tagger_predictions.py deleted file mode 100644 index 84e543a..0000000 --- a/alembic/versions/0046_drop_tagger_predictions.py +++ /dev/null @@ -1,43 +0,0 @@ -"""drop image_record.tagger_predictions (predictions normalized to image_prediction) - -Final step of #768. The per-tag predictions now live in the image_prediction -table (backfilled from the JSON, read by suggestions + allowlist, written by -tag_and_embed). The old JSON column is dead weight — and it's the ~100 GB of -sub-0.70 score tail that bloated image_record's TOAST and broke DB backups -(#739). Dropping it is a fast catalog change; it does NOT reclaim the disk on -its own — run `VACUUM FULL image_record` (or pg_repack) afterward, off-hours, -to return the space to the OS so backups go small. - -DROP COLUMN needs a brief ACCESS EXCLUSIVE lock on image_record; env.py's -lock_timeout guards it, so quiesce the ml-worker if a tagging run is in flight -(see the migration-lock reference). tagger_model_version is kept — it's the -"has this been tagged / is it current?" signal the backfill sweep reads. - -Revision ID: 0046 -Revises: 0045 -Create Date: 2026-06-11 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0046" -down_revision: Union[str, None] = "0045" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_column("image_record", "tagger_predictions") - - -def downgrade() -> None: - # Re-add the column empty. The JSON data is not restored (it lived only in - # this column); a downgrade would re-tag or backfill from image_prediction - # separately if ever needed. - op.add_column( - "image_record", - sa.Column("tagger_predictions", sa.JSON(), nullable=True), - ) diff --git a/alembic/versions/0047_series_chapter_dividers.py b/alembic/versions/0047_series_chapter_dividers.py deleted file mode 100644 index 5afd074..0000000 --- a/alembic/versions/0047_series_chapter_dividers.py +++ /dev/null @@ -1,175 +0,0 @@ -"""series chapters become cosmetic dividers; pages become one series-global run - -FC-6.x reframe (#789). A series is now ONE flat, series-global ordered run of -pages; chapters stop owning pages and become labeled dividers anchored to the -page that begins them. - -Migration (order matters — series_page.chapter_id cascades, so it must be -dropped BEFORE any chapter row is deleted, or pages would cascade away): - a. Renumber series_page.page_number to a series-global 1..N (ordered by the - OLD (chapter_number, page_number)). - b. Add series_chapter.anchor_page_id and populate it with each chapter's first - page (lowest new page_number). - c. Drop series_page.chapter_id (severs the cascade link). - d. Prune chapters that shouldn't become dividers: empty/placeholder ones (no - anchor) and the redundant unlabeled chapter that would sit at page 1. - e. Reshape series_chapter into the divider: drop chapter_number, - is_placeholder, stated_page_start/end; make anchor_page_id NOT NULL + - UNIQUE + FK→series_page ON DELETE CASCADE. - -Revision ID: 0047 -Revises: 0046 -Create Date: 2026-06-11 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0047" -down_revision: Union[str, None] = "0046" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # a. series-global page numbering, preserving the old reading order. - op.execute( - """ - WITH ordered AS ( - SELECT sp.id, - ROW_NUMBER() OVER ( - PARTITION BY sp.series_tag_id - ORDER BY sc.chapter_number, sp.page_number, sp.id - ) AS rn - FROM series_page sp - JOIN series_chapter sc ON sc.id = sp.chapter_id - ) - UPDATE series_page sp - SET page_number = ordered.rn - FROM ordered - WHERE sp.id = ordered.id - """ - ) - - # b. anchor each existing chapter at its first page (lowest new page_number). - op.add_column( - "series_chapter", - sa.Column("anchor_page_id", sa.Integer(), nullable=True), - ) - op.execute( - """ - WITH firsts AS ( - SELECT DISTINCT ON (sp.chapter_id) - sp.chapter_id, sp.id AS page_id - FROM series_page sp - ORDER BY sp.chapter_id, sp.page_number, sp.id - ) - UPDATE series_chapter sc - SET anchor_page_id = firsts.page_id - FROM firsts - WHERE firsts.chapter_id = sc.id - """ - ) - - # c. sever the ownership link (drops the FK + index with the column) BEFORE - # pruning chapters, so deleting a chapter can't cascade-delete its pages. - op.drop_column("series_page", "chapter_id") - - # d. prune chapters that don't become dividers: placeholders / empty ones - # (no anchor), and the unlabeled chapter that would land redundantly at - # page 1 (the series just starts — no divider needed there). - op.execute( - """ - DELETE FROM series_chapter sc - USING ( - SELECT sc2.id - FROM series_chapter sc2 - LEFT JOIN series_page sp ON sp.id = sc2.anchor_page_id - WHERE sc2.anchor_page_id IS NULL - OR (sp.page_number = 1 - AND sc2.title IS NULL - AND sc2.stated_part IS NULL) - ) gone - WHERE sc.id = gone.id - """ - ) - - # e. reshape into the divider model. - op.drop_column("series_chapter", "chapter_number") - op.drop_column("series_chapter", "is_placeholder") - op.drop_column("series_chapter", "stated_page_start") - op.drop_column("series_chapter", "stated_page_end") - op.alter_column("series_chapter", "anchor_page_id", nullable=False) - op.create_unique_constraint( - "uq_series_chapter_anchor_page", "series_chapter", ["anchor_page_id"] - ) - op.create_foreign_key( - "fk_series_chapter_anchor_page", - "series_chapter", - "series_page", - ["anchor_page_id"], - ["id"], - ondelete="CASCADE", - ) - - -def downgrade() -> None: - # Lossy: dividers can't be reconstructed as owning chapters. Collapse back to - # exactly one chapter per series that owns all its pages in order. - op.add_column( - "series_page", sa.Column("chapter_id", sa.Integer(), nullable=True) - ) - op.drop_constraint( - "fk_series_chapter_anchor_page", "series_chapter", type_="foreignkey" - ) - op.drop_constraint( - "uq_series_chapter_anchor_page", "series_chapter", type_="unique" - ) - op.drop_column("series_chapter", "anchor_page_id") - op.add_column( - "series_chapter", - sa.Column( - "chapter_number", sa.Integer(), nullable=False, server_default="1" - ), - ) - op.add_column( - "series_chapter", - sa.Column( - "is_placeholder", sa.Boolean(), nullable=False, - server_default="false", - ), - ) - op.add_column( - "series_chapter", - sa.Column("stated_page_start", sa.Integer(), nullable=True), - ) - op.add_column( - "series_chapter", - sa.Column("stated_page_end", sa.Integer(), nullable=True), - ) - op.execute("DELETE FROM series_chapter") - op.execute( - """ - INSERT INTO series_chapter (series_tag_id, chapter_number) - SELECT DISTINCT series_tag_id, 1 FROM series_page - """ - ) - op.execute( - """ - UPDATE series_page sp - SET chapter_id = sc.id - FROM series_chapter sc - WHERE sc.series_tag_id = sp.series_tag_id - """ - ) - op.alter_column("series_page", "chapter_id", nullable=False) - op.create_foreign_key( - "fk_series_page_chapter", - "series_page", - "series_chapter", - ["chapter_id"], - ["id"], - ondelete="CASCADE", - ) diff --git a/alembic/versions/0048_series_page_pending_status.py b/alembic/versions/0048_series_page_pending_status.py deleted file mode 100644 index 25944a5..0000000 --- a/alembic/versions/0048_series_page_pending_status.py +++ /dev/null @@ -1,45 +0,0 @@ -"""series_page pending staging: status + nullable page_number (#789 Phase 2) - -Pages added from a post no longer append straight into the run — they land -'pending' with a NULL page_number, staged grouped by their source post so the -operator can drop junk (text-free alts, bumpers) and place the keepers into the -sequence. A page only gets a series-global page_number once it's 'placed'. - -Revision ID: 0048 -Revises: 0047 -Create Date: 2026-06-11 - -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0048" -down_revision: Union[str, None] = "0047" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "series_page", - sa.Column( - "status", sa.String(length=16), nullable=False, - server_default="placed", - ), - ) - op.alter_column( - "series_page", "page_number", - existing_type=sa.Integer(), nullable=True, - ) - - -def downgrade() -> None: - # Lossy: pending pages are unsorted staging rows with no order — drop them. - op.execute("DELETE FROM series_page WHERE status = 'pending'") - op.alter_column( - "series_page", "page_number", - existing_type=sa.Integer(), nullable=False, - ) - op.drop_column("series_page", "status") diff --git a/alembic/versions/0049_external_link_table.py b/alembic/versions/0049_external_link_table.py deleted file mode 100644 index 373c807..0000000 --- a/alembic/versions/0049_external_link_table.py +++ /dev/null @@ -1,90 +0,0 @@ -"""external_link table — off-platform file-host links found in post bodies - -Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox / -Pixeldrain and link them in the post text. This table records each such link -(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger -the download worker (a later slice) walks. `url` keeps the FULL link including -the `#fragment` — mega.nz's decryption key lives there; truncating it makes the -file undownloadable. - -CHECK whitelists for host + status include the full enum up front (incl. the -download-worker statuses) so the worker slice needs no constraint migration. - -Revision ID: 0049 -Revises: 0048 -Create Date: 2026-06-14 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0049" -down_revision: Union[str, None] = "0048" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "external_link", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "post_id", sa.Integer(), - sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False, - ), - sa.Column( - "artist_id", sa.Integer(), - sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, - ), - sa.Column("host", sa.String(length=16), nullable=False), - sa.Column("url", sa.Text(), nullable=False), - sa.Column("label", sa.Text(), nullable=True), - sa.Column( - "status", sa.String(length=16), nullable=False, - server_default="pending", - ), - sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), - sa.Column("last_error", sa.Text(), nullable=True), - sa.Column( - "attachment_id", sa.Integer(), - sa.ForeignKey("post_attachment.id", ondelete="SET NULL"), - nullable=True, - ), - sa.Column( - "created_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("duration_seconds", sa.Float(), nullable=True), - sa.CheckConstraint( - "host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')", - name="ck_external_link_host", - ), - sa.CheckConstraint( - "status IN ('pending','downloading','downloaded','failed'," - "'skipped','dead')", - name="ck_external_link_status", - ), - ) - op.create_index( - "ix_external_link_post_id", "external_link", ["post_id"], - ) - op.create_index( - "ix_external_link_artist_id", "external_link", ["artist_id"], - ) - op.create_index( - "ix_external_link_status", "external_link", ["status"], - ) - op.create_index( - "uq_external_link_post_url", "external_link", ["post_id", "url"], - unique=True, - ) - - -def downgrade() -> None: - op.drop_index("uq_external_link_post_url", table_name="external_link") - op.drop_index("ix_external_link_status", table_name="external_link") - op.drop_index("ix_external_link_artist_id", table_name="external_link") - op.drop_index("ix_external_link_post_id", table_name="external_link") - op.drop_table("external_link") diff --git a/alembic/versions/0050_external_link_host_toggles.py b/alembic/versions/0050_external_link_host_toggles.py deleted file mode 100644 index ac78e75..0000000 --- a/alembic/versions/0050_external_link_host_toggles.py +++ /dev/null @@ -1,38 +0,0 @@ -"""import_settings: per-host enable toggles for external file-host downloads - -Operator levers (#830): disable a single host (e.g. mega.nz when it's -rate-limiting/banning) without touching the others. The worker reads these via -getattr and defaults to enabled, so the toggles default TRUE (works out of the -box, rule #26). - -Revision ID: 0050 -Revises: 0049 -Create Date: 2026-06-14 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0050" -down_revision: Union[str, None] = "0049" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain") - - -def upgrade() -> None: - for host in _HOSTS: - op.add_column( - "import_settings", - sa.Column( - f"extdl_{host}_enabled", sa.Boolean(), nullable=False, - server_default=sa.true(), - ), - ) - - -def downgrade() -> None: - for host in _HOSTS: - op.drop_column("import_settings", f"extdl_{host}_enabled") diff --git a/alembic/versions/0051_image_source_provenance.py b/alembic/versions/0051_image_source_provenance.py deleted file mode 100644 index 595077d..0000000 --- a/alembic/versions/0051_image_source_provenance.py +++ /dev/null @@ -1,38 +0,0 @@ -"""image_record: source_url + source_filehash (inline-image localization) - -#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline -images instead of hotlinking the public CDN. The join key between a body -`` and the local file is the CDN's 32-hex filehash (the same -identity extract_media dedups by). Persist it (indexed) plus the full source -URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing -rows — those fall back to hotlinking until re-downloaded. - -Revision ID: 0051 -Revises: 0050 -Create Date: 2026-06-14 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0051" -down_revision: Union[str, None] = "0050" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True)) - op.add_column( - "image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True) - ) - op.create_index( - "ix_image_record_source_filehash", "image_record", ["source_filehash"] - ) - - -def downgrade() -> None: - op.drop_index("ix_image_record_source_filehash", table_name="image_record") - op.drop_column("image_record", "source_filehash") - op.drop_column("image_record", "source_url") diff --git a/alembic/versions/0052_image_duration_seconds.py b/alembic/versions/0052_image_duration_seconds.py deleted file mode 100644 index ec2a180..0000000 --- a/alembic/versions/0052_image_duration_seconds.py +++ /dev/null @@ -1,32 +0,0 @@ -"""image_record: duration_seconds (Tier-1 video near-dup key) - -#871. Videos previously deduped on sha256 only (pHash is images-only), so a -different encode/remux of the same video imported as a distinct record. Persist -the container duration so the importer can treat same-artist videos with matching -duration (+ aspect ratio) as the same content and dedup/supersede like images. -NULL for images and for video rows imported before this column existed (a -backfill re-probes those so they participate in dedup). - -Revision ID: 0052 -Revises: 0051 -Create Date: 2026-06-16 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0052" -down_revision: Union[str, None] = "0051" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "image_record", sa.Column("duration_seconds", sa.Float(), nullable=True) - ) - - -def downgrade() -> None: - op.drop_column("image_record", "duration_seconds") diff --git a/alembic/versions/0053_ml_settings_video_tagging.py b/alembic/versions/0053_ml_settings_video_tagging.py deleted file mode 100644 index 1f192a4..0000000 --- a/alembic/versions/0053_ml_settings_video_tagging.py +++ /dev/null @@ -1,49 +0,0 @@ -"""ml_settings: video tagging knobs (cadence sampling + noise floor) - -#747. Video tag quality/perf: sample frames at a fixed cadence (interval) so a -tag's frame-presence reflects real screen time, cap total frames so long videos -stay bounded, and keep a tag only if it appears in >= min_tag_frames sampled -frames. Operator-tunable via Settings → ML (replaces the VIDEO_ML_FRAMES env var). - -Revision ID: 0053 -Revises: 0052 -Create Date: 2026-06-16 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0053" -down_revision: Union[str, None] = "0052" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "video_frame_interval_seconds", sa.Float(), nullable=False, - server_default="4.0", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "video_max_frames", sa.Integer(), nullable=False, server_default="64", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "video_min_tag_frames", sa.Integer(), nullable=False, - server_default="3", - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "video_min_tag_frames") - op.drop_column("ml_settings", "video_max_frames") - op.drop_column("ml_settings", "video_frame_interval_seconds") diff --git a/alembic/versions/0054_subscribestar_ledgers.py b/alembic/versions/0054_subscribestar_ledgers.py deleted file mode 100644 index 59972ae..0000000 --- a/alembic/versions/0054_subscribestar_ledgers.py +++ /dev/null @@ -1,82 +0,0 @@ -"""subscribestar_seen_media + subscribestar_failed_media: per-source ledgers - -Revision ID: 0054 -Revises: 0053 -Create Date: 2026-06-17 - -SubscribeStar native ingester (phase 1 of the gallery-dl → native-core -migration). Mirrors the Patreon ledger tables (0037/0038): a seen-ledger so -routine walks skip already-ingested media (recovery bypasses it) and a -dead-letter ledger so persistently-failing media stops re-burning backfill -chunks. `filehash` is a CDN content hash when present, else a synthesized -``:`` key — hence String(128). UNIQUE (source_id, filehash) -is the upsert key on each. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0054" -down_revision: Union[str, None] = "0053" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "subscribestar_seen_media", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "source_id", - sa.Integer, - sa.ForeignKey("source.id", ondelete="CASCADE"), - nullable=False, - index=True, - ), - sa.Column("filehash", sa.String(128), nullable=False), - sa.Column("post_id", sa.String(64), nullable=True), - sa.Column( - "seen_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.UniqueConstraint( - "source_id", "filehash", name="uq_subscribestar_seen_media_source_id" - ), - ) - op.create_table( - "subscribestar_failed_media", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "source_id", - sa.Integer, - sa.ForeignKey("source.id", ondelete="CASCADE"), - nullable=False, - index=True, - ), - sa.Column("filehash", sa.String(128), nullable=False), - sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), - sa.Column("last_error", sa.Text, nullable=True), - sa.Column( - "first_failed_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.Column( - "last_failed_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.UniqueConstraint( - "source_id", "filehash", name="uq_subscribestar_failed_media_source_id" - ), - ) - - -def downgrade() -> None: - op.drop_table("subscribestar_failed_media") - op.drop_table("subscribestar_seen_media") diff --git a/alembic/versions/0055_image_provenance_from_attachment.py b/alembic/versions/0055_image_provenance_from_attachment.py deleted file mode 100644 index 8b2566b..0000000 --- a/alembic/versions/0055_image_provenance_from_attachment.py +++ /dev/null @@ -1,55 +0,0 @@ -"""image_provenance: from_attachment_id (which archive an image was extracted from) - -Milestone #87. When an image is pulled out of a .zip/.rar, record WHICH archive -PostAttachment it came from, so the provenance UI can show the single archive a -file lives inside instead of every attachment on the post. Nullable FK with -ON DELETE SET NULL — a loose (non-archive) download leaves it NULL, and deleting -the archive attachment forgets the linkage without destroying the (image, post) -provenance edge. Existing rows are NULL until the reextract backfill stamps them. - -Revision ID: 0055 -Revises: 0054 -Create Date: 2026-06-22 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0055" -down_revision: Union[str, None] = "0054" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "image_provenance", - sa.Column("from_attachment_id", sa.Integer(), nullable=True), - ) - op.create_index( - "ix_image_provenance_from_attachment_id", - "image_provenance", - ["from_attachment_id"], - ) - op.create_foreign_key( - "fk_image_provenance_from_attachment", - "image_provenance", - "post_attachment", - ["from_attachment_id"], - ["id"], - ondelete="SET NULL", - ) - - -def downgrade() -> None: - op.drop_constraint( - "fk_image_provenance_from_attachment", - "image_provenance", - type_="foreignkey", - ) - op.drop_index( - "ix_image_provenance_from_attachment_id", - table_name="image_provenance", - ) - op.drop_column("image_provenance", "from_attachment_id") diff --git a/alembic/versions/0056_tag_eval_run.py b/alembic/versions/0056_tag_eval_run.py deleted file mode 100644 index 7d8e91f..0000000 --- a/alembic/versions/0056_tag_eval_run.py +++ /dev/null @@ -1,43 +0,0 @@ -"""tag_eval_run: persisted head-vs-centroid tagging eval runs (#1130) - -Milestone #114 slice 1. A long ml-queue eval whose full report must SURVIVE -navigation, so the run + report live in a row the admin card rehydrates from -(mirrors library_audit_run). running -> ready / error. - -Revision ID: 0056 -Revises: 0055 -Create Date: 2026-06-28 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects.postgresql import JSONB - -revision: str = "0056" -down_revision: Union[str, None] = "0055" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "tag_eval_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("params", JSONB(), nullable=False), - sa.Column("status", sa.String(length=16), nullable=False, server_default="running"), - sa.Column( - "started_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("report", JSONB(), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), - ) - op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"]) - - -def downgrade() -> None: - op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run") - op.drop_table("tag_eval_run") diff --git a/alembic/versions/0057_tag_positive_confirmation.py b/alembic/versions/0057_tag_positive_confirmation.py deleted file mode 100644 index 92335c2..0000000 --- a/alembic/versions/0057_tag_positive_confirmation.py +++ /dev/null @@ -1,40 +0,0 @@ -"""tag_positive_confirmation: operator-affirmed correct positives (#1130) - -Mirror of tag_suggestion_rejection. "Keep" on a doubted positive records here so -the eval's doubts list stops resurfacing confirmed-correct images every run. - -Revision ID: 0057 -Revises: 0056 -Create Date: 2026-06-28 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0057" -down_revision: Union[str, None] = "0056" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "tag_positive_confirmation", - sa.Column( - "image_record_id", sa.Integer(), - sa.ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True, - ), - sa.Column( - "tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True, - ), - sa.Column( - "confirmed_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - ) - - -def downgrade() -> None: - op.drop_table("tag_positive_confirmation") diff --git a/alembic/versions/0058_tag_head.py b/alembic/versions/0058_tag_head.py deleted file mode 100644 index 7ff45f6..0000000 --- a/alembic/versions/0058_tag_head.py +++ /dev/null @@ -1,95 +0,0 @@ -"""tag_head + head_training_run: production heads that learn from tags (#114) - -The eval (#1130) proved the frozen-embedding + trained-head spine; this lands its -production form. tag_head stores one logistic-regression head per concept (the -new suggestion source, replacing Camie + centroid); head_training_run tracks the -batch that (re)trains them. Adds two head-training tunables to ml_settings. - -Revision ID: 0058 -Revises: 0057 -Create Date: 2026-06-28 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from pgvector.sqlalchemy import Vector -from sqlalchemy.dialects.postgresql import JSONB - -revision: str = "0058" -down_revision: Union[str, None] = "0057" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_HEAD_DIM = 1152 - - -def upgrade() -> None: - op.create_table( - "tag_head", - sa.Column( - "tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, - ), - sa.Column("embedding_version", sa.String(length=128), nullable=False), - sa.Column("weights", Vector(_HEAD_DIM), nullable=False), - sa.Column("bias", sa.Float(), nullable=False), - sa.Column("suggest_threshold", sa.Float(), nullable=False), - sa.Column("auto_apply_threshold", sa.Float(), nullable=True), - sa.Column("n_pos", sa.Integer(), nullable=False), - sa.Column("n_neg", sa.Integer(), nullable=False), - sa.Column("ap", sa.Float(), nullable=False), - sa.Column("precision_cv", sa.Float(), nullable=False), - sa.Column("recall", sa.Float(), nullable=False), - sa.Column( - "trained_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("metrics", JSONB(), nullable=True), - ) - - op.create_table( - "head_training_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("params", JSONB(), nullable=False), - sa.Column( - "status", sa.String(length=16), nullable=False, - server_default="running", - ), - sa.Column( - "started_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("n_trained", sa.Integer(), nullable=True), - sa.Column("n_skipped", sa.Integer(), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), - ) - op.create_index( - "ix_head_training_run_status", "head_training_run", ["status"], - ) - - # Head-training tunables on the ml_settings singleton. - op.add_column( - "ml_settings", - sa.Column( - "head_min_positives", sa.Integer(), nullable=False, - server_default="8", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "head_auto_apply_precision", sa.Float(), nullable=False, - server_default="0.97", - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "head_auto_apply_precision") - op.drop_column("ml_settings", "head_min_positives") - op.drop_index("ix_head_training_run_status", table_name="head_training_run") - op.drop_table("head_training_run") - op.drop_table("tag_head") diff --git a/alembic/versions/0059_head_auto_apply.py b/alembic/versions/0059_head_auto_apply.py deleted file mode 100644 index d0bb9b8..0000000 --- a/alembic/versions/0059_head_auto_apply.py +++ /dev/null @@ -1,70 +0,0 @@ -"""head_auto_apply_run + earned-auto-apply settings (#114) - -A graduated head can apply its tag without a human, gated by a master switch + -a support floor. head_auto_apply_run tracks each sweep / dry-run preview. - -Revision ID: 0059 -Revises: 0058 -Create Date: 2026-06-29 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects.postgresql import JSONB - -revision: str = "0059" -down_revision: Union[str, None] = "0058" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "head_auto_apply_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "dry_run", sa.Boolean(), nullable=False, server_default=sa.false() - ), - sa.Column("params", JSONB(), nullable=False), - sa.Column( - "status", sa.String(length=16), nullable=False, - server_default="running", - ), - sa.Column( - "started_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("n_applied", sa.Integer(), nullable=True), - sa.Column("report", JSONB(), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), - ) - op.create_index( - "ix_head_auto_apply_run_status", "head_auto_apply_run", ["status"], - ) - - op.add_column( - "ml_settings", - sa.Column( - "head_auto_apply_enabled", sa.Boolean(), nullable=False, - server_default=sa.true(), # opt-out: on by default (operator-asked) - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "head_auto_apply_min_positives", sa.Integer(), nullable=False, - server_default="30", - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "head_auto_apply_min_positives") - op.drop_column("ml_settings", "head_auto_apply_enabled") - op.drop_index( - "ix_head_auto_apply_run_status", table_name="head_auto_apply_run" - ) - op.drop_table("head_auto_apply_run") diff --git a/alembic/versions/0060_head_metrics.py b/alembic/versions/0060_head_metrics.py deleted file mode 100644 index e94edb8..0000000 --- a/alembic/versions/0060_head_metrics.py +++ /dev/null @@ -1,74 +0,0 @@ -"""head_metric + head_metrics_snapshot: auto-apply observability (#114) - -Running misfire/under-fire counters per concept (captured at correction time, -since image_tag.source is lost on delete) + a daily per-concept time-series so -the operator can tune the precision target + support floor from real data. - -Revision ID: 0060 -Revises: 0059 -Create Date: 2026-06-29 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0060" -down_revision: Union[str, None] = "0059" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "head_metric", - sa.Column( - "tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, - ), - sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"), - sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"), - sa.Column( - "updated_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - ) - - op.create_table( - "head_metrics_snapshot", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="CASCADE"), - ), - sa.Column("name", sa.String(length=255), nullable=False), - sa.Column( - "snapshot_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("n_auto_applied", sa.Integer(), nullable=False, server_default="0"), - sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"), - sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"), - sa.Column("ap", sa.Float(), nullable=True), - sa.Column("precision_cv", sa.Float(), nullable=True), - sa.Column("recall", sa.Float(), nullable=True), - sa.Column("n_pos", sa.Integer(), nullable=True), - ) - op.create_index( - "ix_head_metrics_snapshot_tag_id", "head_metrics_snapshot", ["tag_id"], - ) - op.create_index( - "ix_head_metrics_snapshot_snapshot_at", "head_metrics_snapshot", - ["snapshot_at"], - ) - - -def downgrade() -> None: - op.drop_index( - "ix_head_metrics_snapshot_snapshot_at", table_name="head_metrics_snapshot" - ) - op.drop_index( - "ix_head_metrics_snapshot_tag_id", table_name="head_metrics_snapshot" - ) - op.drop_table("head_metrics_snapshot") - op.drop_table("head_metric") diff --git a/alembic/versions/0061_image_region.py b/alembic/versions/0061_image_region.py deleted file mode 100644 index b3af8a9..0000000 --- a/alembic/versions/0061_image_region.py +++ /dev/null @@ -1,59 +0,0 @@ -"""image_region: detected/proposed regions + their crop embeddings (#114) - -Storage backbone of the crop pipeline. A region = normalized bbox + the crop's -embedding (CCIP for face/figure → character id; SigLIP for concept regions → -head bag-of-embeddings). Also serves as grounded-tag bbox provenance. - -Revision ID: 0061 -Revises: 0060 -Create Date: 2026-06-29 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from pgvector.sqlalchemy import Vector - -revision: str = "0061" -down_revision: Union[str, None] = "0060" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_CCIP_DIM = 768 -_SIGLIP_DIM = 1152 - - -def upgrade() -> None: - op.create_table( - "image_region", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "image_record_id", sa.Integer(), - sa.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, - ), - sa.Column("kind", sa.String(length=16), nullable=False), - # Video/animated: source frame timestamp (seconds); NULL for stills. - sa.Column("frame_time", sa.Float(), nullable=True), - sa.Column("rx", sa.Float(), nullable=False), - sa.Column("ry", sa.Float(), nullable=False), - sa.Column("rw", sa.Float(), nullable=False), - sa.Column("rh", sa.Float(), nullable=False), - sa.Column("score", sa.Float(), nullable=True), - sa.Column("detector_version", sa.String(length=64), nullable=True), - sa.Column("crop_version", sa.String(length=64), nullable=True), - sa.Column("embedding_version", sa.String(length=128), nullable=True), - sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=True), - sa.Column("siglip_embedding", Vector(_SIGLIP_DIM), nullable=True), - sa.Column( - "created_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - ) - op.create_index( - "ix_image_region_image_record_id", "image_region", ["image_record_id"], - ) - - -def downgrade() -> None: - op.drop_index("ix_image_region_image_record_id", table_name="image_region") - op.drop_table("image_region") diff --git a/alembic/versions/0062_gpu_job.py b/alembic/versions/0062_gpu_job.py deleted file mode 100644 index a044995..0000000 --- a/alembic/versions/0062_gpu_job.py +++ /dev/null @@ -1,55 +0,0 @@ -"""gpu_job: the HTTP-leased GPU work queue for the desktop agent (#114) - -The agent stays HTTP-only — the server enqueues per-(image, task) jobs here and -the agent leases/submits over the web API; Redis/Postgres stay private. - -Revision ID: 0062 -Revises: 0061 -Create Date: 2026-06-29 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0062" -down_revision: Union[str, None] = "0061" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "gpu_job", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "image_record_id", sa.Integer(), - sa.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, - ), - sa.Column("task", sa.String(length=32), nullable=False), - sa.Column( - "status", sa.String(length=16), nullable=False, - server_default="pending", - ), - sa.Column("lease_token", sa.String(length=64), nullable=True), - sa.Column("leased_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), - sa.Column("error", sa.Text(), nullable=True), - sa.Column( - "created_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column( - "updated_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - ) - op.create_index("ix_gpu_job_image_record_id", "gpu_job", ["image_record_id"]) - op.create_index("ix_gpu_job_status", "gpu_job", ["status"]) - - -def downgrade() -> None: - op.drop_index("ix_gpu_job_status", table_name="gpu_job") - op.drop_index("ix_gpu_job_image_record_id", table_name="gpu_job") - op.drop_table("gpu_job") diff --git a/alembic/versions/0063_ccip_match_threshold.py b/alembic/versions/0063_ccip_match_threshold.py deleted file mode 100644 index d841398..0000000 --- a/alembic/versions/0063_ccip_match_threshold.py +++ /dev/null @@ -1,33 +0,0 @@ -"""ml_settings.ccip_match_threshold — tunable CCIP character-match cut (#114) - -The v1 matcher used a flat 0.75 cosine; live data showed that over-fires (a -high-reference character matched a scatter of images). 0.85 keeps the confident -single-character matches and drops the noise. Tunable from the GPU agent card. - -Revision ID: 0063 -Revises: 0062 -Create Date: 2026-06-29 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0063" -down_revision: Union[str, None] = "0062" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "ccip_match_threshold", sa.Float(), nullable=False, - server_default="0.85", - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "ccip_match_threshold") diff --git a/alembic/versions/0064_ccip_auto_apply.py b/alembic/versions/0064_ccip_auto_apply.py deleted file mode 100644 index e5323cf..0000000 --- a/alembic/versions/0064_ccip_auto_apply.py +++ /dev/null @@ -1,42 +0,0 @@ -"""ml_settings: CCIP auto-apply switch + threshold (#114) - -Confident CCIP character matches auto-tag (source='ccip_auto') on a daily sweep, -so identity tags keep flowing without pressing a button. ON by default (opt-out, -like head auto-apply); the high threshold (0.92, above the 0.85 suggest cut) + -single-character references keep it safe, and every auto-tag is reversible. - -Revision ID: 0064 -Revises: 0063 -Create Date: 2026-06-30 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0064" -down_revision: Union[str, None] = "0063" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "ccip_auto_apply_enabled", sa.Boolean(), nullable=False, - server_default=sa.true(), - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "ccip_auto_apply_threshold", sa.Float(), nullable=False, - server_default="0.92", - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "ccip_auto_apply_threshold") - op.drop_column("ml_settings", "ccip_auto_apply_enabled") diff --git a/alembic/versions/0065_embedder_model_name.py b/alembic/versions/0065_embedder_model_name.py deleted file mode 100644 index 0a986b3..0000000 --- a/alembic/versions/0065_embedder_model_name.py +++ /dev/null @@ -1,35 +0,0 @@ -"""ml_settings: embedder_model_name (#1190 operator model swap) - -The embedder MODEL VERSION was already a setting (and stamps image_record. -siglip_model_version); the HF model NAME was env-only, so an operator couldn't -actually point the pipeline at a different embedder. Storing the name as a -setting makes the model an operator choice: set name + version → re-embed (the -GPU agent) → retrain heads. Default = the current SigLIP so400m. - -Revision ID: 0065 -Revises: 0064 -Create Date: 2026-06-30 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0065" -down_revision: Union[str, None] = "0064" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "embedder_model_name", sa.String(length=128), nullable=False, - server_default="google/siglip-so400m-patch14-384", - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "embedder_model_name") diff --git a/alembic/versions/0066_drop_centroids.py b/alembic/versions/0066_drop_centroids.py deleted file mode 100644 index d75a334..0000000 --- a/alembic/versions/0066_drop_centroids.py +++ /dev/null @@ -1,57 +0,0 @@ -"""drop the dead per-tag centroid subsystem (#1189 cleanup) - -The v2 pivot replaced per-tag SigLIP centroids with learned heads + CCIP. -Nothing read the centroids anymore — they were recomputed (on merge + a daily -beat) but never consumed for suggestions or auto-apply. Remove the storage + -its two now-unused settings columns. (The recompute tasks, beat, endpoint, -service, and UI card are removed in the same change.) - -Revision ID: 0066 -Revises: 0065 -Create Date: 2026-06-30 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0066" -down_revision: Union[str, None] = "0065" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_table("tag_reference_embedding") - op.drop_column("ml_settings", "centroid_similarity_threshold") - op.drop_column("ml_settings", "min_reference_images") - - -def downgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "min_reference_images", sa.Integer(), nullable=False, - server_default="5", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "centroid_similarity_threshold", sa.Float(), nullable=False, - server_default="0.55", - ), - ) - op.create_table( - "tag_reference_embedding", - sa.Column("tag_id", sa.Integer(), nullable=False), - sa.Column("embedding", sa.LargeBinary(), nullable=False), - sa.Column("reference_count", sa.Integer(), nullable=False), - sa.Column("model_version", sa.String(length=128), nullable=False), - sa.Column( - "updated_at", sa.DateTime(timezone=True), - server_default=sa.func.now(), nullable=False, - ), - sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"), - sa.PrimaryKeyConstraint("tag_id"), - ) diff --git a/alembic/versions/0067_retire_camie_allowlist.py b/alembic/versions/0067_retire_camie_allowlist.py deleted file mode 100644 index e3edd02..0000000 --- a/alembic/versions/0067_retire_camie_allowlist.py +++ /dev/null @@ -1,66 +0,0 @@ -"""retire the Camie tagger + allowlist bulk-apply (#1189) - -The v2 pivot made heads + CCIP the tag source and head auto-apply the earned -propagation. The Camie tagger ran only to feed the allowlist bulk-apply (its -predictions had no other consumer), and the allowlist was a second, un-earned -auto-apply path parallel to heads. Both are retired — drop their storage. - -(image_prediction = Camie's per-image predictions; tag_allowlist = the bulk- -apply allowlist. Nothing references INTO these tables, so the drop is clean.) - -Revision ID: 0067 -Revises: 0066 -Create Date: 2026-06-30 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0067" -down_revision: Union[str, None] = "0066" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_table("image_prediction") - op.drop_table("tag_allowlist") - - -def downgrade() -> None: - op.create_table( - "tag_allowlist", - sa.Column("tag_id", sa.Integer(), nullable=False), - sa.Column( - "min_confidence", sa.Float(), nullable=False, server_default="0.9" - ), - sa.Column( - "created_at", sa.DateTime(timezone=True), - server_default=sa.func.now(), nullable=False, - ), - sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"), - sa.PrimaryKeyConstraint("tag_id"), - sa.CheckConstraint( - "min_confidence >= 0 AND min_confidence <= 1", - name="ck_tag_allowlist_confidence_range", - ), - ) - op.create_table( - "image_prediction", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("image_record_id", sa.Integer(), nullable=False), - sa.Column("raw_name", sa.String(length=255), nullable=False), - sa.Column("category", sa.String(length=32), nullable=False), - sa.Column("score", sa.Float(), nullable=False), - sa.ForeignKeyConstraint( - ["image_record_id"], ["image_record.id"], ondelete="CASCADE" - ), - ) - op.create_index( - "ix_image_prediction_image", "image_prediction", ["image_record_id"] - ) - op.create_index( - "ix_image_prediction_name_score", "image_prediction", - ["raw_name", "score"], - ) diff --git a/alembic/versions/0068_drop_dead_tagger_settings.py b/alembic/versions/0068_drop_dead_tagger_settings.py deleted file mode 100644 index 770676d..0000000 --- a/alembic/versions/0068_drop_dead_tagger_settings.py +++ /dev/null @@ -1,80 +0,0 @@ -"""drop dead tagger/suggestion settings + columns left after Camie retirement (#1199) - -Hygiene follow-up to #1189. These were left inert to bound that change; nothing -reads them now: -- ml_settings: tagger_store_floor + tagger_model_version (only the deleted Camie - tagger used them), suggestion_threshold_character/general (already dead pre- - retirement — scoring uses per-head thresholds), video_min_tag_frames (only the - deleted video-prediction aggregator used it). -- image_record: tagger_model_version (no writer now), centroid_scores (long-dead - JSON cache, no reader). - -Revision ID: 0068 -Revises: 0067 -Create Date: 2026-06-30 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0068" -down_revision: Union[str, None] = "0067" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_column("ml_settings", "suggestion_threshold_character") - op.drop_column("ml_settings", "suggestion_threshold_general") - op.drop_column("ml_settings", "tagger_store_floor") - op.drop_column("ml_settings", "video_min_tag_frames") - op.drop_column("ml_settings", "tagger_model_version") - op.drop_column("image_record", "tagger_model_version") - op.drop_column("image_record", "centroid_scores") - - -def downgrade() -> None: - op.add_column( - "image_record", - sa.Column("centroid_scores", sa.JSON(), nullable=True), - ) - op.add_column( - "image_record", - sa.Column("tagger_model_version", sa.String(length=128), nullable=True), - ) - op.add_column( - "ml_settings", - sa.Column( - "tagger_model_version", sa.String(length=128), nullable=False, - server_default="camie-tagger-v2", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "video_min_tag_frames", sa.Integer(), nullable=False, - server_default="3", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "tagger_store_floor", sa.Float(), nullable=False, - server_default="0.7", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "suggestion_threshold_general", sa.Float(), nullable=False, - server_default="0.7", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "suggestion_threshold_character", sa.Float(), nullable=False, - server_default="0.7", - ), - ) diff --git a/alembic/versions/0069_default_siglip2.py b/alembic/versions/0069_default_siglip2.py deleted file mode 100644 index 7bef8b1..0000000 --- a/alembic/versions/0069_default_siglip2.py +++ /dev/null @@ -1,51 +0,0 @@ -"""default the embedder to SigLIP 2 — for FRESH installs only (#1203) - -Make SigLIP 2 (so400m, 512px; a 1152-d drop-in) the default embedder. New -installs start on it. An EXISTING library is NOT touched: flipping its stored -embedder version would mark every embedding stale (the scorer is version-gated) -and kill suggestions until a full re-embed+retrain — so an existing instance -switches deliberately via Settings → GPU agent → Embedding model → Re-embed → -Retrain. We detect "fresh" by the absence of any embedded image. - -Revision ID: 0069 -Revises: 0068 -Create Date: 2026-06-30 -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0069" -down_revision: Union[str, None] = "0068" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -_NEW_NAME = "google/siglip2-so400m-patch16-512" -_NEW_VERSION = "siglip2-so400m-patch16-512" -_OLD_NAME = "google/siglip-so400m-patch14-384" -_OLD_VERSION = "siglip-so400m-patch14-384" - - -def upgrade() -> None: - # Fresh install (nothing embedded yet) → adopt SigLIP 2. - op.execute( - f""" - UPDATE ml_settings SET - embedder_model_name = '{_NEW_NAME}', - embedder_model_version = '{_NEW_VERSION}' - WHERE NOT EXISTS ( - SELECT 1 FROM image_record WHERE siglip_embedding IS NOT NULL - ) - """ - ) - op.alter_column("ml_settings", "embedder_model_name", server_default=_NEW_NAME) - op.alter_column( - "ml_settings", "embedder_model_version", server_default=_NEW_VERSION - ) - - -def downgrade() -> None: - op.alter_column("ml_settings", "embedder_model_name", server_default=_OLD_NAME) - op.alter_column( - "ml_settings", "embedder_model_version", server_default=_OLD_VERSION - ) diff --git a/alembic/versions/0070_gpu_job_lease_indexes.py b/alembic/versions/0070_gpu_job_lease_indexes.py deleted file mode 100644 index 10ec3f9..0000000 --- a/alembic/versions/0070_gpu_job_lease_indexes.py +++ /dev/null @@ -1,44 +0,0 @@ -"""partial indexes so GPU-job leasing stays O(batch), not O(completed) - -The lease claims the lowest-id pending (or expired-leased) jobs. With only a -plain `status` index, `... ORDER BY id LIMIT n` walked the primary-key index from -the start, skipping the entire prefix of already-done/error rows before reaching -pending ones — so leasing slowed to a crawl as `done` piled up (the whole reason -throughput fell off a cliff mid-run and /status stalled). Two partial indexes fix -it: the pending one is id-ordered so the hot path reads just the first n entries, -and the leased-expiry one keeps the crash-recovery reclaim + the orphan sweep -cheap. They cover only the small live slice of the table, so they stay tiny even -as the done/error history grows to millions. - -Revision ID: 0070 -Revises: 0069 -Create Date: 2026-06-30 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0070" -down_revision: Union[str, None] = "0069" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Hot path: lowest-id pending jobs. Index on id, restricted to pending, so - # `WHERE status='pending' ORDER BY id LIMIT n` is a short index-order scan. - op.create_index( - "ix_gpu_job_pending", "gpu_job", ["id"], - postgresql_where=sa.text("status = 'pending'"), - ) - # Crash-recovery: expired leases, for the lease backstop + recover_orphaned. - op.create_index( - "ix_gpu_job_leased_expires", "gpu_job", ["lease_expires_at"], - postgresql_where=sa.text("status = 'leased'"), - ) - - -def downgrade() -> None: - op.drop_index("ix_gpu_job_leased_expires", table_name="gpu_job") - op.drop_index("ix_gpu_job_pending", table_name="gpu_job") diff --git a/alembic/versions/0071_image_record_earliest_post_date.py b/alembic/versions/0071_image_record_earliest_post_date.py deleted file mode 100644 index b2e8f0c..0000000 --- a/alembic/versions/0071_image_record_earliest_post_date.py +++ /dev/null @@ -1,80 +0,0 @@ -"""image_record.earliest_post_date: original-publish gallery sort key + index - -Revision ID: 0071 -Revises: 0070 -Create Date: 2026-07-01 - -effective_date (0035) keys off the PRIMARY post — which is often the repost / -download the file actually came from — and falls back to created_at, so the -gallery's default order surfaces download dates rather than when content was -first posted (operator-flagged 2026-07-01). Materialize a second sort key, -earliest_post_date = MIN(post_date) across ALL of an image's provenance posts -(every post it appears in), falling back to created_at only when no linked post -carries a date. Indexed (DESC, id DESC) so the "post date" gallery sort is an -index range scan just like effective_date. - -Backfill mirrors 0035: created_at baseline, then override with the MIN over -image_provenance ⋈ post. New rows get the created_at-equivalent server default; -services/importer.py recomputes it whenever a dated post is linked. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0071" -down_revision: Union[str, None] = "0070" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Add nullable first so the backfill can populate before NOT NULL. - op.add_column( - "image_record", - sa.Column("earliest_post_date", sa.DateTime(timezone=True), nullable=True), - ) - # Baseline: download date. Set-based (no per-row binds) → immune to the - # 65535 bind-parameter ceiling regardless of library size. - op.execute( - """ - UPDATE image_record - SET earliest_post_date = created_at - """ - ) - # Override with the earliest post_date across EVERY post the image appears - # in (image_provenance is the many-to-many edge; ignore posts with no date). - op.execute( - """ - UPDATE image_record AS ir - SET earliest_post_date = sub.min_date - FROM ( - SELECT ip.image_record_id AS iid, MIN(p.post_date) AS min_date - FROM image_provenance AS ip - JOIN post AS p ON p.id = ip.post_id - WHERE p.post_date IS NOT NULL - GROUP BY ip.image_record_id - ) AS sub - WHERE ir.id = sub.iid - """ - ) - op.alter_column( - "image_record", - "earliest_post_date", - nullable=False, - server_default=sa.text("now()"), - ) - # DESC/DESC matches the gallery's ORDER BY earliest_post_date DESC, id DESC - # so the "post date" scroll is a forward index scan; raw SQL because - # alembic's column list doesn't express per-column DESC cleanly. - op.execute( - "CREATE INDEX ix_image_record_earliest_post_date " - "ON image_record (earliest_post_date DESC, id DESC)" - ) - - -def downgrade() -> None: - op.drop_index( - "ix_image_record_earliest_post_date", table_name="image_record" - ) - op.drop_column("image_record", "earliest_post_date") diff --git a/alembic/versions/0072_gpu_job_triage_status.py b/alembic/versions/0072_gpu_job_triage_status.py deleted file mode 100644 index 1dce875..0000000 --- a/alembic/versions/0072_gpu_job_triage_status.py +++ /dev/null @@ -1,32 +0,0 @@ -"""gpu_job.triage_status — the probe's verdict on an errored job's FILE - -Failure triage (#125): a periodic sweep probes each errored image's file -(sha256 + decode, verify_integrity's machinery) exactly once and stores the -verdict here — 'defect' (the file is bad: recovery material, excluded from -/retry_errors) or 'file_ok' (failure was operational, safe to retry). NULL -means not yet probed; selecting on NULL is what makes the sweep resumable. -No index: the errored slice the sweep scans is tiny by design (tombstones). - -Revision ID: 0072 -Revises: 0071 -Create Date: 2026-07-02 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0072" -down_revision: Union[str, None] = "0071" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "gpu_job", sa.Column("triage_status", sa.String(16), nullable=True) - ) - - -def downgrade() -> None: - op.drop_column("gpu_job", "triage_status") diff --git a/alembic/versions/0073_drop_tag_eval_run.py b/alembic/versions/0073_drop_tag_eval_run.py deleted file mode 100644 index 4aedb38..0000000 --- a/alembic/versions/0073_drop_tag_eval_run.py +++ /dev/null @@ -1,46 +0,0 @@ -"""drop tag_eval_run — the head-vs-centroid eval harness is retired - -The eval (#1130) existed to prove the heads tagging spine on the operator's own -data. It did; the operator accepted the system and retired the harness -(2026-07-02) — card, API, task, model and this table all go. The eval's data -loaders + metric helpers live on in services/ml/training_data.py, where the -production heads trainer uses them nightly. - -Revision ID: 0073 -Revises: 0072 -Create Date: 2026-07-02 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -revision: str = "0073" -down_revision: Union[str, None] = "0072" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run") - op.drop_table("tag_eval_run") - - -def downgrade() -> None: - # Recreates the shape from 0056 (data is not restorable). - op.create_table( - "tag_eval_run", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column("params", postgresql.JSONB(), nullable=False), - sa.Column("status", sa.String(length=16), nullable=False, - server_default="running"), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now()), - sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("report", postgresql.JSONB(), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("last_progress_at", sa.DateTime(timezone=True), - nullable=True), - ) - op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"]) diff --git a/alembic/versions/0074_ml_settings_cpu_embed_enabled.py b/alembic/versions/0074_ml_settings_cpu_embed_enabled.py deleted file mode 100644 index 48ff8ea..0000000 --- a/alembic/versions/0074_ml_settings_cpu_embed_enabled.py +++ /dev/null @@ -1,35 +0,0 @@ -"""ml_settings.cpu_embed_enabled — the CPU embed fallback becomes a switch - -B3 (operator 2026-07-02): the ml-worker's only processing role is the CPU -whole-image embed for stacks without a GPU agent. ON by default (a fresh -install works agent-less); agent-equipped stacks that drop the ml-worker -container turn it off so import hooks stop queueing embed work into a queue -nothing consumes — the daily GPU 'embed' backfill covers those images. - -Revision ID: 0074 -Revises: 0073 -Create Date: 2026-07-02 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0074" -down_revision: Union[str, None] = "0073" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "cpu_embed_enabled", sa.Boolean(), nullable=False, - server_default=sa.true(), - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "cpu_embed_enabled") diff --git a/alembic/versions/0075_tag_is_system.py b/alembic/versions/0075_tag_is_system.py deleted file mode 100644 index a6b7e7a..0000000 --- a/alembic/versions/0075_tag_is_system.py +++ /dev/null @@ -1,60 +0,0 @@ -"""tag.is_system + seed the three hygiene system tags - -Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a -character poison that character's head and CCIP references; banners/editor -screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the -product ships — not operator configuration — so the seed lives here. - -Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive, -matching TagService.rename's collision stance) instead of inserting a -duplicate, so an operator who already tagged `wip` keeps their applications. - -Revision ID: 0075 -Revises: 0074 -Create Date: 2026-07-03 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0075" -down_revision: Union[str, None] = "0074" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") - - -def upgrade() -> None: - op.add_column( - "tag", - sa.Column( - "is_system", sa.Boolean(), nullable=False, - server_default=sa.false(), - ), - ) - conn = op.get_bind() - for name in SYSTEM_TAG_NAMES: - adopted = conn.execute( - sa.text( - "UPDATE tag SET is_system = true " - "WHERE lower(name) = lower(:name) AND kind = 'general'" - ), - {"name": name}, - ) - if adopted.rowcount == 0: - conn.execute( - sa.text( - "INSERT INTO tag (name, kind, is_system) " - "VALUES (:name, 'general', true)" - ), - {"name": name}, - ) - - -def downgrade() -> None: - # The seeded rows survive as ordinary general tags — dropping the flag is - # enough to disarm the mechanism, and deleting rows would orphan any - # operator applications made while the flag existed. - op.drop_column("tag", "is_system") diff --git a/alembic/versions/0076_pixiv_ledgers.py b/alembic/versions/0076_pixiv_ledgers.py deleted file mode 100644 index 2655130..0000000 --- a/alembic/versions/0076_pixiv_ledgers.py +++ /dev/null @@ -1,82 +0,0 @@ -"""pixiv_seen_media + pixiv_failed_media: per-source ledgers - -Revision ID: 0076 -Revises: 0075 -Create Date: 2026-07-03 - -Pixiv native ingester (milestone #129, gallery-dl → native-core migration). -Mirrors the Patreon (0037/0038) and SubscribeStar (0054) ledger tables: a -seen-ledger so routine walks skip already-ingested media (recovery bypasses -it) and a dead-letter ledger so persistently-failing media stops re-burning -backfill chunks. Pixiv URLs carry no content hash, so `filehash` is always the -synthesized ``:p`` / ``:ugoira`` key — String(128) -matches the siblings. UNIQUE (source_id, filehash) is the upsert key on each. -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0076" -down_revision: Union[str, None] = "0075" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "pixiv_seen_media", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "source_id", - sa.Integer, - sa.ForeignKey("source.id", ondelete="CASCADE"), - nullable=False, - index=True, - ), - sa.Column("filehash", sa.String(128), nullable=False), - sa.Column("post_id", sa.String(64), nullable=True), - sa.Column( - "seen_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.UniqueConstraint( - "source_id", "filehash", name="uq_pixiv_seen_media_source_id" - ), - ) - op.create_table( - "pixiv_failed_media", - sa.Column("id", sa.Integer, primary_key=True), - sa.Column( - "source_id", - sa.Integer, - sa.ForeignKey("source.id", ondelete="CASCADE"), - nullable=False, - index=True, - ), - sa.Column("filehash", sa.String(128), nullable=False), - sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), - sa.Column("last_error", sa.Text, nullable=True), - sa.Column( - "first_failed_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.Column( - "last_failed_at", - sa.DateTime(timezone=True), - nullable=False, - server_default=sa.text("NOW()"), - ), - sa.UniqueConstraint( - "source_id", "filehash", name="uq_pixiv_failed_media_source_id" - ), - ) - - -def downgrade() -> None: - op.drop_table("pixiv_failed_media") - op.drop_table("pixiv_seen_media") diff --git a/alembic/versions/0077_artist_name_not_unique.py b/alembic/versions/0077_artist_name_not_unique.py deleted file mode 100644 index 6a09288..0000000 --- a/alembic/versions/0077_artist_name_not_unique.py +++ /dev/null @@ -1,32 +0,0 @@ -"""drop uq_artist_name — decouple display name from identity/storage - -Revision ID: 0077 -Revises: 0076 -Create Date: 2026-07-04 - -Artist model fragility fix (milestone #130). One `slug` column was doing -identity + storage-path + display, and BOTH `name` and `slug` were UNIQUE, so -the display name couldn't be edited freely and two genuinely different creators -collided. Decouple: `slug` stays the immutable, unique storage/identity key (the -on-disk path component — untouched here); `name` becomes freely editable, NON- -unique display text. This migration only drops the `uq_artist_name` constraint; -no data moves and no path changes. -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0077" -down_revision: Union[str, None] = "0076" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.drop_constraint("uq_artist_name", "artist", type_="unique") - - -def downgrade() -> None: - # Re-adding the UNIQUE would fail if duplicate names now exist; callers that - # need to reverse this must dedupe names first. - op.create_unique_constraint("uq_artist_name", "artist", ["name"]) diff --git a/alembic/versions/0078_ml_settings_detectors.py b/alembic/versions/0078_ml_settings_detectors.py deleted file mode 100644 index 6d04601..0000000 --- a/alembic/versions/0078_ml_settings_detectors.py +++ /dev/null @@ -1,83 +0,0 @@ -"""ml_settings crop-proposer / detector config (#134) - -Move the WHERE-to-crop detector config (per-proposer enable + weights + conf, -plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU -agent in the lease (like the embedder model) — no restart, agent env is now -bootstrap-only. All server_defaults are the working values so existing rows + -fresh installs crop out-of-the-box with all three proposers ON. - -Revision ID: 0078 -Revises: 0077 -Create Date: 2026-07-05 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0078" -down_revision: Union[str, None] = "0077" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -_ANATOMY_DEFAULT = ( - "https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt" -) -_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt" - - -def upgrade() -> None: - op.add_column("ml_settings", sa.Column( - "detector_person_enabled", sa.Boolean(), nullable=False, - server_default=sa.true())) - op.add_column("ml_settings", sa.Column( - "detector_person_weights", sa.String(512), nullable=False, - server_default="yolo11n.pt")) - op.add_column("ml_settings", sa.Column( - "detector_person_conf", sa.Float(), nullable=False, - server_default=sa.text("0.35"))) - op.add_column("ml_settings", sa.Column( - "detector_anatomy_enabled", sa.Boolean(), nullable=False, - server_default=sa.true())) - op.add_column("ml_settings", sa.Column( - "detector_anatomy_weights", sa.String(512), nullable=False, - server_default=_ANATOMY_DEFAULT)) - op.add_column("ml_settings", sa.Column( - "detector_anatomy_conf", sa.Float(), nullable=False, - server_default=sa.text("0.30"))) - op.add_column("ml_settings", sa.Column( - "detector_panel_enabled", sa.Boolean(), nullable=False, - server_default=sa.true())) - op.add_column("ml_settings", sa.Column( - "detector_panel_weights", sa.String(512), nullable=False, - server_default=_PANEL_DEFAULT)) - op.add_column("ml_settings", sa.Column( - "detector_panel_conf", sa.Float(), nullable=False, - server_default=sa.text("0.30"))) - op.add_column("ml_settings", sa.Column( - "detector_max_figures", sa.Integer(), nullable=False, - server_default=sa.text("8"))) - op.add_column("ml_settings", sa.Column( - "detector_max_components", sa.Integer(), nullable=False, - server_default=sa.text("8"))) - op.add_column("ml_settings", sa.Column( - "detector_max_panels", sa.Integer(), nullable=False, - server_default=sa.text("8"))) - op.add_column("ml_settings", sa.Column( - "detector_max_regions", sa.Integer(), nullable=False, - server_default=sa.text("128"))) - op.add_column("ml_settings", sa.Column( - "detector_dedupe_iou", sa.Float(), nullable=False, - server_default=sa.text("0.85"))) - - -def downgrade() -> None: - for col in ( - "detector_person_enabled", "detector_person_weights", "detector_person_conf", - "detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf", - "detector_panel_enabled", "detector_panel_weights", "detector_panel_conf", - "detector_max_figures", "detector_max_components", "detector_max_panels", - "detector_max_regions", "detector_dedupe_iou", - ): - op.drop_column("ml_settings", col) diff --git a/alembic/versions/0079_character_prototypes.py b/alembic/versions/0079_character_prototypes.py deleted file mode 100644 index 8ada2f4..0000000 --- a/alembic/versions/0079_character_prototypes.py +++ /dev/null @@ -1,77 +0,0 @@ -"""character prototype store (#1317) — precomputed, incremental CCIP references - -New tables character_prototype + ccip_prototype_state, plus MLSettings columns -ccip_ref_signature (cheap global change gate) + ccip_prototype_cap (per-character -reference cap). The reference set the CCIP matcher uses becomes a precomputed -artifact refreshed incrementally off the request path. See milestone 138 / -backend.app.services.ml.character_prototypes. - -Revision ID: 0079 -Revises: 0078 -Create Date: 2026-07-06 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op -from pgvector.sqlalchemy import Vector - -revision: str = "0079" -down_revision: Union[str, None] = "0078" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - -# Matches models.image_region.CCIP_DIM (the CCIP figure-embedding width). -_CCIP_DIM = 768 - - -def upgrade() -> None: - op.create_table( - "character_prototype", - sa.Column("id", sa.Integer(), primary_key=True), - sa.Column( - "tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, - ), - sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=False), - sa.Column( - "region_id", sa.Integer(), - sa.ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True, - ), - ) - op.create_index( - "ix_character_prototype_tag_id", "character_prototype", ["tag_id"] - ) - op.create_table( - "ccip_prototype_state", - sa.Column( - "tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, - ), - sa.Column("fingerprint", sa.String(64), nullable=False), - sa.Column( - "updated_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - ) - op.add_column( - "ml_settings", - sa.Column("ccip_ref_signature", sa.String(128), nullable=True), - ) - op.add_column( - "ml_settings", - sa.Column( - "ccip_prototype_cap", sa.Integer(), nullable=False, - server_default=sa.text("64"), - ), - ) - - -def downgrade() -> None: - op.drop_column("ml_settings", "ccip_prototype_cap") - op.drop_column("ml_settings", "ccip_ref_signature") - op.drop_table("ccip_prototype_state") - op.drop_index( - "ix_character_prototype_tag_id", table_name="character_prototype" - ) - op.drop_table("character_prototype") diff --git a/alembic/versions/0080_tag_head_train_fingerprint.py b/alembic/versions/0080_tag_head_train_fingerprint.py deleted file mode 100644 index b4bd224..0000000 --- a/alembic/versions/0080_tag_head_train_fingerprint.py +++ /dev/null @@ -1,31 +0,0 @@ -"""tag_head.train_fingerprint (#1317 phase 2) — incremental head retraining - -A per-head training-data fingerprint (positive + rejection count/latest-timestamp) -so a manual Retrain refits only the tags whose data changed; the nightly run -ignores it (full reconcile). Nullable — a NULL fingerprint (existing heads) forces -a refit on the first incremental run, then it's stamped. - -Revision ID: 0080 -Revises: 0079 -Create Date: 2026-07-06 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0080" -down_revision: Union[str, None] = "0079" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "tag_head", - sa.Column("train_fingerprint", sa.String(128), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("tag_head", "train_fingerprint") diff --git a/alembic/versions/0081_stricter_auto_apply_defaults.py b/alembic/versions/0081_stricter_auto_apply_defaults.py deleted file mode 100644 index 8030eec..0000000 --- a/alembic/versions/0081_stricter_auto_apply_defaults.py +++ /dev/null @@ -1,43 +0,0 @@ -"""stricter auto-apply defaults (milestone 139) — cut auto-apply misfires - -head_auto_apply_min_positives 30→50 and ccip_auto_apply_threshold 0.92→0.95 -(operator-asked 2026-07-06). The head graduation precision bar stays 0.97 — the -operator confirmed the general-tag confidence was already well tuned; only the -support floor + the CCIP match confidence are raised. The model defaults change -for fresh installs; here we bump the existing singleton row IFF it is still at -the previous default, so a deliberate operator change is NOT clobbered. - -Revision ID: 0081 -Revises: 0080 -Create Date: 2026-07-06 -""" -from typing import Sequence, Union - -from alembic import op - -revision: str = "0081" -down_revision: Union[str, None] = "0080" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.execute( - "UPDATE ml_settings SET head_auto_apply_min_positives = 50 " - "WHERE head_auto_apply_min_positives = 30" - ) - op.execute( - "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.95 " - "WHERE ccip_auto_apply_threshold = 0.92" - ) - - -def downgrade() -> None: - op.execute( - "UPDATE ml_settings SET head_auto_apply_min_positives = 30 " - "WHERE head_auto_apply_min_positives = 50" - ) - op.execute( - "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.92 " - "WHERE ccip_auto_apply_threshold = 0.95" - ) diff --git a/alembic/versions/0082_presentation_auto_hide.py b/alembic/versions/0082_presentation_auto_hide.py deleted file mode 100644 index 8dc1f3c..0000000 --- a/alembic/versions/0082_presentation_auto_hide.py +++ /dev/null @@ -1,85 +0,0 @@ -"""presentation-chrome auto-hide (#141) — settings knobs + review table - -MLSettings gains presentation_auto_apply_enabled / _threshold and -presentation_conflict_threshold: banner + editor-screenshot auto-hide on the -sweep with a FLAT threshold (decoupled from content-head graduation), and a -conflict threshold that flags an auto-hide that "also looks like content". - -New table presentation_review records an auto-hidden chrome image that also -scored high on a content head, surfaced in the Hidden view for a keep-hidden / -un-hide decision. Resolved rows are pruned by retention. - -Revision ID: 0082 -Revises: 0081 -Create Date: 2026-07-07 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0082" -down_revision: Union[str, None] = "0081" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "presentation_auto_apply_enabled", sa.Boolean(), nullable=False, - server_default=sa.text("true"), - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "presentation_auto_apply_threshold", sa.Float(), nullable=False, - server_default=sa.text("0.90"), - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "presentation_conflict_threshold", sa.Float(), nullable=False, - server_default=sa.text("0.50"), - ), - ) - op.create_table( - "presentation_review", - sa.Column( - "image_record_id", sa.Integer(), - sa.ForeignKey("image_record.id", ondelete="CASCADE"), - primary_key=True, - ), - sa.Column( - "tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, - ), - sa.Column( - "conflict_tag_id", sa.Integer(), - sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, - ), - sa.Column("conflict_score", sa.Float(), nullable=False), - sa.Column( - "created_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.func.now(), - ), - sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True), - ) - # The review list queries the unresolved flags (resolved_at IS NULL). - op.create_index( - "ix_presentation_review_resolved_at", "presentation_review", - ["resolved_at"], - ) - - -def downgrade() -> None: - op.drop_index( - "ix_presentation_review_resolved_at", table_name="presentation_review" - ) - op.drop_table("presentation_review") - op.drop_column("ml_settings", "presentation_conflict_threshold") - op.drop_column("ml_settings", "presentation_auto_apply_threshold") - op.drop_column("ml_settings", "presentation_auto_apply_enabled") diff --git a/alembic/versions/0083_post_translation.py b/alembic/versions/0083_post_translation.py deleted file mode 100644 index d491d03..0000000 --- a/alembic/versions/0083_post_translation.py +++ /dev/null @@ -1,73 +0,0 @@ -"""post-text translation via Interpreter (milestone 143) — Post columns + settings - -Post gains the translated title/description + the detected source language, -Interpreter engine_version (cache key), and translated_at — filled by the -translate sweep. ImportSettings gains translation_enabled (OFF by default), -interpreter_base_url (EMPTY — the operator sets their own, behind a reverse -proxy), and translation_target_lang (en). - -Revision ID: 0083 -Revises: 0082 -Create Date: 2026-07-07 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0083" -down_revision: Union[str, None] = "0082" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "post", sa.Column("post_title_translated", sa.Text(), nullable=True) - ) - op.add_column( - "post", sa.Column("description_translated", sa.Text(), nullable=True) - ) - op.add_column( - "post", - sa.Column("translated_source_lang", sa.String(8), nullable=True), - ) - op.add_column( - "post", - sa.Column("translation_engine_version", sa.String(128), nullable=True), - ) - op.add_column( - "post", - sa.Column("translated_at", sa.DateTime(timezone=True), nullable=True), - ) - op.add_column( - "import_settings", - sa.Column( - "translation_enabled", sa.Boolean(), nullable=False, - server_default=sa.text("false"), - ), - ) - op.add_column( - "import_settings", - sa.Column( - "interpreter_base_url", sa.Text(), nullable=False, server_default="", - ), - ) - op.add_column( - "import_settings", - sa.Column( - "translation_target_lang", sa.Text(), nullable=False, - server_default="en", - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "translation_target_lang") - op.drop_column("import_settings", "interpreter_base_url") - op.drop_column("import_settings", "translation_enabled") - op.drop_column("post", "translated_at") - op.drop_column("post", "translation_engine_version") - op.drop_column("post", "translated_source_lang") - op.drop_column("post", "description_translated") - op.drop_column("post", "post_title_translated") diff --git a/alembic/versions/0084_translation_strictness_override.py b/alembic/versions/0084_translation_strictness_override.py deleted file mode 100644 index cd514b4..0000000 --- a/alembic/versions/0084_translation_strictness_override.py +++ /dev/null @@ -1,51 +0,0 @@ -"""translation strictness setting + per-post translation override (milestone 155) - -ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance -floor, now operator-tunable in the UI; default 0.9 — stricter than the old -hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at -~0.86). Post gains ``translation_override`` — a sticky per-post choice of -auto / force / original so the operator can force a skipped translation on, or -knock a wrongly-translated one back to the original, and have it survive a -Re-translate-all. - -Revision ID: 0084 -Revises: 0083 -Create Date: 2026-07-10 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0084" -down_revision: Union[str, None] = "0083" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "import_settings", - sa.Column( - "translation_min_confidence", sa.Float(), nullable=False, - server_default=sa.text("0.9"), - ), - ) - op.add_column( - "post", - sa.Column( - "translation_override", sa.String(16), nullable=False, - server_default="auto", - ), - ) - op.create_check_constraint( - "ck_post_translation_override", - "post", - "translation_override IN ('auto', 'force', 'original')", - ) - - -def downgrade() -> None: - op.drop_constraint("ck_post_translation_override", "post", type_="check") - op.drop_column("post", "translation_override") - op.drop_column("import_settings", "translation_min_confidence") diff --git a/alembic/versions/0085_wip_title_tagging.py b/alembic/versions/0085_wip_title_tagging.py deleted file mode 100644 index 4d260b1..0000000 --- a/alembic/versions/0085_wip_title_tagging.py +++ /dev/null @@ -1,35 +0,0 @@ -"""title-based WIP auto-tagging (task #1458) — ImportSettings toggle - -ImportSettings gains wip_title_tagging_enabled (ON by default): when a freshly -imported post's title explicitly declares work-in-progress ("WIP" / "work in -progress"), the importer applies the `wip` system tag to its images. No new -table — the tag itself is the seeded `wip` system tag (migration 0075) and the -application reuses image_tag with source='wip_title'. - -Revision ID: 0085 -Revises: 0084 -Create Date: 2026-07-12 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0085" -down_revision: Union[str, None] = "0084" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "import_settings", - sa.Column( - "wip_title_tagging_enabled", sa.Boolean(), nullable=False, - server_default=sa.text("true"), - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "wip_title_tagging_enabled") diff --git a/alembic/versions/0086_process_auto_apply_settings.py b/alembic/versions/0086_process_auto_apply_settings.py deleted file mode 100644 index 16f03c7..0000000 --- a/alembic/versions/0086_process_auto_apply_settings.py +++ /dev/null @@ -1,61 +0,0 @@ -"""process auto-apply settings + review mode (#1464) — system-tag refactor - -The system-tag behavior refactor gives `wip` / `editor screenshot` (the PROCESS -group) their own provisional auto-apply, parallel to the presentation (chrome) -sweep. MLSettings gains three knobs: enabled (OFF by default — a new whole-library -auto-tagger is opt-in), the flat apply threshold, and the ring-loud conflict -threshold. presentation_review gains a `mode` column so one review surface serves -both chrome and process flags (existing rows backfill 'chrome'). server_defaults -so the existing rows fill cleanly. - -Revision ID: 0086 -Revises: 0085 -Create Date: 2026-07-13 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0086" -down_revision: Union[str, None] = "0085" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "ml_settings", - sa.Column( - "process_auto_apply_enabled", sa.Boolean(), nullable=False, - server_default=sa.text("false"), - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "process_auto_apply_threshold", sa.Float(), nullable=False, - server_default="0.90", - ), - ) - op.add_column( - "ml_settings", - sa.Column( - "process_conflict_threshold", sa.Float(), nullable=False, - server_default="0.50", - ), - ) - op.add_column( - "presentation_review", - sa.Column( - "mode", sa.String(16), nullable=False, - server_default="chrome", - ), - ) - - -def downgrade() -> None: - op.drop_column("presentation_review", "mode") - op.drop_column("ml_settings", "process_conflict_threshold") - op.drop_column("ml_settings", "process_auto_apply_threshold") - op.drop_column("ml_settings", "process_auto_apply_enabled") diff --git a/alembic/versions/0087_baseline.py b/alembic/versions/0087_baseline.py new file mode 100644 index 0000000..bf803ae --- /dev/null +++ b/alembic/versions/0087_baseline.py @@ -0,0 +1,872 @@ +"""Collapsed baseline — the whole schema in one revision. + +Replaces revisions 0001..0087, which narrated the build-out of this project +and were deleted in milestone 328 step 1. A new install creates the schema in +one step instead of replaying that history. + +WHY THE REVISION ID IS "0087" AND NOT "0001" +-------------------------------------------- +It is deliberately the id of the LAST revision this baseline collapses, so an +existing database needs no intervention at all: + + * a fresh install finds current=none, head=0087, runs this file once, and + ends stamped at 0087. + * an existing install is ALREADY at 0087, so `alembic upgrade head` finds + current == head and does nothing. + +The alternative — numbering this 0001 and stamping every existing database — +means running `alembic stamp` against live data, and stamp VALIDATES NOTHING. +It writes a version string whether or not the schema actually matches, so a +wrong baseline would be discovered later, by the next real migration, with no +clean way back. Keeping the id removes that operation instead of making it +safe. Future revisions continue at 0088. + +The one case this makes worse, and it fails LOUDLY rather than silently: a +database still sitting between 0001 and 0086 (i.e. never upgraded to head) +cannot be located in this chain and errors out. Upgrade to 0087 on a +pre-squash build first, then take this one. + +WHAT IS HAND-WRITTEN HERE +------------------------- +Most of this file is `alembic revision --autogenerate` output, but four +things are NOT in SQLAlchemy metadata and the generator cannot produce them. +Each fails differently, and none of them fail at generation time: + + 1. CREATE EXTENSION vector (was 0001) — without it the VECTOR + columns below cannot be created at all. + 2. CREATE EXTENSION tsm_system_rows (was 0004) — used by the random-sample + query path; its absence surfaces only when that query runs. + 3. The HNSW index on image_record.siglip_embedding (was 0036). Raw SQL + because alembic's create_index cannot express `USING hnsw (... + vector_cosine_ops)`. Its absence is the quietest failure of the four: + everything works, similarity search just stops using an index. + 4. `import pgvector.sqlalchemy.vector`. Autogenerate EMITS references to + pgvector.sqlalchemy.vector.VECTOR but does not add the import, so the + generated file dies with NameError on first run. + +The acceptance test for this file is not that it reads correctly — it is +`.forgejo/workflows/baseline.yml`, which builds a database from the old +0001..0087 chain (read out of git) and one from this file, and diffs +pg_dump --schema-only output. That is what proves nothing was missed. + +Revision ID: 0087 +Revises: +Create Date: 2026-08-30 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# Autogenerate references pgvector.sqlalchemy.vector.VECTOR without importing +# it. Item 4 above. +import pgvector.sqlalchemy.vector + +revision: str = "0087" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Extensions FIRST: the VECTOR columns below cannot be created without + # `vector`, so ordering here is load-bearing, not tidiness. + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows") + + op.create_table('app_setting', + sa.Column('key', sa.String(length=64), nullable=False), + sa.Column('value', sa.Text(), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('key', name=op.f('pk_app_setting')) + ) + op.create_table('artist', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('slug', sa.String(length=255), nullable=False), + sa.Column('notes', sa.Text(), nullable=True), + sa.Column('is_subscription', sa.Boolean(), nullable=False), + sa.Column('auto_check', sa.Boolean(), nullable=False), + sa.Column('check_interval_seconds', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_artist')), + sa.UniqueConstraint('slug', name=op.f('uq_artist_slug')) + ) + op.create_table('backup_run', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('kind', sa.String(length=16), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('tag', sa.String(length=64), nullable=True), + sa.Column('triggered_by', sa.String(length=32), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('sql_path', sa.Text(), nullable=True), + sa.Column('tar_path', sa.Text(), nullable=True), + sa.Column('size_bytes', sa.BigInteger(), nullable=True), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('manifest', sa.JSON(), server_default='{}', nullable=False), + sa.Column('restored_from_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['restored_from_id'], ['backup_run.id'], name=op.f('fk_backup_run_restored_from_id_backup_run'), ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_backup_run')) + ) + op.create_index(op.f('ix_backup_run_finished_at'), 'backup_run', ['finished_at'], unique=False) + op.create_index(op.f('ix_backup_run_kind'), 'backup_run', ['kind'], unique=False) + op.create_index(op.f('ix_backup_run_started_at'), 'backup_run', ['started_at'], unique=False) + op.create_index(op.f('ix_backup_run_status'), 'backup_run', ['status'], unique=False) + op.create_index(op.f('ix_backup_run_tag'), 'backup_run', ['tag'], unique=False) + op.create_table('credential', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('platform', sa.String(length=64), nullable=False), + sa.Column('credential_type', sa.String(length=32), nullable=False), + sa.Column('encrypted_blob', sa.LargeBinary(), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_verified', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_credential')), + sa.UniqueConstraint('platform', name=op.f('uq_credential_platform')) + ) + op.create_table('head_auto_apply_run', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('dry_run', sa.Boolean(), nullable=False), + sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('n_applied', sa.Integer(), nullable=True), + sa.Column('report', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_head_auto_apply_run')) + ) + op.create_index(op.f('ix_head_auto_apply_run_status'), 'head_auto_apply_run', ['status'], unique=False) + op.create_table('head_training_run', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('n_trained', sa.Integer(), nullable=True), + sa.Column('n_skipped', sa.Integer(), nullable=True), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_head_training_run')) + ) + op.create_index(op.f('ix_head_training_run_status'), 'head_training_run', ['status'], unique=False) + op.create_table('import_batch', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('triggered_by', sa.String(length=32), nullable=False), + sa.Column('source_path', sa.Text(), nullable=False), + sa.Column('scan_mode', sa.String(length=16), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('total_files', sa.Integer(), nullable=False), + sa.Column('imported', sa.Integer(), nullable=False), + sa.Column('skipped', sa.Integer(), nullable=False), + sa.Column('failed', sa.Integer(), nullable=False), + sa.Column('attachments', sa.Integer(), nullable=False), + sa.Column('refreshed', sa.Integer(), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.PrimaryKeyConstraint('id', name=op.f('pk_import_batch')) + ) + op.create_index(op.f('ix_import_batch_status'), 'import_batch', ['status'], unique=False) + op.create_table('import_settings', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('import_scan_path', sa.Text(), nullable=False), + sa.Column('min_width', sa.Integer(), nullable=False), + sa.Column('min_height', sa.Integer(), nullable=False), + sa.Column('skip_transparent', sa.Boolean(), nullable=False), + sa.Column('transparency_threshold', sa.Float(), nullable=False), + sa.Column('skip_single_color', sa.Boolean(), nullable=False), + sa.Column('single_color_threshold', sa.Float(), nullable=False), + sa.Column('single_color_tolerance', sa.Integer(), nullable=False), + sa.Column('phash_threshold', sa.Integer(), nullable=False), + sa.Column('download_rate_limit_seconds', sa.Float(), nullable=False), + sa.Column('download_validate_files', sa.Boolean(), nullable=False), + sa.Column('download_schedule_default_seconds', sa.Integer(), nullable=False), + sa.Column('download_event_retention_days', sa.Integer(), nullable=False), + sa.Column('download_failure_warning_threshold', sa.Integer(), nullable=False), + sa.Column('backup_db_nightly_enabled', sa.Boolean(), nullable=False), + sa.Column('backup_db_nightly_hour_utc', sa.Integer(), nullable=False), + sa.Column('backup_db_keep_last_n', sa.Integer(), nullable=False), + sa.Column('backup_images_keep_last_n', sa.Integer(), nullable=False), + sa.Column('series_suggest_enabled', sa.Boolean(), nullable=False), + sa.Column('series_suggest_threshold', sa.Float(), nullable=False), + sa.Column('extdl_mega_enabled', sa.Boolean(), server_default='true', nullable=False), + sa.Column('extdl_gdrive_enabled', sa.Boolean(), server_default='true', nullable=False), + sa.Column('extdl_mediafire_enabled', sa.Boolean(), server_default='true', nullable=False), + sa.Column('extdl_dropbox_enabled', sa.Boolean(), server_default='true', nullable=False), + sa.Column('extdl_pixeldrain_enabled', sa.Boolean(), server_default='true', nullable=False), + sa.Column('translation_enabled', sa.Boolean(), server_default='false', nullable=False), + sa.Column('interpreter_base_url', sa.Text(), server_default='', nullable=False), + sa.Column('translation_target_lang', sa.Text(), server_default='en', nullable=False), + sa.Column('translation_min_confidence', sa.Float(), server_default='0.9', nullable=False), + sa.Column('wip_title_tagging_enabled', sa.Boolean(), server_default='true', nullable=False), + sa.Column('wip_soft_title_tagging_enabled', sa.Boolean(), server_default='false', nullable=False), + sa.CheckConstraint('id = 1', name=op.f('ck_import_settings_singleton')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_import_settings')) + ) + op.create_table('library_audit_run', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('rule', sa.String(length=32), nullable=False), + sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('scanned_count', sa.Integer(), nullable=False), + sa.Column('matched_count', sa.Integer(), nullable=False), + sa.Column('matched_ids', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('resume_after_id', sa.Integer(), nullable=False), + sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_library_audit_run')) + ) + op.create_index(op.f('ix_library_audit_run_rule'), 'library_audit_run', ['rule'], unique=False) + op.create_index(op.f('ix_library_audit_run_status'), 'library_audit_run', ['status'], unique=False) + op.create_table('ml_settings', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('cpu_embed_enabled', sa.Boolean(), nullable=False), + sa.Column('video_frame_interval_seconds', sa.Float(), nullable=False), + sa.Column('video_max_frames', sa.Integer(), nullable=False), + sa.Column('head_min_positives', sa.Integer(), nullable=False), + sa.Column('head_auto_apply_precision', sa.Float(), nullable=False), + sa.Column('head_auto_apply_enabled', sa.Boolean(), nullable=False), + sa.Column('head_auto_apply_min_positives', sa.Integer(), nullable=False), + sa.Column('ccip_match_threshold', sa.Float(), nullable=False), + sa.Column('ccip_auto_apply_enabled', sa.Boolean(), nullable=False), + sa.Column('ccip_auto_apply_threshold', sa.Float(), nullable=False), + sa.Column('presentation_auto_apply_enabled', sa.Boolean(), nullable=False), + sa.Column('presentation_auto_apply_threshold', sa.Float(), nullable=False), + sa.Column('presentation_conflict_threshold', sa.Float(), nullable=False), + sa.Column('process_auto_apply_enabled', sa.Boolean(), nullable=False), + sa.Column('process_auto_apply_threshold', sa.Float(), nullable=False), + sa.Column('process_conflict_threshold', sa.Float(), nullable=False), + sa.Column('embedder_model_version', sa.String(length=128), nullable=False), + sa.Column('embedder_model_name', sa.String(length=128), nullable=False), + sa.Column('detector_person_enabled', sa.Boolean(), nullable=False), + sa.Column('detector_person_weights', sa.String(length=512), nullable=False), + sa.Column('detector_person_conf', sa.Float(), nullable=False), + sa.Column('detector_anatomy_enabled', sa.Boolean(), nullable=False), + sa.Column('detector_anatomy_weights', sa.String(length=512), nullable=False), + sa.Column('detector_anatomy_conf', sa.Float(), nullable=False), + sa.Column('detector_panel_enabled', sa.Boolean(), nullable=False), + sa.Column('detector_panel_weights', sa.String(length=512), nullable=False), + sa.Column('detector_panel_conf', sa.Float(), nullable=False), + sa.Column('detector_max_figures', sa.Integer(), nullable=False), + sa.Column('detector_max_components', sa.Integer(), nullable=False), + sa.Column('detector_max_panels', sa.Integer(), nullable=False), + sa.Column('detector_max_regions', sa.Integer(), nullable=False), + sa.Column('detector_dedupe_iou', sa.Float(), nullable=False), + sa.Column('ccip_ref_signature', sa.String(length=128), nullable=True), + sa.Column('ccip_prototype_cap', sa.Integer(), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.CheckConstraint('id = 1', name=op.f('ck_ml_settings_singleton')), + sa.PrimaryKeyConstraint('id', name=op.f('pk_ml_settings')) + ) + op.create_table('tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('kind', sa.Enum('artist', 'character', 'fandom', 'general', 'series', 'archive', 'post', name='tag_kind'), nullable=False), + sa.Column('fandom_id', sa.Integer(), nullable=True), + sa.Column('is_system', sa.Boolean(), server_default=sa.text('false'), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.CheckConstraint("(fandom_id IS NULL) OR (kind = 'character')", name=op.f('ck_tag_ck_tag_fandom_requires_character')), + sa.ForeignKeyConstraint(['fandom_id'], ['tag.id'], name=op.f('fk_tag_fandom_id_tag'), ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_tag')) + ) + op.create_index(op.f('ix_tag_fandom_id'), 'tag', ['fandom_id'], unique=False) + op.create_table('task_run', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('celery_task_id', sa.String(length=64), nullable=False), + sa.Column('queue', sa.String(length=32), nullable=False), + sa.Column('task_name', sa.String(length=128), nullable=False), + sa.Column('target_id', sa.Integer(), nullable=True), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('duration_ms', sa.Integer(), nullable=True), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('error_type', sa.String(length=128), nullable=True), + sa.Column('error_message', sa.Text(), nullable=True), + sa.Column('retry_count', sa.Integer(), nullable=True), + sa.Column('worker_hostname', sa.String(length=128), nullable=True), + sa.Column('args_summary', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id', name=op.f('pk_task_run')) + ) + op.create_index(op.f('ix_task_run_celery_task_id'), 'task_run', ['celery_task_id'], unique=False) + op.create_index(op.f('ix_task_run_finished_at'), 'task_run', ['finished_at'], unique=False) + op.create_index(op.f('ix_task_run_queue'), 'task_run', ['queue'], unique=False) + op.create_index(op.f('ix_task_run_started_at'), 'task_run', ['started_at'], unique=False) + op.create_index(op.f('ix_task_run_status'), 'task_run', ['status'], unique=False) + op.create_index(op.f('ix_task_run_task_name'), 'task_run', ['task_name'], unique=False) + op.create_table('artist_visit', + sa.Column('artist_id', sa.Integer(), nullable=False), + sa.Column('last_viewed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_artist_visit_artist_id_artist'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('artist_id', name=op.f('pk_artist_visit')) + ) + op.create_table('ccip_prototype_state', + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('fingerprint', sa.String(length=64), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_ccip_prototype_state_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_ccip_prototype_state')) + ) + op.create_table('head_metric', + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('n_misfires', sa.Integer(), nullable=False), + sa.Column('n_underfires', sa.Integer(), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metric_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_head_metric')) + ) + op.create_table('head_metrics_snapshot', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('snapshot_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('n_auto_applied', sa.Integer(), nullable=False), + sa.Column('n_misfires', sa.Integer(), nullable=False), + sa.Column('n_underfires', sa.Integer(), nullable=False), + sa.Column('ap', sa.Float(), nullable=True), + sa.Column('precision_cv', sa.Float(), nullable=True), + sa.Column('recall', sa.Float(), nullable=True), + sa.Column('n_pos', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metrics_snapshot_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_head_metrics_snapshot')) + ) + op.create_index(op.f('ix_head_metrics_snapshot_snapshot_at'), 'head_metrics_snapshot', ['snapshot_at'], unique=False) + op.create_index(op.f('ix_head_metrics_snapshot_tag_id'), 'head_metrics_snapshot', ['tag_id'], unique=False) + op.create_table('source', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('artist_id', sa.Integer(), nullable=False), + sa.Column('platform', sa.String(length=64), nullable=False), + sa.Column('url', sa.Text(), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=False), + sa.Column('config_overrides', sa.JSON(), nullable=True), + sa.Column('last_checked_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('error_type', sa.String(length=32), nullable=True), + sa.Column('check_interval_override', sa.Integer(), nullable=True), + sa.Column('consecutive_failures', sa.Integer(), nullable=False), + sa.Column('backfill_runs_remaining', sa.Integer(), server_default='0', nullable=False), + sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_source_artist_id_artist'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_source')) + ) + op.create_index(op.f('ix_source_artist_id'), 'source', ['artist_id'], unique=False) + op.create_index(op.f('ix_source_error_type'), 'source', ['error_type'], unique=False) + op.create_table('tag_alias', + sa.Column('alias_string', sa.String(length=255), nullable=False), + sa.Column('alias_category', sa.String(length=32), nullable=False), + sa.Column('canonical_tag_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['canonical_tag_id'], ['tag.id'], name=op.f('fk_tag_alias_canonical_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('alias_string', 'alias_category', name=op.f('pk_tag_alias')) + ) + op.create_index(op.f('ix_tag_alias_canonical_tag_id'), 'tag_alias', ['canonical_tag_id'], unique=False) + op.create_table('tag_head', + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('embedding_version', sa.String(length=128), nullable=False), + sa.Column('weights', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=False), + sa.Column('bias', sa.Float(), nullable=False), + sa.Column('suggest_threshold', sa.Float(), nullable=False), + sa.Column('auto_apply_threshold', sa.Float(), nullable=True), + sa.Column('n_pos', sa.Integer(), nullable=False), + sa.Column('n_neg', sa.Integer(), nullable=False), + sa.Column('ap', sa.Float(), nullable=False), + sa.Column('precision_cv', sa.Float(), nullable=False), + sa.Column('recall', sa.Float(), nullable=False), + sa.Column('trained_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('train_fingerprint', sa.String(length=128), nullable=True), + sa.Column('metrics', postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_head_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_tag_head')) + ) + op.create_table('patreon_failed_media', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('filehash', sa.String(length=128), nullable=False), + sa.Column('attempts', sa.Integer(), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_patreon_failed_media_source_id_source'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_patreon_failed_media')), + sa.UniqueConstraint('source_id', 'filehash', name='uq_patreon_failed_media_source_id') + ) + op.create_index(op.f('ix_patreon_failed_media_source_id'), 'patreon_failed_media', ['source_id'], unique=False) + op.create_table('patreon_seen_media', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('filehash', sa.String(length=128), nullable=False), + sa.Column('post_id', sa.String(length=64), nullable=True), + sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_patreon_seen_media_source_id_source'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_patreon_seen_media')), + sa.UniqueConstraint('source_id', 'filehash', name='uq_patreon_seen_media_source_id') + ) + op.create_index(op.f('ix_patreon_seen_media_source_id'), 'patreon_seen_media', ['source_id'], unique=False) + op.create_table('pixiv_failed_media', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('filehash', sa.String(length=128), nullable=False), + sa.Column('attempts', sa.Integer(), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_pixiv_failed_media_source_id_source'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_pixiv_failed_media')), + sa.UniqueConstraint('source_id', 'filehash', name='uq_pixiv_failed_media_source_id') + ) + op.create_index(op.f('ix_pixiv_failed_media_source_id'), 'pixiv_failed_media', ['source_id'], unique=False) + op.create_table('pixiv_seen_media', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('filehash', sa.String(length=128), nullable=False), + sa.Column('post_id', sa.String(length=64), nullable=True), + sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_pixiv_seen_media_source_id_source'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_pixiv_seen_media')), + sa.UniqueConstraint('source_id', 'filehash', name='uq_pixiv_seen_media_source_id') + ) + op.create_index(op.f('ix_pixiv_seen_media_source_id'), 'pixiv_seen_media', ['source_id'], unique=False) + op.create_table('post', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=True), + sa.Column('artist_id', sa.Integer(), nullable=False), + sa.Column('external_post_id', sa.String(length=128), nullable=False), + sa.Column('post_url', sa.Text(), nullable=True), + sa.Column('post_title', sa.Text(), nullable=True), + sa.Column('post_date', sa.DateTime(timezone=True), nullable=True), + sa.Column('raw_metadata', sa.JSON(), nullable=True), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('attachment_count', sa.Integer(), nullable=True), + sa.Column('post_title_translated', sa.Text(), nullable=True), + sa.Column('description_translated', sa.Text(), nullable=True), + sa.Column('translated_source_lang', sa.String(length=8), nullable=True), + sa.Column('translation_engine_version', sa.String(length=128), nullable=True), + sa.Column('translated_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('translation_override', sa.String(length=16), server_default='auto', nullable=False), + sa.Column('downloaded_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.CheckConstraint("translation_override IN ('auto', 'force', 'original')", name=op.f('ck_post_ck_post_translation_override')), + sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_artist_id_artist'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_post_source_id_source'), ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_post')), + sa.UniqueConstraint('source_id', 'external_post_id', name='uq_post_source_external_id') + ) + op.create_index(op.f('ix_post_artist_id'), 'post', ['artist_id'], unique=False) + op.create_index(op.f('ix_post_source_id'), 'post', ['source_id'], unique=False) + op.create_table('subscribestar_failed_media', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('filehash', sa.String(length=128), nullable=False), + sa.Column('attempts', sa.Integer(), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_subscribestar_failed_media_source_id_source'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_subscribestar_failed_media')), + sa.UniqueConstraint('source_id', 'filehash', name='uq_subscribestar_failed_media_source_id') + ) + op.create_index(op.f('ix_subscribestar_failed_media_source_id'), 'subscribestar_failed_media', ['source_id'], unique=False) + op.create_table('subscribestar_seen_media', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('filehash', sa.String(length=128), nullable=False), + sa.Column('post_id', sa.String(length=64), nullable=True), + sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_subscribestar_seen_media_source_id_source'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_subscribestar_seen_media')), + sa.UniqueConstraint('source_id', 'filehash', name='uq_subscribestar_seen_media_source_id') + ) + op.create_index(op.f('ix_subscribestar_seen_media_source_id'), 'subscribestar_seen_media', ['source_id'], unique=False) + op.create_table('download_event', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=False), + sa.Column('post_id', sa.Integer(), nullable=True), + sa.Column('status', sa.String(length=32), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('bytes_downloaded', sa.BigInteger(), nullable=False), + sa.Column('files_count', sa.Integer(), nullable=False), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_download_event_post_id_post'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_download_event_source_id_source'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_download_event')) + ) + op.create_index(op.f('ix_download_event_post_id'), 'download_event', ['post_id'], unique=False) + op.create_index(op.f('ix_download_event_source_id'), 'download_event', ['source_id'], unique=False) + op.create_table('image_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('path', sa.Text(), nullable=False), + sa.Column('sha256', sa.String(length=64), nullable=False), + sa.Column('phash', sa.String(length=32), nullable=True), + sa.Column('size_bytes', sa.BigInteger(), nullable=False), + sa.Column('mime', sa.String(length=64), nullable=False), + sa.Column('width', sa.Integer(), nullable=True), + sa.Column('height', sa.Integer(), nullable=True), + sa.Column('duration_seconds', sa.Float(), nullable=True), + sa.Column('integrity_status', sa.String(length=24), nullable=False), + sa.Column('thumbnail_path', sa.Text(), nullable=True), + sa.Column('source_url', sa.Text(), nullable=True), + sa.Column('source_filehash', sa.String(length=32), nullable=True), + sa.Column('origin', sa.Enum('downloaded', 'imported_filesystem', 'uploaded', name='origin_enum'), nullable=False), + sa.Column('primary_post_id', sa.Integer(), nullable=True), + sa.Column('artist_id', sa.Integer(), nullable=True), + sa.Column('siglip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=True), + sa.Column('siglip_model_version', sa.String(length=128), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('effective_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('earliest_post_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_image_record_artist_id_artist'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['primary_post_id'], ['post.id'], name=op.f('fk_image_record_primary_post_id_post'), ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_image_record')), + sa.UniqueConstraint('path', name=op.f('uq_image_record_path')) + ) + op.create_index(op.f('ix_image_record_artist_id'), 'image_record', ['artist_id'], unique=False) + op.create_index(op.f('ix_image_record_integrity_status'), 'image_record', ['integrity_status'], unique=False) + op.create_index(op.f('ix_image_record_phash'), 'image_record', ['phash'], unique=False) + op.create_index(op.f('ix_image_record_primary_post_id'), 'image_record', ['primary_post_id'], unique=False) + op.create_index(op.f('ix_image_record_sha256'), 'image_record', ['sha256'], unique=True) + op.create_index(op.f('ix_image_record_source_filehash'), 'image_record', ['source_filehash'], unique=False) + op.create_table('post_attachment', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('post_id', sa.Integer(), nullable=True), + sa.Column('artist_id', sa.Integer(), nullable=True), + sa.Column('sha256', sa.String(length=64), nullable=False), + sa.Column('path', sa.Text(), nullable=False), + sa.Column('original_filename', sa.Text(), nullable=False), + sa.Column('ext', sa.String(length=32), nullable=False), + sa.Column('mime', sa.String(length=128), nullable=True), + sa.Column('size_bytes', sa.BigInteger(), nullable=False), + sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_attachment_artist_id_artist'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_post_attachment_post_id_post'), ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_post_attachment')) + ) + op.create_index(op.f('ix_post_attachment_artist_id'), 'post_attachment', ['artist_id'], unique=False) + op.create_index(op.f('ix_post_attachment_post_id'), 'post_attachment', ['post_id'], unique=False) + op.create_index(op.f('ix_post_attachment_sha256'), 'post_attachment', ['sha256'], unique=False) + op.create_index('uq_post_attachment_null_post_sha', 'post_attachment', ['sha256'], unique=True, postgresql_where=sa.text('post_id IS NULL')) + op.create_index('uq_post_attachment_post_sha', 'post_attachment', ['post_id', 'sha256'], unique=True, postgresql_where=sa.text('post_id IS NOT NULL')) + op.create_table('series_suggestion', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('post_id', sa.Integer(), nullable=False), + sa.Column('series_tag_id', sa.Integer(), nullable=False), + sa.Column('score', sa.Float(), nullable=False), + sa.Column('signals', sa.JSON(), nullable=True), + sa.Column('status', sa.String(length=16), server_default='pending', nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_series_suggestion_post_id_post'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_suggestion_series_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_series_suggestion')), + sa.UniqueConstraint('post_id', 'series_tag_id', name='uq_series_suggestion_post_series') + ) + op.create_index(op.f('ix_series_suggestion_post_id'), 'series_suggestion', ['post_id'], unique=False) + op.create_index(op.f('ix_series_suggestion_series_tag_id'), 'series_suggestion', ['series_tag_id'], unique=False) + op.create_index(op.f('ix_series_suggestion_status'), 'series_suggestion', ['status'], unique=False) + op.create_table('external_link', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('post_id', sa.Integer(), nullable=False), + sa.Column('artist_id', sa.Integer(), nullable=True), + sa.Column('host', sa.String(length=16), nullable=False), + sa.Column('url', sa.Text(), nullable=False), + sa.Column('label', sa.Text(), nullable=True), + sa.Column('status', sa.String(length=16), server_default='pending', nullable=False), + sa.Column('attempts', sa.Integer(), server_default=sa.text('0'), nullable=False), + sa.Column('last_error', sa.Text(), nullable=True), + sa.Column('attachment_id', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('duration_seconds', sa.Float(), nullable=True), + sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_external_link_artist_id_artist'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['attachment_id'], ['post_attachment.id'], name=op.f('fk_external_link_attachment_id_post_attachment'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_external_link_post_id_post'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_external_link')) + ) + op.create_index(op.f('ix_external_link_artist_id'), 'external_link', ['artist_id'], unique=False) + op.create_index(op.f('ix_external_link_post_id'), 'external_link', ['post_id'], unique=False) + op.create_index('ix_external_link_status', 'external_link', ['status'], unique=False) + op.create_index('uq_external_link_post_url', 'external_link', ['post_id', 'url'], unique=True) + op.create_table('gpu_job', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('image_record_id', sa.Integer(), nullable=False), + sa.Column('task', sa.String(length=32), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('lease_token', sa.String(length=64), nullable=True), + sa.Column('leased_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('lease_expires_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('attempts', sa.Integer(), nullable=False), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('triage_status', sa.String(length=16), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_gpu_job_image_record_id_image_record'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_gpu_job')) + ) + op.create_index(op.f('ix_gpu_job_image_record_id'), 'gpu_job', ['image_record_id'], unique=False) + op.create_index('ix_gpu_job_leased_expires', 'gpu_job', ['lease_expires_at'], unique=False, postgresql_where=sa.text("status = 'leased'")) + op.create_index('ix_gpu_job_pending', 'gpu_job', ['id'], unique=False, postgresql_where=sa.text("status = 'pending'")) + op.create_index(op.f('ix_gpu_job_status'), 'gpu_job', ['status'], unique=False) + op.create_table('image_provenance', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('image_record_id', sa.Integer(), nullable=False), + sa.Column('post_id', sa.Integer(), nullable=False), + sa.Column('source_id', sa.Integer(), nullable=True), + sa.Column('from_attachment_id', sa.Integer(), nullable=True), + sa.Column('captured_metadata', sa.JSON(), nullable=True), + sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['from_attachment_id'], ['post_attachment.id'], name=op.f('fk_image_provenance_from_attachment_id_post_attachment'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_provenance_image_record_id_image_record'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_image_provenance_post_id_post'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_image_provenance_source_id_source'), ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_image_provenance')), + sa.UniqueConstraint('image_record_id', 'post_id', name='uq_image_provenance_image_post') + ) + op.create_index(op.f('ix_image_provenance_from_attachment_id'), 'image_provenance', ['from_attachment_id'], unique=False) + op.create_index(op.f('ix_image_provenance_image_record_id'), 'image_provenance', ['image_record_id'], unique=False) + op.create_index(op.f('ix_image_provenance_post_id'), 'image_provenance', ['post_id'], unique=False) + op.create_index(op.f('ix_image_provenance_source_id'), 'image_provenance', ['source_id'], unique=False) + op.create_table('image_region', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('image_record_id', sa.Integer(), nullable=False), + sa.Column('kind', sa.String(length=16), nullable=False), + sa.Column('frame_time', sa.Float(), nullable=True), + sa.Column('rx', sa.Float(), nullable=False), + sa.Column('ry', sa.Float(), nullable=False), + sa.Column('rw', sa.Float(), nullable=False), + sa.Column('rh', sa.Float(), nullable=False), + sa.Column('score', sa.Float(), nullable=True), + sa.Column('detector_version', sa.String(length=64), nullable=True), + sa.Column('crop_version', sa.String(length=64), nullable=True), + sa.Column('embedding_version', sa.String(length=128), nullable=True), + sa.Column('ccip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=768), nullable=True), + sa.Column('siglip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_region_image_record_id_image_record'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_image_region')) + ) + op.create_index(op.f('ix_image_region_image_record_id'), 'image_region', ['image_record_id'], unique=False) + op.create_table('image_tag', + sa.Column('image_record_id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('source', sa.String(length=32), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_tag_image_record_id_image_record'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_image_tag_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_image_tag')) + ) + op.create_table('import_task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('batch_id', sa.Integer(), nullable=False), + sa.Column('source_path', sa.Text(), nullable=False), + sa.Column('task_type', sa.String(length=16), nullable=False), + sa.Column('status', sa.String(length=16), nullable=False), + sa.Column('recovery_count', sa.Integer(), nullable=False), + sa.Column('refetched', sa.Boolean(), nullable=False), + sa.Column('result_image_id', sa.Integer(), nullable=True), + sa.Column('error', sa.Text(), nullable=True), + sa.Column('size_bytes', sa.BigInteger(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), + sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['batch_id'], ['import_batch.id'], name=op.f('fk_import_task_batch_id_import_batch'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['result_image_id'], ['image_record.id'], name=op.f('fk_import_task_result_image_id_image_record'), ondelete='SET NULL'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_import_task')) + ) + op.create_index(op.f('ix_import_task_batch_id'), 'import_task', ['batch_id'], unique=False) + op.create_index(op.f('ix_import_task_status'), 'import_task', ['status'], unique=False) + op.create_table('presentation_review', + sa.Column('image_record_id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('conflict_tag_id', sa.Integer(), nullable=True), + sa.Column('conflict_score', sa.Float(), nullable=False), + sa.Column('mode', sa.String(length=16), server_default='chrome', nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(['conflict_tag_id'], ['tag.id'], name=op.f('fk_presentation_review_conflict_tag_id_tag'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_presentation_review_image_record_id_image_record'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_presentation_review_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_presentation_review')) + ) + op.create_table('series_page', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('series_tag_id', sa.Integer(), nullable=False), + sa.Column('image_id', sa.Integer(), nullable=False), + sa.Column('status', sa.String(length=16), server_default='placed', nullable=False), + sa.Column('page_number', sa.Integer(), nullable=True), + sa.Column('stated_page', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], name=op.f('fk_series_page_image_id_image_record'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_page_series_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_series_page')), + sa.UniqueConstraint('image_id', name=op.f('uq_series_page_image_id')) + ) + op.create_index(op.f('ix_series_page_series_tag_id'), 'series_page', ['series_tag_id'], unique=False) + op.create_table('tag_positive_confirmation', + sa.Column('image_record_id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('confirmed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_tag_positive_confirmation_image_record_id_image_record'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_positive_confirmation_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_positive_confirmation')) + ) + op.create_index(op.f('ix_tag_positive_confirmation_tag_id'), 'tag_positive_confirmation', ['tag_id'], unique=False) + op.create_table('tag_suggestion_rejection', + sa.Column('image_record_id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('rejected_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_tag_suggestion_rejection_image_record_id_image_record'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_suggestion_rejection_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_suggestion_rejection')) + ) + op.create_index(op.f('ix_tag_suggestion_rejection_tag_id'), 'tag_suggestion_rejection', ['tag_id'], unique=False) + op.create_table('character_prototype', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('tag_id', sa.Integer(), nullable=False), + sa.Column('ccip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=768), nullable=False), + sa.Column('region_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['region_id'], ['image_region.id'], name=op.f('fk_character_prototype_region_id_image_region'), ondelete='SET NULL'), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_character_prototype_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_character_prototype')) + ) + op.create_index(op.f('ix_character_prototype_tag_id'), 'character_prototype', ['tag_id'], unique=False) + op.create_table('series_chapter', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('series_tag_id', sa.Integer(), nullable=False), + sa.Column('anchor_page_id', sa.Integer(), nullable=False), + sa.Column('title', sa.Text(), nullable=True), + sa.Column('stated_part', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), + sa.ForeignKeyConstraint(['anchor_page_id'], ['series_page.id'], name=op.f('fk_series_chapter_anchor_page_id_series_page'), ondelete='CASCADE'), + sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_chapter_series_tag_id_tag'), ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id', name=op.f('pk_series_chapter')), + sa.UniqueConstraint('anchor_page_id', name=op.f('uq_series_chapter_anchor_page_id')) + ) + op.create_index(op.f('ix_series_chapter_series_tag_id'), 'series_chapter', ['series_tag_id'], unique=False) + + # The HNSW index, item 3 above. Must match the query's cosine-distance + # operator class or the planner will not use it. + op.execute( + "CREATE INDEX ix_image_record_siglip_hnsw " + "ON image_record USING hnsw (siglip_embedding vector_cosine_ops)" + ) + + +def downgrade() -> None: + # Dropping image_record takes its indexes with it, so the HNSW index needs + # no separate drop. The extensions are deliberately left in place: they are + # database-scoped and something else may be using them. + op.drop_index(op.f('ix_series_chapter_series_tag_id'), table_name='series_chapter') + op.drop_table('series_chapter') + op.drop_index(op.f('ix_character_prototype_tag_id'), table_name='character_prototype') + op.drop_table('character_prototype') + op.drop_index(op.f('ix_tag_suggestion_rejection_tag_id'), table_name='tag_suggestion_rejection') + op.drop_table('tag_suggestion_rejection') + op.drop_index(op.f('ix_tag_positive_confirmation_tag_id'), table_name='tag_positive_confirmation') + op.drop_table('tag_positive_confirmation') + op.drop_index(op.f('ix_series_page_series_tag_id'), table_name='series_page') + op.drop_table('series_page') + op.drop_table('presentation_review') + op.drop_index(op.f('ix_import_task_status'), table_name='import_task') + op.drop_index(op.f('ix_import_task_batch_id'), table_name='import_task') + op.drop_table('import_task') + op.drop_table('image_tag') + op.drop_index(op.f('ix_image_region_image_record_id'), table_name='image_region') + op.drop_table('image_region') + op.drop_index(op.f('ix_image_provenance_source_id'), table_name='image_provenance') + op.drop_index(op.f('ix_image_provenance_post_id'), table_name='image_provenance') + op.drop_index(op.f('ix_image_provenance_image_record_id'), table_name='image_provenance') + op.drop_index(op.f('ix_image_provenance_from_attachment_id'), table_name='image_provenance') + op.drop_table('image_provenance') + op.drop_index(op.f('ix_gpu_job_status'), table_name='gpu_job') + op.drop_index('ix_gpu_job_pending', table_name='gpu_job', postgresql_where=sa.text("status = 'pending'")) + op.drop_index('ix_gpu_job_leased_expires', table_name='gpu_job', postgresql_where=sa.text("status = 'leased'")) + op.drop_index(op.f('ix_gpu_job_image_record_id'), table_name='gpu_job') + op.drop_table('gpu_job') + op.drop_index('uq_external_link_post_url', table_name='external_link') + op.drop_index('ix_external_link_status', table_name='external_link') + op.drop_index(op.f('ix_external_link_post_id'), table_name='external_link') + op.drop_index(op.f('ix_external_link_artist_id'), table_name='external_link') + op.drop_table('external_link') + op.drop_index(op.f('ix_series_suggestion_status'), table_name='series_suggestion') + op.drop_index(op.f('ix_series_suggestion_series_tag_id'), table_name='series_suggestion') + op.drop_index(op.f('ix_series_suggestion_post_id'), table_name='series_suggestion') + op.drop_table('series_suggestion') + op.drop_index('uq_post_attachment_post_sha', table_name='post_attachment', postgresql_where=sa.text('post_id IS NOT NULL')) + op.drop_index('uq_post_attachment_null_post_sha', table_name='post_attachment', postgresql_where=sa.text('post_id IS NULL')) + op.drop_index(op.f('ix_post_attachment_sha256'), table_name='post_attachment') + op.drop_index(op.f('ix_post_attachment_post_id'), table_name='post_attachment') + op.drop_index(op.f('ix_post_attachment_artist_id'), table_name='post_attachment') + op.drop_table('post_attachment') + op.drop_index(op.f('ix_image_record_source_filehash'), table_name='image_record') + op.drop_index(op.f('ix_image_record_sha256'), table_name='image_record') + op.drop_index(op.f('ix_image_record_primary_post_id'), table_name='image_record') + op.drop_index(op.f('ix_image_record_phash'), table_name='image_record') + op.drop_index(op.f('ix_image_record_integrity_status'), table_name='image_record') + op.drop_index(op.f('ix_image_record_artist_id'), table_name='image_record') + op.drop_table('image_record') + op.drop_index(op.f('ix_download_event_source_id'), table_name='download_event') + op.drop_index(op.f('ix_download_event_post_id'), table_name='download_event') + op.drop_table('download_event') + op.drop_index(op.f('ix_subscribestar_seen_media_source_id'), table_name='subscribestar_seen_media') + op.drop_table('subscribestar_seen_media') + op.drop_index(op.f('ix_subscribestar_failed_media_source_id'), table_name='subscribestar_failed_media') + op.drop_table('subscribestar_failed_media') + op.drop_index(op.f('ix_post_source_id'), table_name='post') + op.drop_index(op.f('ix_post_artist_id'), table_name='post') + op.drop_table('post') + op.drop_index(op.f('ix_pixiv_seen_media_source_id'), table_name='pixiv_seen_media') + op.drop_table('pixiv_seen_media') + op.drop_index(op.f('ix_pixiv_failed_media_source_id'), table_name='pixiv_failed_media') + op.drop_table('pixiv_failed_media') + op.drop_index(op.f('ix_patreon_seen_media_source_id'), table_name='patreon_seen_media') + op.drop_table('patreon_seen_media') + op.drop_index(op.f('ix_patreon_failed_media_source_id'), table_name='patreon_failed_media') + op.drop_table('patreon_failed_media') + op.drop_table('tag_head') + op.drop_index(op.f('ix_tag_alias_canonical_tag_id'), table_name='tag_alias') + op.drop_table('tag_alias') + op.drop_index(op.f('ix_source_error_type'), table_name='source') + op.drop_index(op.f('ix_source_artist_id'), table_name='source') + op.drop_table('source') + op.drop_index(op.f('ix_head_metrics_snapshot_tag_id'), table_name='head_metrics_snapshot') + op.drop_index(op.f('ix_head_metrics_snapshot_snapshot_at'), table_name='head_metrics_snapshot') + op.drop_table('head_metrics_snapshot') + op.drop_table('head_metric') + op.drop_table('ccip_prototype_state') + op.drop_table('artist_visit') + op.drop_index(op.f('ix_task_run_task_name'), table_name='task_run') + op.drop_index(op.f('ix_task_run_status'), table_name='task_run') + op.drop_index(op.f('ix_task_run_started_at'), table_name='task_run') + op.drop_index(op.f('ix_task_run_queue'), table_name='task_run') + op.drop_index(op.f('ix_task_run_finished_at'), table_name='task_run') + op.drop_index(op.f('ix_task_run_celery_task_id'), table_name='task_run') + op.drop_table('task_run') + op.drop_index(op.f('ix_tag_fandom_id'), table_name='tag') + op.drop_table('tag') + op.drop_table('ml_settings') + op.drop_index(op.f('ix_library_audit_run_status'), table_name='library_audit_run') + op.drop_index(op.f('ix_library_audit_run_rule'), table_name='library_audit_run') + op.drop_table('library_audit_run') + op.drop_table('import_settings') + op.drop_index(op.f('ix_import_batch_status'), table_name='import_batch') + op.drop_table('import_batch') + op.drop_index(op.f('ix_head_training_run_status'), table_name='head_training_run') + op.drop_table('head_training_run') + op.drop_index(op.f('ix_head_auto_apply_run_status'), table_name='head_auto_apply_run') + op.drop_table('head_auto_apply_run') + op.drop_table('credential') + op.drop_index(op.f('ix_backup_run_tag'), table_name='backup_run') + op.drop_index(op.f('ix_backup_run_status'), table_name='backup_run') + op.drop_index(op.f('ix_backup_run_started_at'), table_name='backup_run') + op.drop_index(op.f('ix_backup_run_kind'), table_name='backup_run') + op.drop_index(op.f('ix_backup_run_finished_at'), table_name='backup_run') + op.drop_table('backup_run') + op.drop_table('artist') + op.drop_table('app_setting') diff --git a/alembic/versions/0087_wip_soft_title_tagging.py b/alembic/versions/0087_wip_soft_title_tagging.py deleted file mode 100644 index 58478e1..0000000 --- a/alembic/versions/0087_wip_soft_title_tagging.py +++ /dev/null @@ -1,33 +0,0 @@ -"""soft WIP title tier toggle (#1474) — ImportSettings.wip_soft_title_tagging_enabled - -The soft tier also tags sketch/doodle/scribble titles, but with a provisional source -that never trains the head. OFF by default (a lower-precision tier is opt-in). -server_default so the existing singleton row (id=1) fills cleanly. - -Revision ID: 0087 -Revises: 0086 -Create Date: 2026-07-13 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - -revision: str = "0087" -down_revision: Union[str, None] = "0086" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "import_settings", - sa.Column( - "wip_soft_title_tagging_enabled", sa.Boolean(), nullable=False, - server_default=sa.text("false"), - ), - ) - - -def downgrade() -> None: - op.drop_column("import_settings", "wip_soft_title_tagging_enabled") diff --git a/backend/app/utils/artist_backfill.py b/backend/app/utils/artist_backfill.py deleted file mode 100644 index 3979b24..0000000 --- a/backend/app/utils/artist_backfill.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Literal SQL for the FC-2d-vii-c artist backfill / artist-tag delete. - -Intentionally pure string constants — NO model/slug imports, NO logic — -so migration 0008 and its test share one drift-proof source of truth. -Backfill steps are ordered primary -> provenance -> artist-tag and each -only touches rows still NULL (idempotent, first match wins). The -artist-tag step matches Artist.name = Tag.name: the importer always -created both from the same artist_name string. -""" - -BACKFILL_PRIMARY_SQL = """ -UPDATE image_record AS ir -SET artist_id = s.artist_id -FROM post p -JOIN source s ON s.id = p.source_id -WHERE ir.primary_post_id = p.id - AND ir.artist_id IS NULL -""" - -BACKFILL_PROVENANCE_SQL = """ -UPDATE image_record AS ir -SET artist_id = s.artist_id -FROM ( - SELECT DISTINCT ON (ip.image_record_id) - ip.image_record_id, src.artist_id - FROM image_provenance ip - JOIN source src ON src.id = ip.source_id - ORDER BY ip.image_record_id, ip.id -) AS s -WHERE ir.id = s.image_record_id - AND ir.artist_id IS NULL -""" - -BACKFILL_TAG_SQL = """ -UPDATE image_record AS ir -SET artist_id = a.id -FROM image_tag it -JOIN tag t ON t.id = it.tag_id AND t.kind = 'artist' -JOIN artist a ON a.name = t.name -WHERE it.image_record_id = ir.id - AND ir.artist_id IS NULL -""" - -DELETE_ARTIST_TAGS_SQL = "DELETE FROM tag WHERE kind = 'artist'" diff --git a/tests/test_migration_0002.py b/tests/test_migration_0002.py deleted file mode 100644 index 1786731..0000000 --- a/tests/test_migration_0002.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Smoke test for migration 0002: confirms model classes import and the -tag-kind uniqueness rule shape is correct. -""" - -from backend.app.models import ( - Base, - ImportBatch, - ImportSettings, - ImportTask, - Tag, - TagKind, -) - - -def test_new_tables_registered(): - expected = {"import_batch", "import_task", "import_settings"} - assert expected.issubset(Base.metadata.tables.keys()) - - -def test_tag_has_kind_and_fandom_id(): - cols = {c.name for c in Tag.__table__.columns} - assert "kind" in cols - assert "fandom_id" in cols - assert "namespace" not in cols - - -def test_tag_kind_enum_values(): - # Current TagKind enum after alembic 0023 dropped meta + rating - # (operator-retired 2026-05-26). `artist` is still in the enum - # for backward-compat with historical rows, though new artist - # tags don't get created (Artist row is canonical per FC-2d-vii-c). - expected = { - "artist", - "character", - "fandom", - "general", - "series", - "archive", - "post", - } - assert {k.value for k in TagKind} == expected - - -def test_image_record_has_integrity_status(): - from backend.app.models import ImageRecord - cols = {c.name for c in ImageRecord.__table__.columns} - assert "integrity_status" in cols - - -def test_import_task_has_state_columns(): - cols = {c.name for c in ImportTask.__table__.columns} - for required in ("batch_id", "source_path", "task_type", "status", "result_image_id"): - assert required in cols - - -def test_import_settings_singleton_constraint(): - constraints = {c.name for c in ImportSettings.__table__.constraints} - assert "ck_import_settings_singleton" in constraints diff --git a/tests/test_migration_0003.py b/tests/test_migration_0003.py deleted file mode 100644 index 1752120..0000000 --- a/tests/test_migration_0003.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Smoke test for migration 0003: model classes import, schema shape correct.""" - -from backend.app.models import ( - Base, - ImageRecord, - MLSettings, - TagAlias, - TagSuggestionRejection, -) - - -def test_new_tables_registered(): - expected = { - "tag_suggestion_rejection", - "tag_alias", - "ml_settings", - } - assert expected.issubset(Base.metadata.tables.keys()) - - -def test_image_record_columns_renamed(): - cols = {c.name for c in ImageRecord.__table__.columns} - # Legacy tagger columns are all gone: tagger_predictions/wd14_* dropped in - # 0046, tagger_model_version + centroid_scores dropped in 0068 (#1199, Camie - # retirement). The SigLIP embedding columns are the live ML fields. - assert "siglip_embedding" in cols - assert "siglip_model_version" in cols - assert "tagger_model_version" not in cols - assert "centroid_scores" not in cols - assert "tagger_predictions" not in cols - assert "wd14_predictions" not in cols - - -def test_tag_alias_composite_pk(): - pk_cols = {c.name for c in TagAlias.__table__.primary_key.columns} - assert pk_cols == {"alias_string", "alias_category"} - - -def test_ml_settings_singleton_constraint(): - names = {c.name for c in MLSettings.__table__.constraints} - assert "ck_ml_settings_singleton" in names - - -def test_tag_suggestion_rejection_pk(): - pk_cols = {c.name for c in TagSuggestionRejection.__table__.primary_key.columns} - assert pk_cols == {"image_record_id", "tag_id"} diff --git a/tests/test_migration_0004.py b/tests/test_migration_0004.py deleted file mode 100644 index 7975d0c..0000000 --- a/tests/test_migration_0004.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Integration: the tsm_system_rows extension is installed by migration 0004. - -Needs a real Postgres (CI does not provision one), so integration-marked. -""" - -import pytest -from sqlalchemy import text - -pytestmark = pytest.mark.integration - - -@pytest.mark.asyncio -async def test_tsm_system_rows_extension_present(db): - row = ( - await db.execute( - text("SELECT 1 FROM pg_extension WHERE extname = 'tsm_system_rows'") - ) - ).first() - assert row is not None - - -@pytest.mark.asyncio -async def test_system_rows_sampling_is_usable(db): - # Should parse and execute even on an empty table. - await db.execute( - text("SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(1)") - ) diff --git a/tests/test_migration_0007.py b/tests/test_migration_0007.py deleted file mode 100644 index 6f716f4..0000000 --- a/tests/test_migration_0007.py +++ /dev/null @@ -1,48 +0,0 @@ -"""FC-2d-iv: post.description + post.attachment_count round-trip.""" - -from datetime import UTC, datetime - -import pytest - -from backend.app.models import Artist, Post, Source - -pytestmark = pytest.mark.integration - - -async def _post(db, **post_kwargs): - artist = Artist(name="Nadia", slug="nadia") - db.add(artist) - await db.flush() - src = Source(artist_id=artist.id, platform="web", url="http://x") - db.add(src) - await db.flush() - post = Post( - source_id=src.id, artist_id=artist.id, external_post_id="p1", - post_date=datetime(2026, 3, 1, tzinfo=UTC), - **post_kwargs, - ) - db.add(post) - await db.flush() - return post.id - - -def test_post_has_new_columns(): - cols = {c.name for c in Post.__table__.columns} - assert "description" in cols - assert "attachment_count" in cols - - -@pytest.mark.asyncio -async def test_description_and_attachment_count_round_trip(db): - pid = await _post(db, description="

hi

", attachment_count=3) - row = await db.get(Post, pid) - assert row.description == "

hi

" - assert row.attachment_count == 3 - - -@pytest.mark.asyncio -async def test_new_fields_default_null(db): - pid = await _post(db) - row = await db.get(Post, pid) - assert row.description is None - assert row.attachment_count is None diff --git a/tests/test_migration_0008.py b/tests/test_migration_0008.py deleted file mode 100644 index 0d7552a..0000000 --- a/tests/test_migration_0008.py +++ /dev/null @@ -1,137 +0,0 @@ -"""FC-2d-vii-c: image_record.artist_id + backfill + artist-tag delete.""" - -from datetime import UTC, datetime, timedelta - -import pytest -from sqlalchemy import func, select, text - -from backend.app.models import ( - Artist, - ImageProvenance, - ImageRecord, - Post, - Source, - Tag, - TagKind, -) -from backend.app.models.tag import image_tag -from backend.app.utils.artist_backfill import ( - BACKFILL_PRIMARY_SQL, - BACKFILL_PROVENANCE_SQL, - BACKFILL_TAG_SQL, - DELETE_ARTIST_TAGS_SQL, -) - -pytestmark = pytest.mark.integration - - -def test_image_record_has_artist_id_column(): - assert "artist_id" in {c.name for c in ImageRecord.__table__.columns} - - -async def _img(db, n): - rec = ImageRecord( - path=f"/images/bf/{n}.jpg", sha256=f"bf{n:062d}", - size_bytes=1, mime="image/jpeg", width=1, height=1, - origin="imported_filesystem", integrity_status="unknown", - ) - rec.created_at = datetime.now(UTC) - timedelta(minutes=n) - db.add(rec) - await db.flush() - return rec - - -async def _artist_source(db, name, slug): - a = Artist(name=name, slug=slug) - db.add(a) - await db.flush() - s = Source(artist_id=a.id, platform="patreon", - url=f"https://p.test/{slug}") - db.add(s) - await db.flush() - return a, s - - -async def _run_backfill(db): - await db.execute(text(BACKFILL_PRIMARY_SQL)) - await db.execute(text(BACKFILL_PROVENANCE_SQL)) - await db.execute(text(BACKFILL_TAG_SQL)) - - -@pytest.mark.asyncio -async def test_backfill_primary_post(db): - rec = await _img(db, 1) - a, s = await _artist_source(db, "Alice", "alice") - post = Post(source_id=s.id, artist_id=a.id, external_post_id="1") - db.add(post) - await db.flush() - rec.primary_post_id = post.id - await db.flush() - await _run_backfill(db) - got = await db.scalar( - select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) - ) - assert got == a.id - - -@pytest.mark.asyncio -async def test_backfill_provenance_fallback(db): - rec = await _img(db, 1) - a, s = await _artist_source(db, "Bob", "bob") - post = Post(source_id=s.id, artist_id=a.id, external_post_id="2") - db.add(post) - await db.flush() - db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id, - source_id=s.id)) - await db.flush() - await _run_backfill(db) - got = await db.scalar( - select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) - ) - assert got == a.id - - -@pytest.mark.asyncio -async def test_backfill_artist_tag_by_name(db): - rec = await _img(db, 1) - a = Artist(name="Carol", slug="carol") - db.add(a) - await db.flush() - tag = Tag(name="Carol", kind=TagKind.artist) - db.add(tag) - await db.flush() - await db.execute(image_tag.insert().values( - image_record_id=rec.id, tag_id=tag.id, source="auto")) - await db.flush() - await _run_backfill(db) - got = await db.scalar( - select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) - ) - assert got == a.id - - -@pytest.mark.asyncio -async def test_no_signal_stays_null(db): - rec = await _img(db, 1) - await _run_backfill(db) - got = await db.scalar( - select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) - ) - assert got is None - - -@pytest.mark.asyncio -async def test_delete_removes_only_artist_tags(db): - artist_tag = Tag(name="Dave", kind=TagKind.artist) - general_tag = Tag(name="forest", kind=TagKind.general) - db.add_all([artist_tag, general_tag]) - await db.flush() - await db.execute(text(DELETE_ARTIST_TAGS_SQL)) - remaining = await db.scalar( - select(func.count()).select_from(Tag).where(Tag.kind == TagKind.artist) - ) - assert remaining == 0 - survived = await db.scalar( - select(func.count()).select_from(Tag).where(Tag.kind == TagKind.general) - ) - assert survived >= 1 diff --git a/tests/test_migration_0009.py b/tests/test_migration_0009.py deleted file mode 100644 index f2a0d1f..0000000 --- a/tests/test_migration_0009.py +++ /dev/null @@ -1,37 +0,0 @@ -"""FC-2d-iii: post_attachment table + import_batch.attachments column.""" - -import pytest - -from backend.app.models import ImportBatch, PostAttachment - -pytestmark = pytest.mark.integration - - -def test_post_attachment_columns(): - cols = {c.name for c in PostAttachment.__table__.columns} - assert { - "id", "post_id", "artist_id", "sha256", "path", - "original_filename", "ext", "mime", "size_bytes", "captured_at", - } <= cols - - -def test_import_batch_has_attachments_counter(): - assert "attachments" in {c.name for c in ImportBatch.__table__.columns} - - -@pytest.mark.asyncio -async def test_post_attachment_roundtrip(db): - from backend.app.models import Artist - - a = Artist(name="Zed", slug="zed") - db.add(a) - await db.flush() - att = PostAttachment( - post_id=None, artist_id=a.id, sha256="z" + "0" * 63, - path="/images/attachments/z00/z.zip", original_filename="pack.zip", - ext=".zip", mime="application/zip", size_bytes=123, - ) - db.add(att) - await db.flush() - got = await db.get(PostAttachment, att.id) - assert got.original_filename == "pack.zip" and got.post_id is None diff --git a/tests/test_migration_0010.py b/tests/test_migration_0010.py deleted file mode 100644 index a261e6c..0000000 --- a/tests/test_migration_0010.py +++ /dev/null @@ -1,35 +0,0 @@ -import pytest -from sqlalchemy.exc import IntegrityError - -from backend.app.models import Artist, Source - -pytestmark = pytest.mark.integration - - -@pytest.mark.asyncio -async def test_duplicate_artist_platform_url_rejected(db): - artist = Artist(name="Alice", slug="alice") - db.add(artist) - await db.flush() - db.add(Source( - artist_id=artist.id, platform="patreon", - url="https://patreon.com/alice", enabled=True, - )) - await db.flush() - db.add(Source( - artist_id=artist.id, platform="patreon", - url="https://patreon.com/alice", enabled=True, - )) - with pytest.raises(IntegrityError): - await db.flush() - - -@pytest.mark.asyncio -async def test_same_url_under_different_artist_ok(db): - a = Artist(name="A", slug="a") - b = Artist(name="B", slug="b") - db.add_all([a, b]) - await db.flush() - db.add(Source(artist_id=a.id, platform="patreon", url="https://x/y", enabled=True)) - db.add(Source(artist_id=b.id, platform="patreon", url="https://x/y", enabled=True)) - await db.flush() # must NOT raise diff --git a/tests/test_migration_0011.py b/tests/test_migration_0011.py deleted file mode 100644 index 17e9702..0000000 --- a/tests/test_migration_0011.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest -from sqlalchemy import inspect, text - -pytestmark = pytest.mark.integration - - -@pytest.mark.asyncio -async def test_credential_has_credential_type_not_kind(db): - cols = (await db.run_sync( - lambda sync_session: [c["name"] for c in inspect(sync_session.bind).get_columns("credential")] - )) - assert "credential_type" in cols - assert "kind" not in cols - assert "status" not in cols - assert "last_verified" in cols - - -@pytest.mark.asyncio -async def test_credential_round_trip(db): - from backend.app.models import Credential - - db.add(Credential( - platform="patreon", - credential_type="cookies", - encrypted_blob=b"\x00\x01\x02", - )) - await db.flush() - row = (await db.execute( - text("SELECT credential_type, last_verified FROM credential WHERE platform='patreon'") - )).one() - assert row.credential_type == "cookies" - assert row.last_verified is None diff --git a/tests/test_migration_0012.py b/tests/test_migration_0012.py deleted file mode 100644 index a993ab4..0000000 --- a/tests/test_migration_0012.py +++ /dev/null @@ -1,32 +0,0 @@ -import pytest -from sqlalchemy import select - -from backend.app.models import AppSetting - -pytestmark = pytest.mark.integration - - -@pytest.mark.asyncio -async def test_app_setting_table_round_trip(db): - db.add(AppSetting(key="extension_api_key", value="abc123")) - await db.flush() - row = (await db.execute( - select(AppSetting).where(AppSetting.key == "extension_api_key") - )).scalar_one() - assert row.value == "abc123" - assert row.updated_at is not None - - -@pytest.mark.asyncio -async def test_app_setting_upsert(db): - db.add(AppSetting(key="k", value="v1")) - await db.flush() - row = (await db.execute( - select(AppSetting).where(AppSetting.key == "k") - )).scalar_one() - row.value = "v2" - await db.flush() - again = (await db.execute( - select(AppSetting.value).where(AppSetting.key == "k") - )).scalar_one() - assert again == "v2" diff --git a/tests/test_migration_0013.py b/tests/test_migration_0013.py deleted file mode 100644 index 1a18384..0000000 --- a/tests/test_migration_0013.py +++ /dev/null @@ -1,31 +0,0 @@ -import pytest -from sqlalchemy import inspect, select - -from backend.app.models import ImportSettings - -pytestmark = pytest.mark.integration - - -@pytest.mark.asyncio -async def test_download_event_has_metadata(db): - cols = await db.run_sync( - lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("download_event")} - ) - assert "metadata" in cols - assert cols["metadata"]["nullable"] is False - - -@pytest.mark.asyncio -async def test_import_settings_has_downloader_fields(db): - cols = await db.run_sync( - lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("import_settings")} - ) - assert "download_rate_limit_seconds" in cols - assert "download_validate_files" in cols - - -@pytest.mark.asyncio -async def test_import_settings_defaults(db): - row = (await db.execute(select(ImportSettings).where(ImportSettings.id == 1))).scalar_one() - assert row.download_rate_limit_seconds == 3.0 - assert row.download_validate_files is True -- 2.54.0 From 6959e1220c113888c81ab00490ae030e8eb279b0 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 14:34:28 -0400 Subject: [PATCH 08/17] Revert "db: collapse alembic 0001..0087 into one baseline" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts 2529b51. Not a retreat — a reordering, on the operator's call, and the better sequence. The squash's acceptance test (run 4971) found ~130 places where the ORM models do not describe the deployed schema (#3275), including a unique=True the database never had and two UNIQUE indexes that exist only in migrations. Collapsing now would have baked all of that into the one file a public installer starts from. So: fix the drift first as ordinary migrations on the intact chain, let the operator deploy so their database moves to the corrected head, and only then collapse. The baseline is then generated from reconciled models and reproduces a schema worth reproducing. Nothing is lost by reverting. The baseline was never deployed, and regenerating it after the fixes is strictly better than patching this copy — it will come out of autogenerate correct rather than needing the same hand-finishing twice. --- .../versions/0001_initial_unified_schema.py | 277 ++++++ .../0002_fc2a_tag_kinds_and_import_tasks.py | 208 +++++ alembic/versions/0003_fc2b_ml_pipeline.py | 172 ++++ .../versions/0004_fc2c_i_tsm_system_rows.py | 23 + .../versions/0005_fc2c_iii_a_series_page.py | 50 + alembic/versions/0006_fc2d_phash_threshold.py | 30 + .../0007_fc2d_post_metadata_fields.py | 31 + .../0008_fc2d_vii_c_artist_deconfliction.py | 52 ++ .../versions/0009_fc2d_iii_post_attachment.py | 68 ++ ..._fc3a_source_unique_artist_platform_url.py | 32 + .../0011_fc3b_credential_schema_alignment.py | 41 + alembic/versions/0012_fc3b_app_setting.py | 36 + .../0013_fc3c_download_event_metadata.py | 52 ++ alembic/versions/0014_fc3d_scheduling.py | 58 ++ alembic/versions/0015_fc5_migration_run.py | 51 + alembic/versions/0016_fc3i_task_run.py | 86 ++ alembic/versions/0017_fc3h_backup_run.py | 82 ++ alembic/versions/0018_fc3h_backup_settings.py | 62 ++ .../versions/0019_import_batch_refreshed.py | 38 + alembic/versions/0020_library_audit_run.py | 65 ++ .../versions/0021_image_provenance_unique.py | 54 ++ .../0022_source_per_artist_platform.py | 223 +++++ .../0023_drop_meta_rating_tag_kinds.py | 99 ++ ...24_backfill_post_title_from_description.py | 80 ++ .../0025_fix_subscribestar_post_ids.py | 288 ++++++ ...26_import_task_recovery_count_refetched.py | 53 ++ alembic/versions/0027_drop_migration_run.py | 50 + ...se_sidecar_synthetics_into_real_sources.py | 190 ++++ ...029_drop_artist_copyright_ml_thresholds.py | 71 ++ ...ullable_post_source_id_denorm_artist_id.py | 145 +++ .../0031_source_backfill_runs_remaining.py | 45 + alembic/versions/0032_source_error_type.py | 41 + .../0033_suggestion_threshold_default_070.py | 48 + alembic/versions/0034_artist_visit.py | 53 ++ .../0035_image_record_effective_date.py | 70 ++ .../0036_siglip_embedding_hnsw_index.py | 41 + alembic/versions/0037_patreon_seen_media.py | 53 ++ alembic/versions/0038_patreon_failed_media.py | 58 ++ alembic/versions/0039_library_audit_resume.py | 40 + alembic/versions/0040_series_chapters.py | 108 +++ alembic/versions/0041_series_suggestions.py | 98 ++ .../0042_series_chapter_stated_part.py | 32 + .../0043_post_attachment_per_post_unique.py | 62 ++ .../0044_ml_settings_tagger_store_floor.py | 37 + .../versions/0045_image_prediction_table.py | 69 ++ .../versions/0046_drop_tagger_predictions.py | 43 + .../versions/0047_series_chapter_dividers.py | 175 ++++ .../0048_series_page_pending_status.py | 45 + alembic/versions/0049_external_link_table.py | 90 ++ .../0050_external_link_host_toggles.py | 38 + .../versions/0051_image_source_provenance.py | 38 + .../versions/0052_image_duration_seconds.py | 32 + .../0053_ml_settings_video_tagging.py | 49 + .../versions/0054_subscribestar_ledgers.py | 82 ++ .../0055_image_provenance_from_attachment.py | 55 ++ alembic/versions/0056_tag_eval_run.py | 43 + .../0057_tag_positive_confirmation.py | 40 + alembic/versions/0058_tag_head.py | 95 ++ alembic/versions/0059_head_auto_apply.py | 70 ++ alembic/versions/0060_head_metrics.py | 74 ++ alembic/versions/0061_image_region.py | 59 ++ alembic/versions/0062_gpu_job.py | 55 ++ alembic/versions/0063_ccip_match_threshold.py | 33 + alembic/versions/0064_ccip_auto_apply.py | 42 + alembic/versions/0065_embedder_model_name.py | 35 + alembic/versions/0066_drop_centroids.py | 57 ++ .../versions/0067_retire_camie_allowlist.py | 66 ++ .../0068_drop_dead_tagger_settings.py | 80 ++ alembic/versions/0069_default_siglip2.py | 51 + .../versions/0070_gpu_job_lease_indexes.py | 44 + .../0071_image_record_earliest_post_date.py | 80 ++ .../versions/0072_gpu_job_triage_status.py | 32 + alembic/versions/0073_drop_tag_eval_run.py | 46 + .../0074_ml_settings_cpu_embed_enabled.py | 35 + alembic/versions/0075_tag_is_system.py | 60 ++ alembic/versions/0076_pixiv_ledgers.py | 82 ++ .../versions/0077_artist_name_not_unique.py | 32 + .../versions/0078_ml_settings_detectors.py | 83 ++ alembic/versions/0079_character_prototypes.py | 77 ++ .../0080_tag_head_train_fingerprint.py | 31 + .../0081_stricter_auto_apply_defaults.py | 43 + .../versions/0082_presentation_auto_hide.py | 85 ++ alembic/versions/0083_post_translation.py | 73 ++ .../0084_translation_strictness_override.py | 51 + alembic/versions/0085_wip_title_tagging.py | 35 + .../0086_process_auto_apply_settings.py | 61 ++ alembic/versions/0087_baseline.py | 872 ------------------ .../versions/0087_wip_soft_title_tagging.py | 33 + backend/app/utils/artist_backfill.py | 44 + tests/test_migration_0002.py | 58 ++ tests/test_migration_0003.py | 46 + tests/test_migration_0004.py | 27 + tests/test_migration_0007.py | 48 + tests/test_migration_0008.py | 137 +++ tests/test_migration_0009.py | 37 + tests/test_migration_0010.py | 35 + tests/test_migration_0011.py | 32 + tests/test_migration_0012.py | 32 + tests/test_migration_0013.py | 31 + 99 files changed, 6579 insertions(+), 872 deletions(-) create mode 100644 alembic/versions/0001_initial_unified_schema.py create mode 100644 alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py create mode 100644 alembic/versions/0003_fc2b_ml_pipeline.py create mode 100644 alembic/versions/0004_fc2c_i_tsm_system_rows.py create mode 100644 alembic/versions/0005_fc2c_iii_a_series_page.py create mode 100644 alembic/versions/0006_fc2d_phash_threshold.py create mode 100644 alembic/versions/0007_fc2d_post_metadata_fields.py create mode 100644 alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py create mode 100644 alembic/versions/0009_fc2d_iii_post_attachment.py create mode 100644 alembic/versions/0010_fc3a_source_unique_artist_platform_url.py create mode 100644 alembic/versions/0011_fc3b_credential_schema_alignment.py create mode 100644 alembic/versions/0012_fc3b_app_setting.py create mode 100644 alembic/versions/0013_fc3c_download_event_metadata.py create mode 100644 alembic/versions/0014_fc3d_scheduling.py create mode 100644 alembic/versions/0015_fc5_migration_run.py create mode 100644 alembic/versions/0016_fc3i_task_run.py create mode 100644 alembic/versions/0017_fc3h_backup_run.py create mode 100644 alembic/versions/0018_fc3h_backup_settings.py create mode 100644 alembic/versions/0019_import_batch_refreshed.py create mode 100644 alembic/versions/0020_library_audit_run.py create mode 100644 alembic/versions/0021_image_provenance_unique.py create mode 100644 alembic/versions/0022_source_per_artist_platform.py create mode 100644 alembic/versions/0023_drop_meta_rating_tag_kinds.py create mode 100644 alembic/versions/0024_backfill_post_title_from_description.py create mode 100644 alembic/versions/0025_fix_subscribestar_post_ids.py create mode 100644 alembic/versions/0026_import_task_recovery_count_refetched.py create mode 100644 alembic/versions/0027_drop_migration_run.py create mode 100644 alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py create mode 100644 alembic/versions/0029_drop_artist_copyright_ml_thresholds.py create mode 100644 alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py create mode 100644 alembic/versions/0031_source_backfill_runs_remaining.py create mode 100644 alembic/versions/0032_source_error_type.py create mode 100644 alembic/versions/0033_suggestion_threshold_default_070.py create mode 100644 alembic/versions/0034_artist_visit.py create mode 100644 alembic/versions/0035_image_record_effective_date.py create mode 100644 alembic/versions/0036_siglip_embedding_hnsw_index.py create mode 100644 alembic/versions/0037_patreon_seen_media.py create mode 100644 alembic/versions/0038_patreon_failed_media.py create mode 100644 alembic/versions/0039_library_audit_resume.py create mode 100644 alembic/versions/0040_series_chapters.py create mode 100644 alembic/versions/0041_series_suggestions.py create mode 100644 alembic/versions/0042_series_chapter_stated_part.py create mode 100644 alembic/versions/0043_post_attachment_per_post_unique.py create mode 100644 alembic/versions/0044_ml_settings_tagger_store_floor.py create mode 100644 alembic/versions/0045_image_prediction_table.py create mode 100644 alembic/versions/0046_drop_tagger_predictions.py create mode 100644 alembic/versions/0047_series_chapter_dividers.py create mode 100644 alembic/versions/0048_series_page_pending_status.py create mode 100644 alembic/versions/0049_external_link_table.py create mode 100644 alembic/versions/0050_external_link_host_toggles.py create mode 100644 alembic/versions/0051_image_source_provenance.py create mode 100644 alembic/versions/0052_image_duration_seconds.py create mode 100644 alembic/versions/0053_ml_settings_video_tagging.py create mode 100644 alembic/versions/0054_subscribestar_ledgers.py create mode 100644 alembic/versions/0055_image_provenance_from_attachment.py create mode 100644 alembic/versions/0056_tag_eval_run.py create mode 100644 alembic/versions/0057_tag_positive_confirmation.py create mode 100644 alembic/versions/0058_tag_head.py create mode 100644 alembic/versions/0059_head_auto_apply.py create mode 100644 alembic/versions/0060_head_metrics.py create mode 100644 alembic/versions/0061_image_region.py create mode 100644 alembic/versions/0062_gpu_job.py create mode 100644 alembic/versions/0063_ccip_match_threshold.py create mode 100644 alembic/versions/0064_ccip_auto_apply.py create mode 100644 alembic/versions/0065_embedder_model_name.py create mode 100644 alembic/versions/0066_drop_centroids.py create mode 100644 alembic/versions/0067_retire_camie_allowlist.py create mode 100644 alembic/versions/0068_drop_dead_tagger_settings.py create mode 100644 alembic/versions/0069_default_siglip2.py create mode 100644 alembic/versions/0070_gpu_job_lease_indexes.py create mode 100644 alembic/versions/0071_image_record_earliest_post_date.py create mode 100644 alembic/versions/0072_gpu_job_triage_status.py create mode 100644 alembic/versions/0073_drop_tag_eval_run.py create mode 100644 alembic/versions/0074_ml_settings_cpu_embed_enabled.py create mode 100644 alembic/versions/0075_tag_is_system.py create mode 100644 alembic/versions/0076_pixiv_ledgers.py create mode 100644 alembic/versions/0077_artist_name_not_unique.py create mode 100644 alembic/versions/0078_ml_settings_detectors.py create mode 100644 alembic/versions/0079_character_prototypes.py create mode 100644 alembic/versions/0080_tag_head_train_fingerprint.py create mode 100644 alembic/versions/0081_stricter_auto_apply_defaults.py create mode 100644 alembic/versions/0082_presentation_auto_hide.py create mode 100644 alembic/versions/0083_post_translation.py create mode 100644 alembic/versions/0084_translation_strictness_override.py create mode 100644 alembic/versions/0085_wip_title_tagging.py create mode 100644 alembic/versions/0086_process_auto_apply_settings.py delete mode 100644 alembic/versions/0087_baseline.py create mode 100644 alembic/versions/0087_wip_soft_title_tagging.py create mode 100644 backend/app/utils/artist_backfill.py create mode 100644 tests/test_migration_0002.py create mode 100644 tests/test_migration_0003.py create mode 100644 tests/test_migration_0004.py create mode 100644 tests/test_migration_0007.py create mode 100644 tests/test_migration_0008.py create mode 100644 tests/test_migration_0009.py create mode 100644 tests/test_migration_0010.py create mode 100644 tests/test_migration_0011.py create mode 100644 tests/test_migration_0012.py create mode 100644 tests/test_migration_0013.py diff --git a/alembic/versions/0001_initial_unified_schema.py b/alembic/versions/0001_initial_unified_schema.py new file mode 100644 index 0000000..0580b45 --- /dev/null +++ b/alembic/versions/0001_initial_unified_schema.py @@ -0,0 +1,277 @@ +"""initial unified schema + +Revision ID: 0001 +Revises: +Create Date: 2026-05-13 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector + +revision: str = "0001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS vector") + + op.create_table( + "artist", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("slug", sa.String(length=255), nullable=False), + sa.Column("notes", sa.Text(), nullable=True), + sa.Column("is_subscription", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column("auto_check", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("check_interval_seconds", sa.Integer(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_artist"), + sa.UniqueConstraint("name", name="uq_artist_name"), + sa.UniqueConstraint("slug", name="uq_artist_slug"), + ) + + op.create_table( + "source", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("artist_id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.true()), + sa.Column("config_overrides", sa.JSON(), nullable=True), + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("check_interval_override", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["artist_id"], ["artist.id"], name="fk_source_artist_id_artist", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name="pk_source"), + ) + op.create_index("ix_source_artist_id", "source", ["artist_id"]) + + op.create_table( + "credential", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("platform", sa.String(length=64), nullable=False), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("encrypted_blob", sa.LargeBinary(), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False, server_default="active"), + sa.Column( + "captured_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint("id", name="pk_credential"), + sa.UniqueConstraint("platform", name="uq_credential_platform"), + ) + + op.create_table( + "post", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("external_post_id", sa.String(length=128), nullable=False), + sa.Column("post_url", sa.Text(), nullable=True), + sa.Column("post_title", sa.Text(), nullable=True), + sa.Column("post_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("raw_metadata", sa.JSON(), nullable=True), + sa.Column( + "downloaded_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["source_id"], ["source.id"], name="fk_post_source_id_source", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id", name="pk_post"), + sa.UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), + ) + op.create_index("ix_post_source_id", "post", ["source_id"]) + + op.create_table( + "image_record", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("sha256", sa.String(length=64), nullable=False), + sa.Column("phash", sa.String(length=32), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("mime", sa.String(length=64), nullable=False), + sa.Column("width", sa.Integer(), nullable=True), + sa.Column("height", sa.Integer(), nullable=True), + sa.Column("thumbnail_path", sa.Text(), nullable=True), + sa.Column( + "origin", + sa.Enum( + "downloaded", + "imported_filesystem", + "uploaded", + name="origin_enum", + ), + nullable=False, + ), + sa.Column("primary_post_id", sa.Integer(), nullable=True), + sa.Column("wd14_predictions", sa.JSON(), nullable=True), + sa.Column("wd14_model_version", sa.String(length=128), nullable=True), + sa.Column("siglip_embedding", Vector(1152), nullable=True), + sa.Column("siglip_model_version", sa.String(length=128), nullable=True), + sa.Column("centroid_scores", sa.JSON(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["primary_post_id"], + ["post.id"], + name="fk_image_record_primary_post_id_post", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_image_record"), + sa.UniqueConstraint("path", name="uq_image_record_path"), + sa.UniqueConstraint("sha256", name="uq_image_record_sha256"), + ) + op.create_index("ix_image_record_sha256", "image_record", ["sha256"]) + op.create_index("ix_image_record_phash", "image_record", ["phash"]) + op.create_index("ix_image_record_primary_post_id", "image_record", ["primary_post_id"]) + + op.create_table( + "image_provenance", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("post_id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("captured_metadata", sa.JSON(), nullable=True), + sa.Column( + "captured_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], + ["image_record.id"], + name="fk_image_provenance_image_record_id_image_record", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["post.id"], + name="fk_image_provenance_post_id_post", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["source_id"], + ["source.id"], + name="fk_image_provenance_source_id_source", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_image_provenance"), + ) + op.create_index("ix_image_provenance_image_record_id", "image_provenance", ["image_record_id"]) + op.create_index("ix_image_provenance_post_id", "image_provenance", ["post_id"]) + op.create_index("ix_image_provenance_source_id", "image_provenance", ["source_id"]) + + op.create_table( + "tag", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("namespace", sa.String(length=64), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_tag"), + sa.UniqueConstraint("name", name="uq_tag_name"), + ) + op.create_index("ix_tag_name", "tag", ["name"]) + op.create_index("ix_tag_namespace", "tag", ["namespace"]) + + op.create_table( + "image_tag", + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column("source", sa.String(length=32), nullable=False, server_default="manual"), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], + ["image_record.id"], + name="fk_image_tag_image_record_id_image_record", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], name="fk_image_tag_tag_id_tag", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("image_record_id", "tag_id", name="pk_image_tag"), + ) + + op.create_table( + "download_event", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("source_id", sa.Integer(), nullable=False), + sa.Column("post_id", sa.Integer(), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("bytes_downloaded", sa.BigInteger(), nullable=False, server_default="0"), + sa.Column("files_count", sa.Integer(), nullable=False, server_default="0"), + sa.Column("error", sa.Text(), nullable=True), + sa.ForeignKeyConstraint( + ["source_id"], + ["source.id"], + name="fk_download_event_source_id_source", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["post_id"], + ["post.id"], + name="fk_download_event_post_id_post", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_download_event"), + ) + op.create_index("ix_download_event_source_id", "download_event", ["source_id"]) + op.create_index("ix_download_event_post_id", "download_event", ["post_id"]) + + +def downgrade() -> None: + op.drop_table("download_event") + op.drop_table("image_tag") + op.drop_table("tag") + op.drop_table("image_provenance") + op.drop_table("image_record") + op.execute("DROP TYPE IF EXISTS origin_enum") + op.drop_table("post") + op.drop_table("credential") + op.drop_table("source") + op.drop_table("artist") + op.execute("DROP EXTENSION IF EXISTS vector") diff --git a/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py b/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py new file mode 100644 index 0000000..b9dac2c --- /dev/null +++ b/alembic/versions/0002_fc2a_tag_kinds_and_import_tasks.py @@ -0,0 +1,208 @@ +"""fc2a: tag kinds, import_task, import_batch, integrity_status + +Revision ID: 0002 +Revises: 0001 +Create Date: 2026-05-14 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0002" +down_revision: Union[str, None] = "0001" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +TAG_KINDS = ( + "artist", + "character", + "fandom", + "general", + "series", + "archive", + "post", + "meta", + "rating", +) + + +def upgrade() -> None: + # --- Tag kind enum + fandom_id --- + tag_kind = sa.Enum(*TAG_KINDS, name="tag_kind") + tag_kind.create(op.get_bind(), checkfirst=True) + + op.add_column( + "tag", + sa.Column("kind", tag_kind, nullable=False, server_default="general"), + ) + op.add_column( + "tag", + sa.Column("fandom_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_tag_fandom_id_tag", + "tag", + "tag", + ["fandom_id"], + ["id"], + ondelete="SET NULL", + ) + + # Drop the old global uniqueness on name; add kind+fandom-aware uniqueness. + op.drop_constraint("uq_tag_name", "tag", type_="unique") + op.drop_index("ix_tag_name", table_name="tag") + op.execute( + """ + CREATE UNIQUE INDEX uq_tag_name_kind_fandom + ON tag (name, kind, COALESCE(fandom_id, 0)) + """ + ) + + # CHECK: fandom_id is only allowed for character kind. + op.create_check_constraint( + "ck_tag_fandom_requires_character", + "tag", + "(fandom_id IS NULL) OR (kind = 'character')", + ) + + # Drop the old namespace column — superseded by kind. + op.drop_index("ix_tag_namespace", table_name="tag") + op.drop_column("tag", "namespace") + + # --- ImportBatch --- + op.create_table( + "import_batch", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("triggered_by", sa.String(length=32), nullable=False), + sa.Column("source_path", sa.Text(), nullable=False), + sa.Column("scan_mode", sa.String(length=16), nullable=False), + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("total_files", sa.Integer(), nullable=False, server_default="0"), + sa.Column("imported", sa.Integer(), nullable=False, server_default="0"), + sa.Column("skipped", sa.Integer(), nullable=False, server_default="0"), + sa.Column("failed", sa.Integer(), nullable=False, server_default="0"), + sa.Column("status", sa.String(length=16), nullable=False, server_default="running"), + sa.PrimaryKeyConstraint("id", name="pk_import_batch"), + ) + op.create_index("ix_import_batch_status", "import_batch", ["status"]) + + # --- ImportTask --- + op.create_table( + "import_task", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("batch_id", sa.Integer(), nullable=False), + sa.Column("source_path", sa.Text(), nullable=False), + sa.Column("task_type", sa.String(length=16), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False, server_default="pending"), + sa.Column("result_image_id", sa.Integer(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.func.now(), + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["batch_id"], + ["import_batch.id"], + name="fk_import_task_batch_id_import_batch", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["result_image_id"], + ["image_record.id"], + name="fk_import_task_result_image_id_image_record", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name="pk_import_task"), + ) + op.create_index("ix_import_task_batch_id", "import_task", ["batch_id"]) + op.create_index("ix_import_task_status", "import_task", ["status"]) + op.create_index( + "ix_import_task_created_at_desc", + "import_task", + [sa.text("created_at DESC")], + ) + + # --- ImportSettings (single-row table) --- + op.create_table( + "import_settings", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("import_scan_path", sa.Text(), nullable=False, server_default="/import"), + sa.Column("min_width", sa.Integer(), nullable=False, server_default="0"), + sa.Column("min_height", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "skip_transparent", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column( + "transparency_threshold", + sa.Float(), + nullable=False, + server_default="0.9", + ), + sa.Column( + "skip_single_color", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column( + "single_color_threshold", + sa.Float(), + nullable=False, + server_default="0.95", + ), + sa.Column("single_color_tolerance", sa.Integer(), nullable=False, server_default="30"), + sa.PrimaryKeyConstraint("id", name="pk_import_settings"), + sa.CheckConstraint("id = 1", name="ck_import_settings_singleton"), + ) + # Seed the single row immediately so callers can always SELECT id=1. + op.execute("INSERT INTO import_settings (id) VALUES (1)") + + # --- ImageRecord additions --- + op.add_column( + "image_record", + sa.Column( + "integrity_status", + sa.String(length=24), + nullable=False, + server_default="unknown", + ), + ) + op.create_index( + "ix_image_record_integrity_status", + "image_record", + ["integrity_status"], + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_integrity_status", table_name="image_record") + op.drop_column("image_record", "integrity_status") + + op.drop_table("import_settings") + op.drop_index("ix_import_task_created_at_desc", table_name="import_task") + op.drop_index("ix_import_task_status", table_name="import_task") + op.drop_index("ix_import_task_batch_id", table_name="import_task") + op.drop_table("import_task") + op.drop_index("ix_import_batch_status", table_name="import_batch") + op.drop_table("import_batch") + + op.drop_constraint("ck_tag_fandom_requires_character", "tag", type_="check") + op.execute("DROP INDEX uq_tag_name_kind_fandom") + op.add_column("tag", sa.Column("namespace", sa.String(length=64), nullable=True)) + op.create_index("ix_tag_namespace", "tag", ["namespace"]) + op.create_index("ix_tag_name", "tag", ["name"], unique=False) + op.create_unique_constraint("uq_tag_name", "tag", ["name"]) + op.drop_constraint("fk_tag_fandom_id_tag", "tag", type_="foreignkey") + op.drop_column("tag", "fandom_id") + op.drop_column("tag", "kind") + sa.Enum(name="tag_kind").drop(op.get_bind(), checkfirst=True) diff --git a/alembic/versions/0003_fc2b_ml_pipeline.py b/alembic/versions/0003_fc2b_ml_pipeline.py new file mode 100644 index 0000000..584bffe --- /dev/null +++ b/alembic/versions/0003_fc2b_ml_pipeline.py @@ -0,0 +1,172 @@ +"""fc2b: ML pipeline — allowlist, aliases, centroids, ml_settings + +Revision ID: 0003 +Revises: 0002 +Create Date: 2026-05-15 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector + +revision: str = "0003" +down_revision: Union[str, None] = "0002" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 3.1 rename wd14_* -> tagger_* + op.alter_column("image_record", "wd14_predictions", new_column_name="tagger_predictions") + op.alter_column( + "image_record", "wd14_model_version", new_column_name="tagger_model_version" + ) + + # 3.2 tag_allowlist + op.create_table( + "tag_allowlist", + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column( + "min_confidence", sa.Float(), nullable=False, server_default="0.95" + ), + sa.Column( + "added_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], name="fk_tag_allowlist_tag_id_tag", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("tag_id", name="pk_tag_allowlist"), + sa.CheckConstraint( + "min_confidence > 0 AND min_confidence <= 1", + name="ck_tag_allowlist_confidence_range", + ), + ) + + # 3.3 tag_suggestion_rejection + op.create_table( + "tag_suggestion_rejection", + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column( + "rejected_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["image_record_id"], ["image_record.id"], + name="fk_tsr_image_record_id_image_record", ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], name="fk_tsr_tag_id_tag", ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "image_record_id", "tag_id", name="pk_tag_suggestion_rejection" + ), + ) + op.create_index( + "ix_tag_suggestion_rejection_tag", "tag_suggestion_rejection", ["tag_id"] + ) + + # 3.4 tag_alias + op.create_table( + "tag_alias", + sa.Column("alias_string", sa.String(length=255), nullable=False), + sa.Column("alias_category", sa.String(length=32), nullable=False), + sa.Column("canonical_tag_id", sa.Integer(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["canonical_tag_id"], ["tag.id"], + name="fk_tag_alias_canonical_tag_id_tag", ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "alias_string", "alias_category", name="pk_tag_alias" + ), + ) + op.create_index("ix_tag_alias_canonical", "tag_alias", ["canonical_tag_id"]) + + # 3.5 tag_reference_embedding (centroids) + op.create_table( + "tag_reference_embedding", + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column("embedding", Vector(1152), nullable=False), + sa.Column("reference_count", sa.Integer(), nullable=False), + sa.Column("model_version", sa.String(length=128), nullable=False), + sa.Column( + "updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["tag_id"], ["tag.id"], + name="fk_tag_reference_embedding_tag_id_tag", ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("tag_id", name="pk_tag_reference_embedding"), + ) + + # 3.6 ml_settings singleton + op.create_table( + "ml_settings", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column( + "suggestion_threshold_artist", sa.Float(), nullable=False, + server_default="0.30", + ), + sa.Column( + "suggestion_threshold_character", sa.Float(), nullable=False, + server_default="0.50", + ), + sa.Column( + "suggestion_threshold_copyright", sa.Float(), nullable=False, + server_default="0.50", + ), + sa.Column( + "suggestion_threshold_general", sa.Float(), nullable=False, + server_default="0.95", + ), + sa.Column( + "centroid_similarity_threshold", sa.Float(), nullable=False, + server_default="0.55", + ), + sa.Column( + "min_reference_images", sa.Integer(), nullable=False, + server_default="5", + ), + sa.Column( + "tagger_model_version", sa.String(length=128), nullable=False, + server_default="camie-tagger-v2", + ), + sa.Column( + "embedder_model_version", sa.String(length=128), nullable=False, + server_default="siglip-so400m-patch14-384", + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.PrimaryKeyConstraint("id", name="pk_ml_settings"), + sa.CheckConstraint("id = 1", name="ck_ml_settings_singleton"), + ) + op.execute("INSERT INTO ml_settings (id) VALUES (1)") + + +def downgrade() -> None: + op.drop_table("ml_settings") + op.drop_table("tag_reference_embedding") + op.drop_index("ix_tag_alias_canonical", table_name="tag_alias") + op.drop_table("tag_alias") + op.drop_index( + "ix_tag_suggestion_rejection_tag", table_name="tag_suggestion_rejection" + ) + op.drop_table("tag_suggestion_rejection") + op.drop_table("tag_allowlist") + op.alter_column( + "image_record", "tagger_model_version", new_column_name="wd14_model_version" + ) + op.alter_column( + "image_record", "tagger_predictions", new_column_name="wd14_predictions" + ) diff --git a/alembic/versions/0004_fc2c_i_tsm_system_rows.py b/alembic/versions/0004_fc2c_i_tsm_system_rows.py new file mode 100644 index 0000000..e8bd920 --- /dev/null +++ b/alembic/versions/0004_fc2c_i_tsm_system_rows.py @@ -0,0 +1,23 @@ +"""fc2c-i: enable tsm_system_rows for scalable random sampling + +Revision ID: 0004 +Revises: 0003 +Create Date: 2026-05-15 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0004" +down_revision: Union[str, None] = "0003" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows") + + +def downgrade() -> None: + op.execute("DROP EXTENSION IF EXISTS tsm_system_rows") diff --git a/alembic/versions/0005_fc2c_iii_a_series_page.py b/alembic/versions/0005_fc2c_iii_a_series_page.py new file mode 100644 index 0000000..ffe397e --- /dev/null +++ b/alembic/versions/0005_fc2c_iii_a_series_page.py @@ -0,0 +1,50 @@ +"""fc2c-iii-a: series_page ordered membership + +Revision ID: 0005 +Revises: 0004 +Create Date: 2026-05-16 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0005" +down_revision: Union[str, None] = "0004" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_page", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("series_tag_id", sa.Integer(), nullable=False), + sa.Column("image_id", sa.Integer(), nullable=False), + sa.Column("page_number", sa.Integer(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.ForeignKeyConstraint( + ["series_tag_id"], ["tag.id"], ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["image_id"], ["image_record.id"], ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("image_id", name="uq_series_page_image"), + ) + op.create_index( + "ix_series_page_series_tag_id", "series_page", ["series_tag_id"] + ) + + +def downgrade() -> None: + op.drop_index("ix_series_page_series_tag_id", table_name="series_page") + op.drop_table("series_page") diff --git a/alembic/versions/0006_fc2d_phash_threshold.py b/alembic/versions/0006_fc2d_phash_threshold.py new file mode 100644 index 0000000..895ed46 --- /dev/null +++ b/alembic/versions/0006_fc2d_phash_threshold.py @@ -0,0 +1,30 @@ +"""fc2d: import_settings.phash_threshold + +Revision ID: 0006 +Revises: 0005 +Create Date: 2026-05-17 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0006" +down_revision: Union[str, None] = "0005" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_settings", + sa.Column( + "phash_threshold", sa.Integer(), + nullable=False, server_default="10", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "phash_threshold") diff --git a/alembic/versions/0007_fc2d_post_metadata_fields.py b/alembic/versions/0007_fc2d_post_metadata_fields.py new file mode 100644 index 0000000..24e8ba9 --- /dev/null +++ b/alembic/versions/0007_fc2d_post_metadata_fields.py @@ -0,0 +1,31 @@ +"""fc2d-iv: post.description + post.attachment_count + +Revision ID: 0007 +Revises: 0006 +Create Date: 2026-05-18 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0007" +down_revision: Union[str, None] = "0006" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "post", sa.Column("description", sa.Text(), nullable=True) + ) + op.add_column( + "post", + sa.Column("attachment_count", sa.Integer(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("post", "attachment_count") + op.drop_column("post", "description") diff --git a/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py b/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py new file mode 100644 index 0000000..4019e2d --- /dev/null +++ b/alembic/versions/0008_fc2d_vii_c_artist_deconfliction.py @@ -0,0 +1,52 @@ +"""fc2d-vii-c: image_record.artist_id + backfill + drop artist tags + +Revision ID: 0008 +Revises: 0007 +Create Date: 2026-05-18 + +Internal forward-correctness migration (the big legacy-import migration +stays deferred). downgrade() does NOT recreate deleted artist tags; +downgrade is dev-only and the data is reconstructable by re-import. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +from backend.app.utils.artist_backfill import ( + BACKFILL_PRIMARY_SQL, + BACKFILL_PROVENANCE_SQL, + BACKFILL_TAG_SQL, + DELETE_ARTIST_TAGS_SQL, +) + +revision: str = "0008" +down_revision: Union[str, None] = "0007" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "image_record", + sa.Column("artist_id", sa.Integer(), nullable=True), + ) + op.create_foreign_key( + "fk_image_record_artist_id", "image_record", "artist", + ["artist_id"], ["id"], ondelete="SET NULL", + ) + op.create_index( + "ix_image_record_artist_id", "image_record", ["artist_id"], + ) + op.execute(BACKFILL_PRIMARY_SQL) + op.execute(BACKFILL_PROVENANCE_SQL) + op.execute(BACKFILL_TAG_SQL) + op.execute(DELETE_ARTIST_TAGS_SQL) + + +def downgrade() -> None: + op.drop_index("ix_image_record_artist_id", table_name="image_record") + op.drop_constraint( + "fk_image_record_artist_id", "image_record", type_="foreignkey" + ) + op.drop_column("image_record", "artist_id") diff --git a/alembic/versions/0009_fc2d_iii_post_attachment.py b/alembic/versions/0009_fc2d_iii_post_attachment.py new file mode 100644 index 0000000..4820fa4 --- /dev/null +++ b/alembic/versions/0009_fc2d_iii_post_attachment.py @@ -0,0 +1,68 @@ +"""fc2d-iii: post_attachment + import_batch.attachments + +Revision ID: 0009 +Revises: 0008 +Create Date: 2026-05-19 + +Internal forward-correctness migration (big legacy-import migration +stays deferred). No backfill — no attachments exist yet. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0009" +down_revision: Union[str, None] = "0008" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "post_attachment", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "post_id", sa.Integer(), + sa.ForeignKey("post.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column( + "artist_id", sa.Integer(), + sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column("sha256", sa.String(64), nullable=False), + sa.Column("path", sa.Text(), nullable=False), + sa.Column("original_filename", sa.Text(), nullable=False), + sa.Column("ext", sa.String(32), nullable=False), + sa.Column("mime", sa.String(128), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column( + "captured_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False, + ), + ) + op.create_index( + "ix_post_attachment_sha256", "post_attachment", ["sha256"], + unique=True, + ) + op.create_index( + "ix_post_attachment_post_id", "post_attachment", ["post_id"], + ) + op.create_index( + "ix_post_attachment_artist_id", "post_attachment", ["artist_id"], + ) + op.add_column( + "import_batch", + sa.Column( + "attachments", sa.Integer(), nullable=False, + server_default="0", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_batch", "attachments") + op.drop_index("ix_post_attachment_artist_id", table_name="post_attachment") + op.drop_index("ix_post_attachment_post_id", table_name="post_attachment") + op.drop_index("ix_post_attachment_sha256", table_name="post_attachment") + op.drop_table("post_attachment") diff --git a/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py b/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py new file mode 100644 index 0000000..10f502b --- /dev/null +++ b/alembic/versions/0010_fc3a_source_unique_artist_platform_url.py @@ -0,0 +1,32 @@ +"""fc3a: unique(source.artist_id, source.platform, source.url) + +Revision ID: 0010 +Revises: 0009 +Create Date: 2026-05-20 + +Enforces FC-3a's dedup invariant at the DB level. No backfill — no +existing rows are expected to collide; if they do the migration will +fail loudly (intended). +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0010" +down_revision: Union[str, None] = "0009" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_unique_constraint( + "uq_source_artist_platform_url", + "source", + ["artist_id", "platform", "url"], + ) + + +def downgrade() -> None: + op.drop_constraint( + "uq_source_artist_platform_url", "source", type_="unique" + ) diff --git a/alembic/versions/0011_fc3b_credential_schema_alignment.py b/alembic/versions/0011_fc3b_credential_schema_alignment.py new file mode 100644 index 0000000..00a31a5 --- /dev/null +++ b/alembic/versions/0011_fc3b_credential_schema_alignment.py @@ -0,0 +1,41 @@ +"""fc3b: rename credential.kind -> credential_type, drop status, add last_verified + +Revision ID: 0011 +Revises: 0010 +Create Date: 2026-05-20 + +Aligns the credential table with the GallerySubscriber wire-field names +so the existing browser extension can POST to FC unmodified. Greenfield — +no rows exist in production yet, so no data preservation logic is +needed; the rename uses ALTER COLUMN rather than copy-then-drop. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0011" +down_revision: Union[str, None] = "0010" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.alter_column("credential", "kind", new_column_name="credential_type") + op.drop_column("credential", "status") + op.add_column( + "credential", + sa.Column("last_verified", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("credential", "last_verified") + op.add_column( + "credential", + sa.Column( + "status", sa.String(length=32), nullable=False, + server_default="active", + ), + ) + op.alter_column("credential", "credential_type", new_column_name="kind") diff --git a/alembic/versions/0012_fc3b_app_setting.py b/alembic/versions/0012_fc3b_app_setting.py new file mode 100644 index 0000000..42c06eb --- /dev/null +++ b/alembic/versions/0012_fc3b_app_setting.py @@ -0,0 +1,36 @@ +"""fc3b: app_setting key/value table + +Revision ID: 0012 +Revises: 0011 +Create Date: 2026-05-20 + +A simple key/value table for small app settings that don't fit +ImportSettings. Initially seeds only `extension_api_key` (done in +create_app on first boot — not in the migration, to keep it +deterministic and independent of randomness). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0012" +down_revision: Union[str, None] = "0011" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "app_setting", + sa.Column("key", sa.String(length=64), primary_key=True), + sa.Column("value", sa.Text(), nullable=False), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + ) + + +def downgrade() -> None: + op.drop_table("app_setting") diff --git a/alembic/versions/0013_fc3c_download_event_metadata.py b/alembic/versions/0013_fc3c_download_event_metadata.py new file mode 100644 index 0000000..c88b3f9 --- /dev/null +++ b/alembic/versions/0013_fc3c_download_event_metadata.py @@ -0,0 +1,52 @@ +"""fc3c: download_event.metadata + import_settings downloader fields + +Revision ID: 0013 +Revises: 0012 +Create Date: 2026-05-20 + +Additive only. download_event.metadata is the rich JSONB blob FC-3c +populates per run (run_stats, stdout/stderr, quarantined paths, import +summary). import_settings gains two operator-tunable downloader knobs: +download_rate_limit_seconds (gallery-dl extractor.sleep) and +download_validate_files (toggle the magic-byte validator). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0013" +down_revision: Union[str, None] = "0012" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "download_event", + sa.Column( + "metadata", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_rate_limit_seconds", sa.Float(), + nullable=False, server_default="3.0", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_validate_files", sa.Boolean(), + nullable=False, server_default=sa.true(), + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "download_validate_files") + op.drop_column("import_settings", "download_rate_limit_seconds") + op.drop_column("download_event", "metadata") diff --git a/alembic/versions/0014_fc3d_scheduling.py b/alembic/versions/0014_fc3d_scheduling.py new file mode 100644 index 0000000..955e956 --- /dev/null +++ b/alembic/versions/0014_fc3d_scheduling.py @@ -0,0 +1,58 @@ +"""fc3d: scheduling + source health columns + +Revision ID: 0014 +Revises: 0013 +Create Date: 2026-05-21 + +Additive only. source.consecutive_failures (default 0, DownloadService +finalize hook owns the writes). import_settings gains the three +scheduling knobs (global default interval, event retention, failure +warning threshold). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0014" +down_revision: Union[str, None] = "0013" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "source", + sa.Column( + "consecutive_failures", sa.Integer(), + nullable=False, server_default="0", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_schedule_default_seconds", sa.Integer(), + nullable=False, server_default="28800", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_event_retention_days", sa.Integer(), + nullable=False, server_default="90", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "download_failure_warning_threshold", sa.Integer(), + nullable=False, server_default="5", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "download_failure_warning_threshold") + op.drop_column("import_settings", "download_event_retention_days") + op.drop_column("import_settings", "download_schedule_default_seconds") + op.drop_column("source", "consecutive_failures") diff --git a/alembic/versions/0015_fc5_migration_run.py b/alembic/versions/0015_fc5_migration_run.py new file mode 100644 index 0000000..89d7d89 --- /dev/null +++ b/alembic/versions/0015_fc5_migration_run.py @@ -0,0 +1,51 @@ +"""fc5: migration_run table + +Revision ID: 0015 +Revises: 0014 +Create Date: 2026-05-22 + +Additive only. New table tracks each invocation of the FC-5 migration +tooling (backup, gs, ir, ml_queue, verify, rollback). kind/status are +plain String(32) — values validated at the API layer per the spec, not +a Postgres ENUM (so adding kinds later doesn't need a schema migration). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0015" +down_revision: Union[str, None] = "0014" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "migration_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("kind", sa.String(32), nullable=False, index=True), + sa.Column("status", sa.String(32), nullable=False, index=True), + sa.Column( + "dry_run", sa.Boolean(), nullable=False, server_default=sa.false(), + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "counts", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + sa.Column( + "metadata", postgresql.JSONB, + nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + + +def downgrade() -> None: + op.drop_table("migration_run") diff --git a/alembic/versions/0016_fc3i_task_run.py b/alembic/versions/0016_fc3i_task_run.py new file mode 100644 index 0000000..678b1ee --- /dev/null +++ b/alembic/versions/0016_fc3i_task_run.py @@ -0,0 +1,86 @@ +"""fc3i: task_run table + +Revision ID: 0016 +Revises: 0015 +Create Date: 2026-05-24 + +Additive only. New table records every Celery task attempt via signal +handlers (backend.app.celery_signals). Status is plain String(16) not +Postgres ENUM (per feedback_check_existing_enums: ENUM columns hard- +fail at INSERT, String columns extend cleanly). + +Composite indexes anticipate the three dashboard panes: +- (queue, started_at desc) — per-lane recent activity +- (status, started_at desc) — recent failures pane +- (task_name, started_at desc) — drill-down by task + +Indexed columns get individual indexes via `index=True` on the model; +the composites below cover the multi-column lookups. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0016" +down_revision: Union[str, None] = "0015" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "task_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("celery_task_id", sa.String(length=64), nullable=False), + sa.Column("queue", sa.String(length=32), nullable=False), + sa.Column("task_name", sa.String(length=128), nullable=False), + sa.Column("target_id", sa.Integer(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("duration_ms", sa.Integer(), nullable=True), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="running", + ), + sa.Column("error_type", sa.String(length=128), nullable=True), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column("retry_count", sa.Integer(), nullable=True), + sa.Column("worker_hostname", sa.String(length=128), nullable=True), + sa.Column("args_summary", sa.String(length=255), nullable=True), + ) + + # Single-column indexes (matches Mapped[...].index=True on model). + op.create_index("ix_task_run_celery_task_id", "task_run", ["celery_task_id"]) + op.create_index("ix_task_run_queue", "task_run", ["queue"]) + op.create_index("ix_task_run_task_name", "task_run", ["task_name"]) + op.create_index("ix_task_run_started_at", "task_run", ["started_at"]) + op.create_index("ix_task_run_finished_at", "task_run", ["finished_at"]) + op.create_index("ix_task_run_status", "task_run", ["status"]) + + # Composite indexes for dashboard query patterns. + op.create_index( + "ix_task_run_queue_started", + "task_run", ["queue", sa.text("started_at DESC")], + ) + op.create_index( + "ix_task_run_status_started", + "task_run", ["status", sa.text("started_at DESC")], + ) + op.create_index( + "ix_task_run_name_started", + "task_run", ["task_name", sa.text("started_at DESC")], + ) + + +def downgrade() -> None: + op.drop_index("ix_task_run_name_started", table_name="task_run") + op.drop_index("ix_task_run_status_started", table_name="task_run") + op.drop_index("ix_task_run_queue_started", table_name="task_run") + op.drop_index("ix_task_run_status", table_name="task_run") + op.drop_index("ix_task_run_finished_at", table_name="task_run") + op.drop_index("ix_task_run_started_at", table_name="task_run") + op.drop_index("ix_task_run_task_name", table_name="task_run") + op.drop_index("ix_task_run_queue", table_name="task_run") + op.drop_index("ix_task_run_celery_task_id", table_name="task_run") + op.drop_table("task_run") diff --git a/alembic/versions/0017_fc3h_backup_run.py b/alembic/versions/0017_fc3h_backup_run.py new file mode 100644 index 0000000..5b6a839 --- /dev/null +++ b/alembic/versions/0017_fc3h_backup_run.py @@ -0,0 +1,82 @@ +"""fc3h: backup_run table + +Revision ID: 0017 +Revises: 0016 +Create Date: 2026-05-24 + +Additive. New table records every backup/restore attempt with artifact +metadata. Lifecycle tracking lives in task_run from FC-3i; this is +artifact-only (paths, sizes, tag, restore lineage). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0017" +down_revision: Union[str, None] = "0016" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "backup_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("kind", sa.String(length=16), nullable=False), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="pending", + ), + sa.Column("tag", sa.String(length=64), nullable=True), + sa.Column("triggered_by", sa.String(length=32), nullable=False), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("sql_path", sa.Text(), nullable=True), + sa.Column("tar_path", sa.Text(), nullable=True), + sa.Column("size_bytes", sa.BigInteger(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column( + "manifest", sa.JSON(), nullable=False, server_default="{}", + ), + sa.Column( + "restored_from_id", sa.Integer(), + sa.ForeignKey("backup_run.id", ondelete="SET NULL"), + nullable=True, + ), + ) + + # Single-column indexes (matches Mapped[...].index=True). + op.create_index("ix_backup_run_kind", "backup_run", ["kind"]) + op.create_index("ix_backup_run_status", "backup_run", ["status"]) + op.create_index("ix_backup_run_tag", "backup_run", ["tag"]) + op.create_index("ix_backup_run_started_at", "backup_run", ["started_at"]) + op.create_index("ix_backup_run_finished_at", "backup_run", ["finished_at"]) + + # Composite indexes for dashboard query patterns. + op.create_index( + "ix_backup_run_kind_started", + "backup_run", ["kind", sa.text("started_at DESC")], + ) + op.create_index( + "ix_backup_run_status_finished", + "backup_run", ["status", sa.text("finished_at DESC")], + ) + # Partial index: only tagged rows participate in retention-exempt query. + op.create_index( + "ix_backup_run_tag_partial", + "backup_run", ["tag"], + postgresql_where=sa.text("tag IS NOT NULL"), + ) + + +def downgrade() -> None: + op.drop_index("ix_backup_run_tag_partial", table_name="backup_run") + op.drop_index("ix_backup_run_status_finished", table_name="backup_run") + op.drop_index("ix_backup_run_kind_started", table_name="backup_run") + op.drop_index("ix_backup_run_finished_at", table_name="backup_run") + op.drop_index("ix_backup_run_started_at", table_name="backup_run") + op.drop_index("ix_backup_run_tag", table_name="backup_run") + op.drop_index("ix_backup_run_status", table_name="backup_run") + op.drop_index("ix_backup_run_kind", table_name="backup_run") + op.drop_table("backup_run") diff --git a/alembic/versions/0018_fc3h_backup_settings.py b/alembic/versions/0018_fc3h_backup_settings.py new file mode 100644 index 0000000..517c8f1 --- /dev/null +++ b/alembic/versions/0018_fc3h_backup_settings.py @@ -0,0 +1,62 @@ +"""fc3h: backup_* knobs on import_settings + +Revision ID: 0018 +Revises: 0017 +Create Date: 2026-05-24 + +Adds four columns to the singleton import_settings row: + - backup_db_nightly_enabled (default False — opt-in) + - backup_db_nightly_hour_utc (default 3) + - backup_db_keep_last_n (default 14) + - backup_images_keep_last_n (default 3) + +server_default ensures the singleton row is backfilled in place +without an UPDATE statement. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0018" +down_revision: Union[str, None] = "0017" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_settings", + sa.Column( + "backup_db_nightly_enabled", sa.Boolean(), + nullable=False, server_default=sa.false(), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "backup_db_nightly_hour_utc", sa.Integer(), + nullable=False, server_default="3", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "backup_db_keep_last_n", sa.Integer(), + nullable=False, server_default="14", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "backup_images_keep_last_n", sa.Integer(), + nullable=False, server_default="3", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "backup_images_keep_last_n") + op.drop_column("import_settings", "backup_db_keep_last_n") + op.drop_column("import_settings", "backup_db_nightly_hour_utc") + op.drop_column("import_settings", "backup_db_nightly_enabled") diff --git a/alembic/versions/0019_import_batch_refreshed.py b/alembic/versions/0019_import_batch_refreshed.py new file mode 100644 index 0000000..1770daa --- /dev/null +++ b/alembic/versions/0019_import_batch_refreshed.py @@ -0,0 +1,38 @@ +"""import_batch.refreshed counter for deep-scan sidecar re-application + +Revision ID: 0019 +Revises: 0018 +Create Date: 2026-05-25 + +Adds a `refreshed` counter to `import_batch`, mirroring the existing +`imported`/`skipped`/`failed`/`attachments` columns. Deep scan now +re-applies sidecar metadata to already-imported files (the IR feature +that didn't make the FC port the first time); a "refreshed" outcome +increments this counter so the UI can surface "X new, Y refreshed" +instead of the misleading "Scan complete — no new files" message. + +server_default=0 backfills existing rows in place — no UPDATE needed. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0019" +down_revision: Union[str, None] = "0018" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_batch", + sa.Column( + "refreshed", sa.Integer(), + nullable=False, server_default=sa.text("0"), + ), + ) + + +def downgrade() -> None: + op.drop_column("import_batch", "refreshed") diff --git a/alembic/versions/0020_library_audit_run.py b/alembic/versions/0020_library_audit_run.py new file mode 100644 index 0000000..07a8b87 --- /dev/null +++ b/alembic/versions/0020_library_audit_run.py @@ -0,0 +1,65 @@ +"""fc-cleanup: library_audit_run table for async transparency/single_color audits + +Revision ID: 0020 +Revises: 0019 +Create Date: 2026-05-26 + +The table backs the async audit lifecycle: rule + params snapshot, status +state machine ('running' → 'ready' → 'applied'/'cancelled'/'error'), and +the matched_ids JSONB array that the apply step deletes. Capped at 50k IDs +per row by the scan task (oversize = rule too aggressive, operator narrows +before re-running). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0020" +down_revision: Union[str, None] = "0019" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "library_audit_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("rule", sa.String(32), nullable=False), + sa.Column("params", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column( + "status", sa.String(16), + nullable=False, server_default="running", + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), + nullable=False, server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "scanned_count", sa.Integer(), + nullable=False, server_default="0", + ), + sa.Column( + "matched_count", sa.Integer(), + nullable=False, server_default="0", + ), + sa.Column( + "matched_ids", postgresql.JSONB(astext_type=sa.Text()), + nullable=False, server_default=sa.text("'[]'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + ) + op.create_index( + "ix_library_audit_run_rule", "library_audit_run", ["rule"], + ) + op.create_index( + "ix_library_audit_run_status", "library_audit_run", ["status"], + ) + + +def downgrade() -> None: + op.drop_index("ix_library_audit_run_status", table_name="library_audit_run") + op.drop_index("ix_library_audit_run_rule", table_name="library_audit_run") + op.drop_table("library_audit_run") diff --git a/alembic/versions/0021_image_provenance_unique.py b/alembic/versions/0021_image_provenance_unique.py new file mode 100644 index 0000000..b9be941 --- /dev/null +++ b/alembic/versions/0021_image_provenance_unique.py @@ -0,0 +1,54 @@ +"""provenance-race: dedupe + UNIQUE(image_record_id, post_id) on image_provenance + +Revision ID: 0021 +Revises: 0020 +Create Date: 2026-05-26 + +Closes the race in Importer._apply_sidecar's existence-check + INSERT pattern. +Two workers writing for the same (image, post) pair both saw no existing row +and both inserted, leaving duplicates that then broke .scalar_one_or_none() +on every subsequent deep-scan rederive against those images +(MultipleResultsFound). Most plausibly seeded when the 5-min recovery sweep +re-enqueued a still-running long-import task and the second worker collided +with the first inside _apply_sidecar. + +Migration steps: + 1. DELETE all but min(id) per (image_record_id, post_id) pair. Operator's + DB had 2 affected pairs at write-time; harmless no-op if zero. + 2. Add UNIQUE constraint so the importer's new savepoint+IntegrityError + recovery path can trip on collision and re-select, mirroring + uq_source_artist_platform_url and uq_post_source_external_id. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0021" +down_revision: Union[str, None] = "0020" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + """ + DELETE FROM image_provenance ip1 + USING image_provenance ip2 + WHERE ip1.image_record_id = ip2.image_record_id + AND ip1.post_id = ip2.post_id + AND ip1.id > ip2.id + """ + ) + op.create_unique_constraint( + "uq_image_provenance_image_post", + "image_provenance", + ["image_record_id", "post_id"], + ) + + +def downgrade() -> None: + op.drop_constraint( + "uq_image_provenance_image_post", + "image_provenance", + type_="unique", + ) diff --git a/alembic/versions/0022_source_per_artist_platform.py b/alembic/versions/0022_source_per_artist_platform.py new file mode 100644 index 0000000..d3e75dd --- /dev/null +++ b/alembic/versions/0022_source_per_artist_platform.py @@ -0,0 +1,223 @@ +"""source-collapse: one Source per (artist, platform) — consolidate junk per-post Sources + +Revision ID: 0022 +Revises: 0021 +Create Date: 2026-05-26 + +Closes the operator-flagged 2026-05-26 issue where the filesystem importer +called _find_or_create_source(url=sd.post_url), creating one Source row per +imported post URL. Operator's Atole artist had 406 Source rows where there +should have been 1 (the /cw/Atole subscription Source). + +Source represents a subscription feed (one per artist+platform — the +gallery-dl URL polled by the FC-3 downloader). Posts hang off it. The +filesystem importer was misusing Source as a per-post key. + +Migration steps per (artist_id, platform) group with >1 Source: + 1. Pick canonical — prefer a URL NOT matching '/posts/$' (real + campaign URL like /cw/Atole); else min(id). + 2. PRE-merge any Posts under non-canonical sources whose + external_post_id ALREADY exists under the canonical source. (Same + gallery-dl post imported via two different sidecar paths can plant + two Post rows with identical external_post_id under different + Sources for the same artist.) Repoint ImageProvenance + + ImageRecord.primary_post_id to the canonical-side Post, dedupe + ImageProvenance against alembic 0021's uq, then delete the + non-canonical-side Post. This MUST happen before step 3 — Postgres + fires uq_post_source_external_id row-by-row during the bulk UPDATE + and the merge-after-reparent ordering 500s on first collision + (operator-hit during v26.05.26.1 deploy, 2026-05-26). + 3. Reparent remaining Posts onto canonical (no collisions possible now). + 4. Reparent ImageProvenance.source_id off the non-canonical sources. + 5. Delete the orphan Source rows. + 6. If the canonical Source's URL still looks like a per-post URL (no + campaign URL existed among candidates), rewrite it to + 'sidecar::' so the artist detail page shows + something readable. +""" +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0022" +down_revision: Union[str, None] = "0021" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_POST_URL_RE = r"/posts/[^/]+$" + + +def upgrade() -> None: + conn = op.get_bind() + + # Find (artist_id, platform) groups with > 1 Source row. + groups = conn.execute(text(""" + SELECT artist_id, platform + FROM source + GROUP BY artist_id, platform + HAVING COUNT(*) > 1 + """)).fetchall() + + for artist_id, platform in groups: + rows = conn.execute( + text(""" + SELECT id, url FROM source + WHERE artist_id = :a AND platform = :p + ORDER BY id ASC + """), + {"a": artist_id, "p": platform}, + ).fetchall() + + # Canonical: first row whose URL doesn't look like a per-post URL; + # else min(id). + canonical_id = None + for sid, url in rows: + if not _matches_post_url(url): + canonical_id = sid + break + if canonical_id is None: + canonical_id = rows[0][0] + + other_ids = [sid for sid, _ in rows if sid != canonical_id] + if not other_ids: + continue + + # STEP 2: PRE-merge ALL Posts with duplicate external_post_id + # across the entire (canonical + others) group, BEFORE the bulk + # reparent. Two cases must both be handled: + # (A) canonical has Post X with epid=N; an "other" source has + # Post Y with epid=N → after bulk UPDATE, (canonical, N) + # collides with itself. + # (B) two different "other" sources each have a Post with + # epid=N; canonical has none → after bulk UPDATE, both + # are repointed to (canonical, N) and the second collides. + # The earlier version of this migration only handled (A); the + # operator's deploy 2026-05-26 tripped (B) at line 139. + # Fix: group ALL Posts in the (artist, platform) by epid; for + # any group with count>1, pick the keep (prefer one already + # under canonical; else lowest id) and merge the rest into it. + all_posts = conn.execute( + text(""" + SELECT external_post_id, id, source_id + FROM post + WHERE source_id = :canonical OR source_id = ANY(:others) + ORDER BY external_post_id, id + """), + {"canonical": canonical_id, "others": other_ids}, + ).fetchall() + by_epid: dict = {} + for epid, post_id, src_id in all_posts: + by_epid.setdefault(epid, []).append((post_id, src_id)) + for _epid, posts in by_epid.items(): + if len(posts) <= 1: + continue + # Prefer a Post already under canonical as the keep. + canonical_posts = [p for p in posts if p[1] == canonical_id] + if canonical_posts: + keep_id = canonical_posts[0][0] + else: + keep_id = posts[0][0] # already sorted by id ASC + drop_ids = [p[0] for p in posts if p[0] != keep_id] + for drop_id in drop_ids: + # Pre-delete image_provenance rows under drop_ whose + # image_record_id ALREADY has a provenance under keep — + # the UPDATE below would otherwise repoint them and + # trip uq_image_provenance_image_post (alembic 0021) + # row-by-row before any after-the-fact dedupe could + # run. Operator's v26.05.26.3 deploy 2026-05-26 tripped + # this at line 123. + conn.execute( + text(""" + DELETE FROM image_provenance + WHERE post_id = :drop_ + AND image_record_id IN ( + SELECT image_record_id FROM image_provenance + WHERE post_id = :keep + ) + """), + {"keep": keep_id, "drop_": drop_id}, + ) + # Now safe to repoint the survivors. + conn.execute( + text(""" + UPDATE image_provenance SET post_id = :keep + WHERE post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_record SET primary_post_id = :keep + WHERE primary_post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text("DELETE FROM post WHERE id = :drop_"), + {"drop_": drop_id}, + ) + + # STEP 3: Bulk reparent the remaining Posts off the other + # Sources. After step 2, no collisions on + # (canonical, external_post_id) are possible. + conn.execute( + text(""" + UPDATE post SET source_id = :canonical + WHERE source_id = ANY(:others) + """), + {"canonical": canonical_id, "others": other_ids}, + ) + + # STEP 4: Reparent ImageProvenance.source_id (denormalized FK). + # No UNIQUE on source_id; safe bulk update. + conn.execute( + text(""" + UPDATE image_provenance SET source_id = :canonical + WHERE source_id = ANY(:others) + """), + {"canonical": canonical_id, "others": other_ids}, + ) + + # STEP 5: Drop the orphan Sources. + conn.execute( + text("DELETE FROM source WHERE id = ANY(:others)"), + {"others": other_ids}, + ) + + # If the canonical's URL still looks per-post (no campaign URL + # existed among the candidates), rewrite to a synthetic anchor so + # the artist detail page renders something readable. + canonical_url = conn.execute( + text("SELECT url FROM source WHERE id = :id"), + {"id": canonical_id}, + ).scalar_one() + if _matches_post_url(canonical_url): + slug = conn.execute( + text("SELECT slug FROM artist WHERE id = :id"), + {"id": artist_id}, + ).scalar_one() + conn.execute( + text(""" + UPDATE source + SET url = :new_url, enabled = false + WHERE id = :id + """), + { + "id": canonical_id, + "new_url": f"sidecar:{platform}:{slug}", + }, + ) + + +def downgrade() -> None: + # Lossy migration — orphan Sources deleted, Posts reparented, Posts + # merged. No safe downgrade. If you need to roll back the schema + # invariant, fork from 0021 and re-run filesystem imports. + pass + + +def _matches_post_url(url: str) -> bool: + """True if url ends with /posts/ (gallery-dl-style per-post URL).""" + import re + return bool(re.search(_POST_URL_RE, url or "")) diff --git a/alembic/versions/0023_drop_meta_rating_tag_kinds.py b/alembic/versions/0023_drop_meta_rating_tag_kinds.py new file mode 100644 index 0000000..fc65ea1 --- /dev/null +++ b/alembic/versions/0023_drop_meta_rating_tag_kinds.py @@ -0,0 +1,99 @@ +"""drop meta + rating tag kinds — operator-retired 2026-05-26 + +Revision ID: 0023 +Revises: 0022 +Create Date: 2026-05-26 + +Operator decided meta + rating aren't valid tag kinds for FC. Per-row +behavior: DELETE existing rows (operator chose "clean break" over +"convert to general"). All cascading FKs (image_tag, tag_alias, +tag_allowlist, tag_reference_embedding, tag_suggestion_rejection, +series_page) use ondelete="CASCADE" so a single DELETE on tag cleans +the related rows in one go. + +After the data cleanup, recreate the tag_kind ENUM without 'meta' / +'rating' (Postgres has no `ALTER TYPE ... DROP VALUE`; standard +rename-create-cast-drop dance). The server default 'general' is +dropped before the type swap and restored after. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0023" +down_revision: Union[str, None] = "0022" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # 1. Delete tags of the retired kinds. CASCADE handles related tables. + op.execute("DELETE FROM tag WHERE kind IN ('meta', 'rating')") + + # 2. Drop the CHECK constraint that references the enum's literal + # values. Postgres can't resolve `kind = 'character'` across the + # type swap below — the literal would bind to the new tag_kind + # but the column is on tag_kind_old, producing + # "operator does not exist: tag_kind = tag_kind_old". + # (Operator-hit during the v26.05.26.5 deploy attempt; ck was + # originally added by alembic 0002.) Recreated post-swap. + op.drop_constraint( + "ck_tag_fandom_requires_character", "tag", type_="check" + ) + + # 3. Drop the server default — ALTER COLUMN TYPE can't carry it + # across the type swap below. + op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT") + + # 4. Recreate the tag_kind enum without meta/rating. + op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old") + op.execute( + "CREATE TYPE tag_kind AS ENUM (" + "'artist', 'character', 'fandom', 'general', " + "'series', 'archive', 'post'" + ")" + ) + op.execute( + "ALTER TABLE tag " + "ALTER COLUMN kind TYPE tag_kind " + "USING kind::text::tag_kind" + ) + op.execute("DROP TYPE tag_kind_old") + + # 5. Restore the server default. + op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'") + + # 6. Restore the CHECK constraint (now bound to the new tag_kind). + op.create_check_constraint( + "ck_tag_fandom_requires_character", + "tag", + "(fandom_id IS NULL) OR (kind = 'character')", + ) + + +def downgrade() -> None: + # Add the values back to the enum so old code can boot. The deleted + # tag rows are gone permanently — no safe restore. + op.drop_constraint( + "ck_tag_fandom_requires_character", "tag", type_="check" + ) + op.execute("ALTER TABLE tag ALTER COLUMN kind DROP DEFAULT") + op.execute("ALTER TYPE tag_kind RENAME TO tag_kind_old") + op.execute( + "CREATE TYPE tag_kind AS ENUM (" + "'artist', 'character', 'fandom', 'general', " + "'series', 'archive', 'post', 'meta', 'rating'" + ")" + ) + op.execute( + "ALTER TABLE tag " + "ALTER COLUMN kind TYPE tag_kind " + "USING kind::text::tag_kind" + ) + op.execute("DROP TYPE tag_kind_old") + op.execute("ALTER TABLE tag ALTER COLUMN kind SET DEFAULT 'general'") + op.create_check_constraint( + "ck_tag_fandom_requires_character", + "tag", + "(fandom_id IS NULL) OR (kind = 'character')", + ) diff --git a/alembic/versions/0024_backfill_post_title_from_description.py b/alembic/versions/0024_backfill_post_title_from_description.py new file mode 100644 index 0000000..2b1385c --- /dev/null +++ b/alembic/versions/0024_backfill_post_title_from_description.py @@ -0,0 +1,80 @@ +"""backfill post.post_title from description first-line — 2026-05-27 + +Revision ID: 0024 +Revises: 0023 +Create Date: 2026-05-27 + +SubscribeStar gallery-dl always writes `title: ""` and embeds the leading +sentence inside `content` HTML. FC's sidecar parser was leaving +post_title NULL for every SubscribeStar post since FC-3 shipped. The +parser fix (sidecar._first_line_text fallback) now synthesizes a title +at parse time; this migration applies the same logic retroactively to +existing rows. + +Operator-flagged 2026-05-27 after inspecting +/mnt/Data/Patreon/Cheunart/subscribestar/ sidecars. + +Idempotent: only touches rows where post_title IS NULL or empty AND +description IS NOT NULL. Re-running the migration is a no-op. +""" +from __future__ import annotations + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0024" +down_revision: Union[str, None] = "0023" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_TAG_RE = re.compile(r"<[^>]+>") +_WS_RE = re.compile(r"\s+") + + +def _first_line_text(body: str, limit: int = 120) -> str | None: + """Mirror of sidecar._first_line_text. Kept inline so the migration + doesn't carry a runtime import dependency from app code that may + have moved by the time the migration is replayed years from now.""" + if not body: + return None + text_ = _TAG_RE.sub(" ", body) + text_ = text_.replace("\xa0", " ") + for line in text_.splitlines(): + line = _WS_RE.sub(" ", line).strip() + if line: + if len(line) > limit: + return line[: limit - 1].rstrip() + "…" + return line + return None + + +def upgrade() -> None: + bind = op.get_bind() + rows = bind.execute( + text( + "SELECT id, description FROM post " + "WHERE (post_title IS NULL OR post_title = '') " + "AND description IS NOT NULL AND description <> ''" + ) + ).fetchall() + updated = 0 + for row in rows: + derived = _first_line_text(row.description) + if not derived: + continue + bind.execute( + text("UPDATE post SET post_title = :t WHERE id = :id"), + {"t": derived, "id": row.id}, + ) + updated += 1 + print(f"0024: backfilled post_title on {updated} row(s)") + + +def downgrade() -> None: + # No safe restore — we can't tell which post_titles were derived vs + # genuinely present. Leave the column alone on rollback. + pass diff --git a/alembic/versions/0025_fix_subscribestar_post_ids.py b/alembic/versions/0025_fix_subscribestar_post_ids.py new file mode 100644 index 0000000..b44f430 --- /dev/null +++ b/alembic/versions/0025_fix_subscribestar_post_ids.py @@ -0,0 +1,288 @@ +"""sidecar-audit followup: correct external_post_id + post_url across all platforms + +Revision ID: 0025 +Revises: 0024 +Create Date: 2026-05-27 + +Closes the operator-flagged 2026-05-27 sidecar audit findings. Three +data-correctness bugs across non-Patreon platforms had been silently +corrupting Posts since FC-3 shipped; the parser fix (sidecar.py, same +commit) addresses new imports. This migration cleans up existing rows. + +Per-platform actions: + + subscribestar — gallery-dl wrote the per-attachment id in `id` and + the actual post id in `post_id`. FC's parser picked `id`, so every + multi-image SubscribeStar post was fragmented into N Post rows. + 1. For each SubscribeStar Post, read its sidecar (via the related + ImageRecord's on-disk path), pull `post_id`, overwrite + external_post_id and post_url. + 2. Merge groups of Posts under one source that now share an + external_post_id (fragments of the same actual post). Same + ImageProvenance pre-delete + repoint dance as alembic 0022. + + hentaifoundry — sidecars have NO `url` field; `src` is the image + URL. FC's parser stored post_url=NULL. Read each HF Post's sidecar + for `user` + `index`, derive the canonical /pictures/user// + permalink. external_post_id (= `index`) was already correct. + + discord — gallery-dl wrote the CDN attachment URL in `url`. FC's + parser stored that as post_url. Read each Discord Post's sidecar + for the server/channel/message triple, derive the proper + discord.com/channels/.../ permalink. external_post_id (= + `message_id`) was already correct. + + pixiv — pure-SQL backfill: replace any `i.pximg.net`-style URL on + Post.post_url with the derived `/artworks/` permalink. Pixiv + external_post_id (= `id`) was already correct; no sidecar IO + needed. + +Idempotent: re-running on already-corrected data is a no-op (skips +rows whose derived value matches what's already stored). + +Posts whose related ImageRecord paths don't resolve on disk (orphaned +filesystem state) are skipped with a count in the migration output — +those will be picked up by a future deep-scan. +""" +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0025" +down_revision: Union[str, None] = "0024" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Mirror of sidecar._NUMBERING_PREFIX. Kept inline so the migration is +# self-contained (the operator's banked rule: +# reference_postgres_enum_swap_drop_checks.md says migrations shouldn't +# import from runtime app code). +_NUMBERING_PREFIX = re.compile(r"^\d+_(.+)$") + + +def _find_sidecar(media_path: Path) -> Path | None: + """gallery-dl writes the sidecar under the unprefixed stem + (`HOLLOW-ICHIGO.json`) while the media file gets a NN_ ordering + prefix (`01_HOLLOW-ICHIGO.png`). Try in order: + 1. .json next to the media + 2. .json next to the media (full-name variant) + 3. strip the NN_ prefix from the stem, then .json + """ + if not media_path: + return None + cand = media_path.with_suffix(".json") + if cand.is_file(): + return cand + cand = media_path.parent / f"{media_path.name}.json" + if cand.is_file(): + return cand + m = _NUMBERING_PREFIX.match(media_path.stem) + if m: + cand = media_path.parent / f"{m.group(1)}.json" + if cand.is_file(): + return cand + return None + + +def _str_id(v) -> str | None: + """str() a JSON scalar id; reject bool (JSON booleans are ints in + Python's eyes but they aren't valid sidecar ids).""" + if isinstance(v, bool): + return None + if isinstance(v, (str, int)) and str(v).strip(): + return str(v).strip() + return None + + +def _str_field(v) -> str | None: + if isinstance(v, str) and v.strip(): + return v.strip() + return None + + +def upgrade() -> None: + conn = op.get_bind() + + # ── PART 1: Per-platform corrections requiring filesystem IO ───── + # SubscribeStar, HentaiFoundry, Discord all need fields from the + # sidecar to construct the right post_url. We walk each Post's + # related ImageRecord.path to find the sidecar, read it, derive, + # and update. + targets = conn.execute(text(""" + SELECT p.id, p.external_post_id, p.post_url, s.platform + FROM post p + JOIN source s ON s.id = p.source_id + WHERE s.platform IN ('subscribestar', 'hentaifoundry', 'discord') + """)).fetchall() + + stats: dict[str, dict[str, int]] = { + plat: {"read": 0, "updated": 0, "no_sidecar": 0} + for plat in ("subscribestar", "hentaifoundry", "discord") + } + for post_row in targets: + plat = post_row.platform + path = _first_attachment_path(conn, post_row.id) + if not path: + stats[plat]["no_sidecar"] += 1 + continue + sidecar = _find_sidecar(Path(path)) + if sidecar is None: + stats[plat]["no_sidecar"] += 1 + continue + try: + data = json.loads(sidecar.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + stats[plat]["no_sidecar"] += 1 + continue + stats[plat]["read"] += 1 + + new_epid = post_row.external_post_id + new_url = None + if plat == "subscribestar": + pid = _str_id(data.get("post_id")) + if pid: + new_epid = pid + new_url = f"https://www.subscribestar.com/posts/{pid}" + elif plat == "hentaifoundry": + user = _str_field(data.get("user")) or _str_field(data.get("artist")) + idx = _str_id(data.get("index")) + if user and idx: + new_url = f"https://www.hentai-foundry.com/pictures/user/{user}/{idx}" + elif plat == "discord": + sid = _str_id(data.get("server_id")) + cid = _str_id(data.get("channel_id")) + mid = _str_id(data.get("message_id")) + if sid and cid and mid: + new_url = f"https://discord.com/channels/{sid}/{cid}/{mid}" + + # Idempotent: skip if nothing changed. + if new_epid == post_row.external_post_id and new_url == post_row.post_url: + continue + conn.execute( + text(""" + UPDATE post + SET external_post_id = :epid, post_url = :url + WHERE id = :id + """), + {"epid": new_epid, "url": new_url, "id": post_row.id}, + ) + stats[plat]["updated"] += 1 + + for plat, s in stats.items(): + print( + f"0025: {plat} — read {s['read']} sidecars, " + f"updated {s['updated']} Posts, " + f"{s['no_sidecar']} Posts had no resolvable sidecar" + ) + + # ── PART 2: Merge SubscribeStar fragments now sharing epid ─────── + # After Part 1, each group of Posts under one source with the SAME + # new external_post_id is a fragment-set of the same actual post. + # Merge to one canonical row. Pre-handle the same ImageProvenance + # collision pattern as alembic 0022 (uq_image_provenance_image_post). + fragment_groups = conn.execute(text(""" + SELECT p.source_id, p.external_post_id, + ARRAY_AGG(p.id ORDER BY p.id ASC) AS post_ids + FROM post p + JOIN source s ON s.id = p.source_id + WHERE s.platform = 'subscribestar' + AND p.external_post_id IS NOT NULL + GROUP BY p.source_id, p.external_post_id + HAVING COUNT(*) > 1 + """)).fetchall() + + merged = 0 + for grp in fragment_groups: + post_ids = list(grp.post_ids) + keep_id, *drop_ids = post_ids + for drop_id in drop_ids: + # Pre-DELETE colliding ImageProvenance under drop_ that + # already exist under keep (alembic 0022 banked the pattern). + conn.execute( + text(""" + DELETE FROM image_provenance + WHERE post_id = :drop_ + AND image_record_id IN ( + SELECT image_record_id FROM image_provenance + WHERE post_id = :keep + ) + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_provenance SET post_id = :keep + WHERE post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_record SET primary_post_id = :keep + WHERE primary_post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE post_attachment SET post_id = :keep + WHERE post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text("DELETE FROM post WHERE id = :drop_"), + {"drop_": drop_id}, + ) + merged += 1 + print(f"0025: subscribestar — merged {merged} duplicate Post fragments") + + # ── PART 3: Pixiv post_url backfill (pure SQL) ─────────────────── + # Pixiv's external_post_id is already correct (gallery-dl's `id` is + # the post id). Only post_url needs derivation: replace anything + # under i.pximg.net (the file URL) with the /artworks/ permalink. + pixiv_updated = conn.execute(text(""" + UPDATE post p + SET post_url = 'https://www.pixiv.net/artworks/' || p.external_post_id + FROM source s + WHERE p.source_id = s.id + AND s.platform = 'pixiv' + AND p.external_post_id IS NOT NULL + AND (p.post_url IS NULL + OR p.post_url LIKE 'https://i.pximg.net/%' + OR p.post_url LIKE 'http://i.pximg.net/%') + """)).rowcount + print(f"0025: pixiv — backfilled post_url on {pixiv_updated} Posts") + + +def _first_attachment_path(conn, post_id: int) -> str | None: + """Return any ImageRecord.path attached to this post (via + ImageProvenance). Lowest-id row keeps the migration deterministic + so re-running on the same DB picks the same sidecar.""" + row = conn.execute( + text(""" + SELECT ir.path + FROM image_provenance ip + JOIN image_record ir ON ir.id = ip.image_record_id + WHERE ip.post_id = :pid + ORDER BY ip.id ASC + LIMIT 1 + """), + {"pid": post_id}, + ).first() + return row[0] if row else None + + +def downgrade() -> None: + # Lossy: external_post_id values were overwritten with the correct + # post_id; original per-attachment ids weren't preserved. Post-merge + # also deleted drop rows. No safe restore. To roll back the schema + # invariant, fork from 0024 and re-run sidecar imports. + pass diff --git a/alembic/versions/0026_import_task_recovery_count_refetched.py b/alembic/versions/0026_import_task_recovery_count_refetched.py new file mode 100644 index 0000000..ccbc3da --- /dev/null +++ b/alembic/versions/0026_import_task_recovery_count_refetched.py @@ -0,0 +1,53 @@ +"""import_task.recovery_count + refetched — poison-pill circuit breaker + +Revision ID: 0026 +Revises: 0025 +Create Date: 2026-05-28 + +Backs the import-task resilience work (operator-flagged 2026-05-28): + +- recovery_count: how many times recover_interrupted_tasks has + re-queued this row from a stuck 'processing' state. A row that + hard-crashes the worker (OOM / segfault on a corrupt or oversized + input) leaves no terminal flip, so the sweep re-queues it — and + without a cap it would loop forever, re-crashing the worker each + time. After MAX_RECOVERY_ATTEMPTS the sweep marks it 'failed' with a + diagnostic instead. + +- refetched: whether a one-shot re-download has already been attempted + for this task's file. Bounds the Layer-2 re-fetch remediation to a + single attempt so source-side corruption doesn't loop. + +Both default to 0 / false; additive, no backfill needed. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0026" +down_revision: Union[str, None] = "0025" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_task", + sa.Column( + "recovery_count", sa.Integer(), nullable=False, + server_default="0", + ), + ) + op.add_column( + "import_task", + sa.Column( + "refetched", sa.Boolean(), nullable=False, + server_default=sa.false(), + ), + ) + + +def downgrade() -> None: + op.drop_column("import_task", "refetched") + op.drop_column("import_task", "recovery_count") diff --git a/alembic/versions/0027_drop_migration_run.py b/alembic/versions/0027_drop_migration_run.py new file mode 100644 index 0000000..454481b --- /dev/null +++ b/alembic/versions/0027_drop_migration_run.py @@ -0,0 +1,50 @@ +"""drop migration_run — one-and-done GS/IR migration tooling removed + +Revision ID: 0027 +Revises: 0026 +Create Date: 2026-05-29 + +The GS/IR migration tooling (services/migrators, /api/migrate, the +run_migration task, LegacyMigrationCard, and the MigrationRun model) was +removed after the migration cutover completed. This drops its now-orphaned +run-log table. Downgrade recreates the table (mirrors the old model) so the +migration is reversible. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB + +revision: str = "0027" +down_revision: Union[str, None] = "0026" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_table("migration_run") + + +def downgrade() -> None: + op.create_table( + "migration_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("kind", sa.String(length=32), nullable=False), + sa.Column("status", sa.String(length=32), nullable=False), + sa.Column("dry_run", sa.Boolean(), nullable=False, server_default=sa.false()), + sa.Column( + "started_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "counts", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + sa.Column("error", sa.Text(), nullable=True), + sa.Column( + "metadata", JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb"), + ), + ) + op.create_index("ix_migration_run_kind", "migration_run", ["kind"]) + op.create_index("ix_migration_run_status", "migration_run", ["status"]) diff --git a/alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py b/alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py new file mode 100644 index 0000000..5ec9267 --- /dev/null +++ b/alembic/versions/0028_collapse_sidecar_synthetics_into_real_sources.py @@ -0,0 +1,190 @@ +"""collapse-sidecar-synthetic: repoint Posts/ImageProvenance/DownloadEvents +from `sidecar::` synthetic Source anchors onto the real +Source for the same (artist, platform) when one exists, then delete the +synthetic. + +Revision ID: 0028 +Revises: 0027 +Create Date: 2026-05-31 + +Background: alembic 0022 (2026-05-26) consolidated the old per-post-URL +Source rows into one canonical Source per (artist, platform). When NO +real campaign URL was salvageable among the candidates, it rewrote the +canonical row to url='sidecar::' enabled=false as a +disabled anchor for any Posts already attached. + +That was fine while it was the only Source for that artist+platform. +But: the unique constraint on Source is (artist_id, platform, url), not +(artist_id, platform). When the operator later added the real +subscription via the UI / extension / etc., a SECOND row landed — +the real one — with id > the synthetic. Both coexisted. + +Two follow-on problems surfaced 2026-05-31: + + 1. The Subscriptions UI listed both rows. The synthetic was disabled + so the scheduler never polled it, but it looked like a phantom + subscription. (Fixed in same commit by SourceService.list filter.) + 2. importer._source_for_sidecar picked Source by `ORDER BY id ASC + LIMIT 1`, so EVERY gallery-dl download since the real Source was + added attached its Post to the SYNTHETIC anchor, not the real + Source. (Fixed in same commit by preferring non-sidecar URLs.) + +This migration is the data half of the cleanup: for every (artist, +platform) with both a synthetic AND a real Source, repoint the +synthetic's children (Posts, ImageProvenance, DownloadEvents) onto the +real Source and delete the synthetic. Reuses the same epid/provenance +collision dance from alembic 0022 because the same uniqueness +constraints fire row-by-row during bulk UPDATEs. + +Lone synthetic anchors — those where no real Source for the same +(artist, platform) exists (e.g., filesystem-imported artist with no +subscription added) — are LEFT INTACT. They anchor real imported +content; deleting them would CASCADE-delete the Posts the operator +imported. The SourceService.list filter hides them from the UI; the +operator can delete them by hand if they want the underlying imports +gone. +""" +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0028" +down_revision: Union[str, None] = "0027" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + conn = op.get_bind() + + # Find (artist_id, platform) groups where BOTH a sidecar synthetic + # and at least one real Source exist. + groups = conn.execute(text(""" + SELECT artist_id, platform + FROM source + GROUP BY artist_id, platform + HAVING bool_or(url LIKE 'sidecar:%') + AND bool_or(url NOT LIKE 'sidecar:%') + """)).fetchall() + + for artist_id, platform in groups: + rows = conn.execute( + text(""" + SELECT id, url FROM source + WHERE artist_id = :a AND platform = :p + ORDER BY id ASC + """), + {"a": artist_id, "p": platform}, + ).fetchall() + + synthetic_ids = [sid for sid, url in rows if url.startswith("sidecar:")] + real_rows = [(sid, url) for sid, url in rows if not url.startswith("sidecar:")] + if not synthetic_ids or not real_rows: + continue # belt+suspenders; the GROUP BY already filtered + + # Canonical real: lowest-id non-sidecar Source. + canonical_id = real_rows[0][0] + + # STEP A: PRE-merge Post collisions on (canonical, external_post_id). + # Mirror alembic 0022's pre-merge logic — when synth has Post X + # epid=N and real has Post Y epid=N, the bulk UPDATE below would + # trip uq_post_source_external_id row-by-row. Group all Posts + # under (canonical + synthetics) by epid; for any group >1, + # pick a keep (prefer one already under canonical, else lowest + # id) and merge the rest into it. + all_posts = conn.execute( + text(""" + SELECT external_post_id, id, source_id + FROM post + WHERE source_id = :canonical OR source_id = ANY(:synths) + ORDER BY external_post_id, id + """), + {"canonical": canonical_id, "synths": synthetic_ids}, + ).fetchall() + by_epid: dict = {} + for epid, post_id, src_id in all_posts: + by_epid.setdefault(epid, []).append((post_id, src_id)) + for _epid, posts in by_epid.items(): + if len(posts) <= 1: + continue + canonical_side = [p for p in posts if p[1] == canonical_id] + keep_id = canonical_side[0][0] if canonical_side else posts[0][0] + drop_ids = [p[0] for p in posts if p[0] != keep_id] + for drop_id in drop_ids: + # Pre-delete image_provenance rows under drop_ whose + # image_record_id already has provenance under keep — + # avoids tripping uq_image_provenance_image_post (0021) + # row-by-row during the repoint UPDATE. + conn.execute( + text(""" + DELETE FROM image_provenance + WHERE post_id = :drop_ + AND image_record_id IN ( + SELECT image_record_id FROM image_provenance + WHERE post_id = :keep + ) + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_provenance SET post_id = :keep + WHERE post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text(""" + UPDATE image_record SET primary_post_id = :keep + WHERE primary_post_id = :drop_ + """), + {"keep": keep_id, "drop_": drop_id}, + ) + conn.execute( + text("DELETE FROM post WHERE id = :drop_"), + {"drop_": drop_id}, + ) + + # STEP B: Bulk reparent the remaining Posts off the synthetics. + conn.execute( + text(""" + UPDATE post SET source_id = :canonical + WHERE source_id = ANY(:synths) + """), + {"canonical": canonical_id, "synths": synthetic_ids}, + ) + + # STEP C: Reparent ImageProvenance.source_id (denormalized FK; + # no UNIQUE on source_id, safe bulk). + conn.execute( + text(""" + UPDATE image_provenance SET source_id = :canonical + WHERE source_id = ANY(:synths) + """), + {"canonical": canonical_id, "synths": synthetic_ids}, + ) + + # STEP D: Reparent any DownloadEvent.source_id. Synthetics are + # enabled=false so the scheduler never created events for them; + # this is belt+suspenders for any rows planted by manual force + # or older code paths. + conn.execute( + text(""" + UPDATE download_event SET source_id = :canonical + WHERE source_id = ANY(:synths) + """), + {"canonical": canonical_id, "synths": synthetic_ids}, + ) + + # STEP E: Drop the now-empty synthetics. + conn.execute( + text("DELETE FROM source WHERE id = ANY(:synths)"), + {"synths": synthetic_ids}, + ) + + +def downgrade() -> None: + # Lossy migration — synthetic Sources deleted, Posts repointed and + # potentially merged. No safe downgrade. + pass diff --git a/alembic/versions/0029_drop_artist_copyright_ml_thresholds.py b/alembic/versions/0029_drop_artist_copyright_ml_thresholds.py new file mode 100644 index 0000000..e6c044b --- /dev/null +++ b/alembic/versions/0029_drop_artist_copyright_ml_thresholds.py @@ -0,0 +1,71 @@ +"""drop artist + copyright ml thresholds; lower general default to 0.50 + +Revision ID: 0029 +Revises: 0028 +Create Date: 2026-06-01 + +Operator-flagged 2026-06-01: the view modal's Suggestions panel hides +most general-category predictions because the default threshold is +0.95. Lowering the default to 0.50 (matches character) so general +suggestions surface more aggressively; the value remains tunable in +Settings → ML. + +Same change retires two ML suggestion categories whose Tag.kind +surfaces are unused: + +- `artist`: retired in FC-2d-vii-c — artist identity is acquisition- + derived (image_record.artist_id), never ML-inferred. The threshold + column was a leftover from before that retirement. +- `copyright`: retired 2026-06-01 — the app uses `fandom` for the + franchise/copyright concept (per TagsView.vue's doc comment); no + Tag rows of kind=copyright exist, and the threshold column never + fed anything user-visible. + +Both columns are dropped from ml_settings; the existing row's +suggestion_threshold_general value is bumped from 0.95 to 0.50 iff +it's still at the old default, so deployed installs pick up the new +UX without overriding any operator tuning. +""" +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import text + +revision: str = "0029" +down_revision: Union[str, None] = "0028" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Bump the general threshold for installs still at the old default. + op.execute(text( + "UPDATE ml_settings " + "SET suggestion_threshold_general = 0.50 " + "WHERE id = 1 AND suggestion_threshold_general = 0.95" + )) + op.drop_column("ml_settings", "suggestion_threshold_artist") + op.drop_column("ml_settings", "suggestion_threshold_copyright") + + +def downgrade() -> None: + # Restore the columns with their prior defaults. The bump from + # 0.95 → 0.50 isn't reversible without remembering whether the + # operator had explicitly set 0.95 (unlikely — that was just the + # default) so we leave the current general value as-is. + from sqlalchemy import Column, Float + + op.add_column( + "ml_settings", + Column( + "suggestion_threshold_artist", + Float, nullable=False, server_default="0.30", + ), + ) + op.add_column( + "ml_settings", + Column( + "suggestion_threshold_copyright", + Float, nullable=False, server_default="0.50", + ), + ) diff --git a/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py b/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py new file mode 100644 index 0000000..c37e499 --- /dev/null +++ b/alembic/versions/0030_nullable_post_source_id_denorm_artist_id.py @@ -0,0 +1,145 @@ +"""nullable post.source_id + denormalized post.artist_id; retire sidecar synthetics + +Revision ID: 0030 +Revises: 0029 +Create Date: 2026-06-01 + +Operator-asked 2026-06-01 after the Dymkens orphan investigation: the +sidecar synthetic Source pattern (`sidecar::` rows +with enabled=false) was technically correct but misled the operator +into thinking they had phantom subscriptions. The synthetics existed +solely to satisfy `Post.source_id NOT NULL` for filesystem-imported +content with no real subscription. + +This migration makes the data model honest: + +1. **Post gets a denormalized `artist_id` column** so artist filters + work without traversing `Post → Source.artist_id`. Backfilled from + the existing Source linkage, then NOT NULL'd. +2. **`Post.source_id` becomes nullable**, FK ondelete `CASCADE` → `SET + NULL`. Deleting a Source detaches its Posts instead of destroying + imported content (semantically: subscription ends, archive stays). +3. **`ImageProvenance.source_id` becomes nullable** with the same FK + semantic change. +4. **Sidecar synthetic Sources are deleted** — first NULL out the + FKs from Post + ImageProvenance pointing at them (so the implicit + CASCADE doesn't fire), then delete. DownloadEvent FK is unchanged + (still CASCADE'd, NOT NULL'd) — synthetics have `enabled=false` + so no events exist for them. + +Uniqueness handling: the existing `uq_post_source_external_id` +(source_id, external_post_id) keeps working for source-bound Posts +(Postgres treats NULL != NULL so NULL-source rows aren't deduped by +it). A second partial unique index covers the NULL-source case on +(artist_id, external_post_id) so filesystem-imported posts still +dedupe within an artist. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import text + +revision: str = "0030" +down_revision: Union[str, None] = "0029" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + conn = op.get_bind() + + # Step 1: add Post.artist_id, initially nullable for backfill. + # FK naming follows the Base.metadata naming_convention + # (fk_
__) — alembic 0001 set this up. + op.add_column( + "post", + sa.Column("artist_id", sa.Integer, nullable=True), + ) + op.create_foreign_key( + "fk_post_artist_id_artist", "post", "artist", + ["artist_id"], ["id"], ondelete="CASCADE", + ) + + # Step 2: backfill from Source.artist_id (every existing Post has a + # Source today, so every row gets populated). + conn.execute(text(""" + UPDATE post p + SET artist_id = s.artist_id + FROM source s + WHERE p.source_id = s.id AND p.artist_id IS NULL + """)) + + # Sanity: count any remaining NULLs. Should be zero pre-this-migration. + remaining = conn.execute(text( + "SELECT COUNT(*) FROM post WHERE artist_id IS NULL" + )).scalar_one() + if remaining: + raise RuntimeError( + f"alembic 0030: {remaining} post rows have no resolvable " + f"artist_id after backfill. Investigate before continuing." + ) + + # Step 3: enforce NOT NULL + add index for artist-filter queries. + op.alter_column("post", "artist_id", nullable=False) + op.create_index("ix_post_artist_id", "post", ["artist_id"]) + + # Step 4: relax post.source_id + flip FK to SET NULL. The original FK + # name from alembic 0001 is `fk_post_source_id_source` per the + # NAMING_CONVENTION in models/base.py. + op.alter_column("post", "source_id", nullable=True) + op.drop_constraint("fk_post_source_id_source", "post", type_="foreignkey") + op.create_foreign_key( + "fk_post_source_id_source", "post", "source", + ["source_id"], ["id"], ondelete="SET NULL", + ) + + # Step 5: relax image_provenance.source_id + flip FK to SET NULL. + op.alter_column("image_provenance", "source_id", nullable=True) + op.drop_constraint( + "fk_image_provenance_source_id_source", "image_provenance", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_image_provenance_source_id_source", "image_provenance", "source", + ["source_id"], ["id"], ondelete="SET NULL", + ) + + # Step 6: partial unique index on (artist_id, external_post_id) for + # NULL-source Posts. The existing uq_post_source_external_id keeps + # guarding source-bound rows; NULL-source rows now dedupe within + # an artist. + op.execute( + "CREATE UNIQUE INDEX uq_post_artist_external_id_null_source " + "ON post (artist_id, external_post_id) " + "WHERE source_id IS NULL" + ) + + # Step 7: retire sidecar synthetic Sources. NULL out the references + # FIRST (the new FK is SET NULL so CASCADE wouldn't fire anyway, but + # being explicit makes the intent clear). Then delete the synthetic + # source rows. Any DownloadEvent rows under synthetics CASCADE-die + # with the source — synthetics have enabled=false so there shouldn't + # be any in practice. + conn.execute(text(""" + UPDATE post + SET source_id = NULL + WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%') + """)) + conn.execute(text(""" + UPDATE image_provenance + SET source_id = NULL + WHERE source_id IN (SELECT id FROM source WHERE url LIKE 'sidecar:%') + """)) + deleted = conn.execute(text( + "DELETE FROM source WHERE url LIKE 'sidecar:%' RETURNING id" + )).rowcount + print(f"alembic 0030: deleted {deleted} sidecar synthetic source rows") + + +def downgrade() -> None: + # Lossy migration — the deleted sidecar synthetics can't be + # restored from the orphan post.source_id / image_provenance.source_id + # values, and the partial unique index encodes a constraint that + # NULL-source Posts may now exist. No safe downgrade. + pass diff --git a/alembic/versions/0031_source_backfill_runs_remaining.py b/alembic/versions/0031_source_backfill_runs_remaining.py new file mode 100644 index 0000000..ed140fc --- /dev/null +++ b/alembic/versions/0031_source_backfill_runs_remaining.py @@ -0,0 +1,45 @@ +"""source.backfill_runs_remaining: sticky deep-scan mode + +Revision ID: 0031 +Revises: 0030 +Create Date: 2026-06-01 + +Tick vs backfill mode for subscription downloads. When +`backfill_runs_remaining > 0`, the next N download runs use +`skip: True` + 30-min timeout (walk full history). When 0, runs use +`skip: "exit:20"` + 14.5-min timeout (catch-up mode, exits early once +20 contiguous archived items are seen). + +Operator-flagged 2026-06-01 (Knuxy run #38887): a creator with ~550 +archived posts saturates the 870s catch-up timeout even when there is +no new content, because gallery-dl's default `skip: True` keeps walking. +Tick mode short-circuits that; backfill mode is the explicit opt-in for +deep history scans. + +Default 0 (all existing subscriptions start in tick mode). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0031" +down_revision: Union[str, None] = "0030" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "source", + sa.Column( + "backfill_runs_remaining", + sa.Integer, + nullable=False, + server_default="0", + ), + ) + + +def downgrade() -> None: + op.drop_column("source", "backfill_runs_remaining") diff --git a/alembic/versions/0032_source_error_type.py b/alembic/versions/0032_source_error_type.py new file mode 100644 index 0000000..e264b35 --- /dev/null +++ b/alembic/versions/0032_source_error_type.py @@ -0,0 +1,41 @@ +"""source.error_type: surface ErrorType taxonomy in FailingSourcesCard + +Revision ID: 0032 +Revises: 0031 +Create Date: 2026-06-02 + +Audit 2026-06-02: the backend computes 13 ErrorType categories (auth_error, +rate_limited, not_found, access_denied, validation_failed, etc.) and +stamps each one on DownloadEvent.metadata, but the Source row only carried +the free-text last_error. Operators couldn't bulk-triage failing sources +("all auth_error → rotate cookies, all rate_limited → just wait") without +opening Logs per row. + +This column receives the last error_type from _update_source_health +and gets cleared on a successful run. Nullable + indexed so the failing- +sources rollup can filter/group cheaply. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0032" +down_revision: Union[str, None] = "0031" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "source", + sa.Column("error_type", sa.String(length=32), nullable=True), + ) + op.create_index( + "ix_source_error_type", "source", ["error_type"], + ) + + +def downgrade() -> None: + op.drop_index("ix_source_error_type", table_name="source") + op.drop_column("source", "error_type") diff --git a/alembic/versions/0033_suggestion_threshold_default_070.py b/alembic/versions/0033_suggestion_threshold_default_070.py new file mode 100644 index 0000000..652cf44 --- /dev/null +++ b/alembic/versions/0033_suggestion_threshold_default_070.py @@ -0,0 +1,48 @@ +"""suggestion_threshold default 0.50 → 0.70 + +Revision ID: 0033 +Revises: 0032 +Create Date: 2026-06-02 + +Operator-flagged 2026-06-02 — the 0.50 default (set on 2026-06-01) is +too noisy in practice; raise to 0.70 for both suggestion categories. + +Only conditionally updates singletons whose current value is still the +2026-06-01 default (0.50). Operators who deliberately tuned their row +to some other value (0.55, 0.65, 0.80, etc. via the Settings UI) keep +their pick — the migration only catches the unchanged-default case. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0033" +down_revision: Union[str, None] = "0032" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + "UPDATE ml_settings " + "SET suggestion_threshold_character = 0.70 " + "WHERE id = 1 AND suggestion_threshold_character = 0.50" + ) + op.execute( + "UPDATE ml_settings " + "SET suggestion_threshold_general = 0.70 " + "WHERE id = 1 AND suggestion_threshold_general = 0.50" + ) + + +def downgrade() -> None: + op.execute( + "UPDATE ml_settings " + "SET suggestion_threshold_character = 0.50 " + "WHERE id = 1 AND suggestion_threshold_character = 0.70" + ) + op.execute( + "UPDATE ml_settings " + "SET suggestion_threshold_general = 0.50 " + "WHERE id = 1 AND suggestion_threshold_general = 0.70" + ) diff --git a/alembic/versions/0034_artist_visit.py b/alembic/versions/0034_artist_visit.py new file mode 100644 index 0000000..a2234a6 --- /dev/null +++ b/alembic/versions/0034_artist_visit.py @@ -0,0 +1,53 @@ +"""artist_visit: per-artist last-viewed timestamp for the "+N new" badge + +Revision ID: 0034 +Revises: 0033 +Create Date: 2026-06-03 + +Powers the artists-directory "+N new since last visit" badge + ArtistView +banner. Single row per artist (no user_id yet — rule #47 multi-user ACL +is aspirational; widens to (user_id, artist_id) PK when User lands). + +Seed every existing artist with `last_viewed_at = NOW()` so the badge +starts at 0 across the board — no noisy "you have 5000 unseen images" +on first deploy. New artists auto-get a row via +`ArtistService.find_or_create`. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0034" +down_revision: Union[str, None] = "0033" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "artist_visit", + sa.Column( + "artist_id", + sa.Integer, + sa.ForeignKey("artist.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "last_viewed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + ) + # Seed: every existing artist starts "fully caught up". Without this, + # every operator with N artists would see N badges (worth of every + # image ever imported) on first deploy. + op.execute( + "INSERT INTO artist_visit (artist_id, last_viewed_at) " + "SELECT id, NOW() FROM artist" + ) + + +def downgrade() -> None: + op.drop_table("artist_visit") diff --git a/alembic/versions/0035_image_record_effective_date.py b/alembic/versions/0035_image_record_effective_date.py new file mode 100644 index 0000000..586cf51 --- /dev/null +++ b/alembic/versions/0035_image_record_effective_date.py @@ -0,0 +1,70 @@ +"""image_record.effective_date: materialized gallery sort key + index + +Revision ID: 0035 +Revises: 0034 +Create Date: 2026-06-04 + +The gallery ordered/cursored on COALESCE(post.post_date, +image_record.created_at) across the Post outer join. That expression spans +two tables, so no index can serve it — every /scroll sorted a large slice +of the library, and the frontend fired ten of them serially per initial +load. Materialize the value into image_record.effective_date and index +(effective_date DESC, id DESC) so the cursor scroll is an index range scan. + +Backfill = COALESCE(primary post's post_date, created_at) so existing rows +keep their exact ordering. New rows get the created_at-equivalent server +default; services/importer.py overrides it with the post's date when a +primary post with a date is linked. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0035" +down_revision: Union[str, None] = "0034" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add nullable first so the backfill can populate before NOT NULL. + op.add_column( + "image_record", + sa.Column("effective_date", sa.DateTime(timezone=True), nullable=True), + ) + # Pure set-based UPDATEs (no per-row params) — immune to the 65535 + # bind-parameter ceiling regardless of library size. + op.execute( + """ + UPDATE image_record AS ir + SET effective_date = COALESCE(p.post_date, ir.created_at) + FROM post AS p + WHERE ir.primary_post_id = p.id + """ + ) + op.execute( + """ + UPDATE image_record + SET effective_date = created_at + WHERE effective_date IS NULL + """ + ) + op.alter_column( + "image_record", + "effective_date", + nullable=False, + server_default=sa.text("now()"), + ) + # DESC/DESC matches the gallery's ORDER BY effective_date DESC, id DESC + # so the scroll is a forward index scan; raw SQL because alembic's + # column list doesn't express per-column DESC cleanly. + op.execute( + "CREATE INDEX ix_image_record_effective_date " + "ON image_record (effective_date DESC, id DESC)" + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_effective_date", table_name="image_record") + op.drop_column("image_record", "effective_date") diff --git a/alembic/versions/0036_siglip_embedding_hnsw_index.py b/alembic/versions/0036_siglip_embedding_hnsw_index.py new file mode 100644 index 0000000..a8c1251 --- /dev/null +++ b/alembic/versions/0036_siglip_embedding_hnsw_index.py @@ -0,0 +1,41 @@ +"""image_record.siglip_embedding: HNSW cosine index for "more like this" + +Revision ID: 0036 +Revises: 0035 +Create Date: 2026-06-04 + +Gallery Phase 3 (visual similarity search) ranks images by +`siglip_embedding.cosine_distance(source_embedding)`. Without an index that's +a sequential scan computing a 1152-dim distance for every row — fine at small +scale, but it grows linearly with the library. Add an HNSW index with +`vector_cosine_ops` so the top-N nearest search is sub-50ms ANN. + +1152 dims is under pgvector's 2000-dim HNSW limit, so HNSW (no training, +better recall than IVFFlat) is the right choice. ONE-TIME COST: building the +index over the existing embeddings (~57k vectors on the operator's library) +locks image_record for ~30-60s during this migration on deploy — acceptable +for a single-operator homelab. NULL embeddings (videos / not-yet-embedded +rows) are simply not indexed. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0036" +down_revision: Union[str, None] = "0035" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Raw SQL: alembic's create_index doesn't express the `USING hnsw (... + # vector_cosine_ops)` access-method + opclass cleanly. Must match the + # query's cosine_distance operator class to be usable by the planner. + op.execute( + "CREATE INDEX ix_image_record_siglip_hnsw " + "ON image_record USING hnsw (siglip_embedding vector_cosine_ops)" + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_siglip_hnsw", table_name="image_record") diff --git a/alembic/versions/0037_patreon_seen_media.py b/alembic/versions/0037_patreon_seen_media.py new file mode 100644 index 0000000..255484e --- /dev/null +++ b/alembic/versions/0037_patreon_seen_media.py @@ -0,0 +1,53 @@ +"""patreon_seen_media: per-source ledger of already-ingested Patreon media + +Revision ID: 0037 +Revises: 0036 +Create Date: 2026-06-05 + +Native Patreon ingester (build step 2a). Replaces gallery-dl's +archive.sqlite3 with our own queryable table. The downloader upserts one +row per (source, media) so routine walks skip media we've already +processed; a future "recovery" mode bypasses the ledger to re-walk. + +`filehash` is a 32-hex Patreon CDN MD5, OR a video sentinel of the form +``video::`` — hence String(128). The unique +constraint on (source_id, filehash) is the dedup upsert key. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0037" +down_revision: Union[str, None] = "0036" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "patreon_seen_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("post_id", sa.String(64), nullable=True), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_patreon_seen_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("patreon_seen_media") diff --git a/alembic/versions/0038_patreon_failed_media.py b/alembic/versions/0038_patreon_failed_media.py new file mode 100644 index 0000000..e907ae1 --- /dev/null +++ b/alembic/versions/0038_patreon_failed_media.py @@ -0,0 +1,58 @@ +"""patreon_failed_media: per-source dead-letter ledger for failing Patreon media + +Revision ID: 0038 +Revises: 0037 +Create Date: 2026-06-06 + +Plan #705 (#7). Media that keeps failing to download/validate (404'd CDN, +deleted post, geo-blocked Mux, persistently-corrupt bytes) gets recorded here +with an attempt counter; once it crosses the dead-letter threshold the ingester +skips it on routine walks (recovery still re-attempts). A clean download clears +the row. UNIQUE (source_id, filehash) is the upsert key (same media key the +seen-ledger uses). +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0038" +down_revision: Union[str, None] = "0037" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "patreon_failed_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), + sa.Column("last_error", sa.Text, nullable=True), + sa.Column( + "first_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.Column( + "last_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_patreon_failed_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("patreon_failed_media") diff --git a/alembic/versions/0039_library_audit_resume.py b/alembic/versions/0039_library_audit_resume.py new file mode 100644 index 0000000..6cfb8f9 --- /dev/null +++ b/alembic/versions/0039_library_audit_resume.py @@ -0,0 +1,40 @@ +"""library_audit_run: resume cursor + progress timestamp for chunked scans + +Revision ID: 0039 +Revises: 0038 +Create Date: 2026-06-07 + +scan_library_for_rule used to run one 2h pass that timed out on large libraries +and monopolized the concurrency-1 maintenance queue (operator-flagged). It now +runs short time-boxed chunks that re-enqueue: `resume_after_id` persists the +keyset cursor so the next chunk continues where it left off, and +`last_progress_at` lets the recovery sweep tell a progressing multi-chunk audit +from a genuinely stuck one. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0039" +down_revision: Union[str, None] = "0038" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "library_audit_run", + sa.Column( + "resume_after_id", sa.Integer, nullable=False, server_default="0" + ), + ) + op.add_column( + "library_audit_run", + sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("library_audit_run", "last_progress_at") + op.drop_column("library_audit_run", "resume_after_id") diff --git a/alembic/versions/0040_series_chapters.py b/alembic/versions/0040_series_chapters.py new file mode 100644 index 0000000..a0808df --- /dev/null +++ b/alembic/versions/0040_series_chapters.py @@ -0,0 +1,108 @@ +"""series chapters: chapter layer over series_page (FC-6.1) + +Revision ID: 0040 +Revises: 0039 +Create Date: 2026-06-07 + +A series (Tag kind='series') gains an ordered chapter layer. Reading order +becomes (series_chapter.chapter_number, series_page.page_number). Every existing +series is backfilled into a single auto-chapter (chapter_number=1) holding its +current flat pages, so no data is lost and the old flat ordering is preserved. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0040" +down_revision: Union[str, None] = "0039" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_chapter", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "series_tag_id", + sa.Integer, + sa.ForeignKey("tag.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("chapter_number", sa.Integer, nullable=False), + sa.Column("title", sa.Text, nullable=True), + sa.Column( + "is_placeholder", sa.Boolean, nullable=False, server_default="false" + ), + sa.Column("stated_page_start", sa.Integer, nullable=True), + sa.Column("stated_page_end", sa.Integer, nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + ) + op.create_index( + "ix_series_chapter_series_tag_id", "series_chapter", ["series_tag_id"] + ) + + # New columns on series_page; chapter_id starts nullable so we can backfill. + op.add_column( + "series_page", sa.Column("chapter_id", sa.Integer, nullable=True) + ) + op.add_column( + "series_page", sa.Column("stated_page", sa.Integer, nullable=True) + ) + + conn = op.get_bind() + # One auto-chapter per existing series (any series_tag_id present in pages). + conn.execute( + sa.text( + "INSERT INTO series_chapter " + "(series_tag_id, chapter_number, is_placeholder, created_at, updated_at) " + "SELECT DISTINCT series_tag_id, 1, false, now(), now() " + "FROM series_page" + ) + ) + # Point every existing page at its series' auto-chapter. + conn.execute( + sa.text( + "UPDATE series_page sp " + "SET chapter_id = sc.id " + "FROM series_chapter sc " + "WHERE sc.series_tag_id = sp.series_tag_id" + ) + ) + + # Now lock chapter_id down: NOT NULL + FK (cascade) + index. + op.alter_column("series_page", "chapter_id", nullable=False) + op.create_foreign_key( + "fk_series_page_chapter_id", + "series_page", + "series_chapter", + ["chapter_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_index( + "ix_series_page_chapter_id", "series_page", ["chapter_id"] + ) + + +def downgrade() -> None: + op.drop_index("ix_series_page_chapter_id", table_name="series_page") + op.drop_constraint( + "fk_series_page_chapter_id", "series_page", type_="foreignkey" + ) + op.drop_column("series_page", "stated_page") + op.drop_column("series_page", "chapter_id") + op.drop_index("ix_series_chapter_series_tag_id", table_name="series_chapter") + op.drop_table("series_chapter") diff --git a/alembic/versions/0041_series_suggestions.py b/alembic/versions/0041_series_suggestions.py new file mode 100644 index 0000000..51b690a --- /dev/null +++ b/alembic/versions/0041_series_suggestions.py @@ -0,0 +1,98 @@ +"""series suggestions: assisted-continuation matcher (FC-6.3) + +Revision ID: 0041 +Revises: 0040 +Create Date: 2026-06-07 + +A confirm-only queue of "this post may continue this series" hints, plus two +import_settings knobs (enable + score threshold) for the matcher. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0041" +down_revision: Union[str, None] = "0040" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "series_suggestion", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "post_id", + sa.Integer, + sa.ForeignKey("post.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "series_tag_id", + sa.Integer, + sa.ForeignKey("tag.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("score", sa.Float, nullable=False), + sa.Column("signals", sa.JSON, nullable=True), + sa.Column( + "status", sa.String(16), nullable=False, server_default="pending" + ), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("now()"), + ), + sa.UniqueConstraint( + "post_id", "series_tag_id", name="uq_series_suggestion_post_series" + ), + ) + op.create_index( + "ix_series_suggestion_post_id", "series_suggestion", ["post_id"] + ) + op.create_index( + "ix_series_suggestion_series_tag_id", + "series_suggestion", + ["series_tag_id"], + ) + op.create_index( + "ix_series_suggestion_status", "series_suggestion", ["status"] + ) + + op.add_column( + "import_settings", + sa.Column( + "series_suggest_enabled", + sa.Boolean, + nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "series_suggest_threshold", + sa.Float, + nullable=False, + server_default="0.5", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "series_suggest_threshold") + op.drop_column("import_settings", "series_suggest_enabled") + op.drop_index("ix_series_suggestion_status", table_name="series_suggestion") + op.drop_index( + "ix_series_suggestion_series_tag_id", table_name="series_suggestion" + ) + op.drop_index("ix_series_suggestion_post_id", table_name="series_suggestion") + op.drop_table("series_suggestion") diff --git a/alembic/versions/0042_series_chapter_stated_part.py b/alembic/versions/0042_series_chapter_stated_part.py new file mode 100644 index 0000000..f898e56 --- /dev/null +++ b/alembic/versions/0042_series_chapter_stated_part.py @@ -0,0 +1,32 @@ +"""series chapter stated_part: operator-facing Part N label (FC-6.4) + +Revision ID: 0042 +Revises: 0041 +Create Date: 2026-06-07 + +A chapter's positional chapter_number is auto-managed (rewritten 1..N on +reorder/delete), so it can't double as the installment number the operator wants +to type (e.g. a series authored from a post that is Part 2). Add a nullable +stated_part alongside it — the same split as series_page.page_number (order) vs +series_page.stated_page (printed number). Nullable; the UI falls back to +chapter_number when unset. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0042" +down_revision: Union[str, None] = "0041" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "series_chapter", sa.Column("stated_part", sa.Integer, nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("series_chapter", "stated_part") diff --git a/alembic/versions/0043_post_attachment_per_post_unique.py b/alembic/versions/0043_post_attachment_per_post_unique.py new file mode 100644 index 0000000..e8e38ce --- /dev/null +++ b/alembic/versions/0043_post_attachment_per_post_unique.py @@ -0,0 +1,62 @@ +"""post_attachment: per-post sha uniqueness (empty-post flood fix) + +Revision ID: 0043 +Revises: 0042 +Create Date: 2026-06-08 + +PostAttachment.sha256 was GLOBALLY unique, so a non-art file the creator attaches +to many posts (a standard pdf/zip/link-card) only ever got ONE row — on the first +post — leaving every later post a bare shell (no image, no attachment). The native +Patreon backfill of Anduo surfaced 1589 such shells (operator-flagged 2026-06-08). + +Switch to PER-POST uniqueness: the on-disk blob stays sha-deduped, but each post +gets its own row. Replace the unique sha256 index with a plain lookup index plus +two partial uniques — (post_id, sha256) for real posts and (sha256) for the +NULL-post filesystem case (still one row per file there). + +Existing data has ≤1 row per sha (the old global unique), so the new partial +uniques can't be violated on upgrade — no data backfill needed here. The bare-post +shells themselves are removed by the separate prune-empty-posts cleanup tool. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0043" +down_revision: Union[str, None] = "0042" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Drop the global unique index; recreate it as a plain (non-unique) lookup + # index so sha-based reads keep their index (matches the model's index=True). + op.drop_index("ix_post_attachment_sha256", table_name="post_attachment") + op.create_index( + "ix_post_attachment_sha256", "post_attachment", ["sha256"], + ) + op.create_index( + "uq_post_attachment_post_sha", "post_attachment", + ["post_id", "sha256"], unique=True, + postgresql_where=sa.text("post_id IS NOT NULL"), + ) + op.create_index( + "uq_post_attachment_null_post_sha", "post_attachment", + ["sha256"], unique=True, + postgresql_where=sa.text("post_id IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index( + "uq_post_attachment_null_post_sha", table_name="post_attachment" + ) + op.drop_index( + "uq_post_attachment_post_sha", table_name="post_attachment" + ) + op.drop_index("ix_post_attachment_sha256", table_name="post_attachment") + op.create_index( + "ix_post_attachment_sha256", "post_attachment", ["sha256"], + unique=True, + ) diff --git a/alembic/versions/0044_ml_settings_tagger_store_floor.py b/alembic/versions/0044_ml_settings_tagger_store_floor.py new file mode 100644 index 0000000..e019e36 --- /dev/null +++ b/alembic/versions/0044_ml_settings_tagger_store_floor.py @@ -0,0 +1,37 @@ +"""ml_settings.tagger_store_floor + +The ingest confidence floor below which tagger predictions are not stored, +promoted from the TAGGER_STORE_FLOOR env var to a DB-backed, UI-tunable +setting. Default 0.70 (was an env default of 0.05): the suggestion path +already filters at 0.70 and the centroid/learned path covers low-confidence +preferred tags, so the sub-0.70 tail was redundant weight — it had grown +image_record's TOAST to ~100 GB. See plan-task #764. + +Revision ID: 0044 +Revises: 0043 +Create Date: 2026-06-10 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0044" +down_revision: Union[str, None] = "0043" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "tagger_store_floor", sa.Float(), + nullable=False, server_default="0.7", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "tagger_store_floor") diff --git a/alembic/versions/0045_image_prediction_table.py b/alembic/versions/0045_image_prediction_table.py new file mode 100644 index 0000000..df11de2 --- /dev/null +++ b/alembic/versions/0045_image_prediction_table.py @@ -0,0 +1,69 @@ +"""image_prediction table (DDL only — backfill runs as a background task) + +Normalizes the per-image tagger predictions out of the JSON blob into a +queryable table (#768). This migration creates ONLY the table + indexes — it +is pure DDL and commits instantly, so web boots immediately. + +The data backfill from the existing image_record.tagger_predictions JSON is +deliberately NOT done here. Doing it inline made the whole migration one +transaction over the ~100 GB TOAST: nothing committed until the very end, it +was invisible/unmonitorable mid-run, and an early MATERIALIZED-CTE form spilled +the full 100 GB to temp. Instead the backfill is the +backend.app.tasks.admin.backfill_image_predictions_task — batched by id window, +committed per chunk (visible progress + resumable), idempotent +(ON CONFLICT DO NOTHING). Trigger it from Settings → Maintenance once web is up. + +The old image_record.tagger_predictions column is left in place (vestigial) and +dropped in a follow-up once the backfill + code cutover are verified — dropping +it needs an ACCESS EXCLUSIVE lock on the hot image_record table (the 0044 lock +class), so it's deferred to a quiesced-worker window. + +Revision ID: 0045 +Revises: 0044 +Create Date: 2026-06-10 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0045" +down_revision: Union[str, None] = "0044" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "image_prediction", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "image_record_id", sa.Integer(), + sa.ForeignKey("image_record.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("raw_name", sa.String(length=255), nullable=False), + sa.Column("category", sa.String(length=64), nullable=False), + sa.Column("score", sa.Float(), nullable=False), + sa.UniqueConstraint( + "image_record_id", "raw_name", name="image_raw_name", + ), + ) + op.create_index( + "ix_image_prediction_image", "image_prediction", ["image_record_id"], + ) + op.create_index( + "ix_image_prediction_name_score", "image_prediction", + ["raw_name", "score"], + ) + # No data backfill here — see the module docstring. The one-time copy from + # image_record.tagger_predictions runs as backfill_image_predictions_task + # (batched, resumable, idempotent), kept out of this transaction so web boots + # without waiting on a ~100 GB pass. + + +def downgrade() -> None: + op.drop_index("ix_image_prediction_name_score", "image_prediction") + op.drop_index("ix_image_prediction_image", "image_prediction") + op.drop_table("image_prediction") diff --git a/alembic/versions/0046_drop_tagger_predictions.py b/alembic/versions/0046_drop_tagger_predictions.py new file mode 100644 index 0000000..84e543a --- /dev/null +++ b/alembic/versions/0046_drop_tagger_predictions.py @@ -0,0 +1,43 @@ +"""drop image_record.tagger_predictions (predictions normalized to image_prediction) + +Final step of #768. The per-tag predictions now live in the image_prediction +table (backfilled from the JSON, read by suggestions + allowlist, written by +tag_and_embed). The old JSON column is dead weight — and it's the ~100 GB of +sub-0.70 score tail that bloated image_record's TOAST and broke DB backups +(#739). Dropping it is a fast catalog change; it does NOT reclaim the disk on +its own — run `VACUUM FULL image_record` (or pg_repack) afterward, off-hours, +to return the space to the OS so backups go small. + +DROP COLUMN needs a brief ACCESS EXCLUSIVE lock on image_record; env.py's +lock_timeout guards it, so quiesce the ml-worker if a tagging run is in flight +(see the migration-lock reference). tagger_model_version is kept — it's the +"has this been tagged / is it current?" signal the backfill sweep reads. + +Revision ID: 0046 +Revises: 0045 +Create Date: 2026-06-11 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0046" +down_revision: Union[str, None] = "0045" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_column("image_record", "tagger_predictions") + + +def downgrade() -> None: + # Re-add the column empty. The JSON data is not restored (it lived only in + # this column); a downgrade would re-tag or backfill from image_prediction + # separately if ever needed. + op.add_column( + "image_record", + sa.Column("tagger_predictions", sa.JSON(), nullable=True), + ) diff --git a/alembic/versions/0047_series_chapter_dividers.py b/alembic/versions/0047_series_chapter_dividers.py new file mode 100644 index 0000000..5afd074 --- /dev/null +++ b/alembic/versions/0047_series_chapter_dividers.py @@ -0,0 +1,175 @@ +"""series chapters become cosmetic dividers; pages become one series-global run + +FC-6.x reframe (#789). A series is now ONE flat, series-global ordered run of +pages; chapters stop owning pages and become labeled dividers anchored to the +page that begins them. + +Migration (order matters — series_page.chapter_id cascades, so it must be +dropped BEFORE any chapter row is deleted, or pages would cascade away): + a. Renumber series_page.page_number to a series-global 1..N (ordered by the + OLD (chapter_number, page_number)). + b. Add series_chapter.anchor_page_id and populate it with each chapter's first + page (lowest new page_number). + c. Drop series_page.chapter_id (severs the cascade link). + d. Prune chapters that shouldn't become dividers: empty/placeholder ones (no + anchor) and the redundant unlabeled chapter that would sit at page 1. + e. Reshape series_chapter into the divider: drop chapter_number, + is_placeholder, stated_page_start/end; make anchor_page_id NOT NULL + + UNIQUE + FK→series_page ON DELETE CASCADE. + +Revision ID: 0047 +Revises: 0046 +Create Date: 2026-06-11 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0047" +down_revision: Union[str, None] = "0046" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # a. series-global page numbering, preserving the old reading order. + op.execute( + """ + WITH ordered AS ( + SELECT sp.id, + ROW_NUMBER() OVER ( + PARTITION BY sp.series_tag_id + ORDER BY sc.chapter_number, sp.page_number, sp.id + ) AS rn + FROM series_page sp + JOIN series_chapter sc ON sc.id = sp.chapter_id + ) + UPDATE series_page sp + SET page_number = ordered.rn + FROM ordered + WHERE sp.id = ordered.id + """ + ) + + # b. anchor each existing chapter at its first page (lowest new page_number). + op.add_column( + "series_chapter", + sa.Column("anchor_page_id", sa.Integer(), nullable=True), + ) + op.execute( + """ + WITH firsts AS ( + SELECT DISTINCT ON (sp.chapter_id) + sp.chapter_id, sp.id AS page_id + FROM series_page sp + ORDER BY sp.chapter_id, sp.page_number, sp.id + ) + UPDATE series_chapter sc + SET anchor_page_id = firsts.page_id + FROM firsts + WHERE firsts.chapter_id = sc.id + """ + ) + + # c. sever the ownership link (drops the FK + index with the column) BEFORE + # pruning chapters, so deleting a chapter can't cascade-delete its pages. + op.drop_column("series_page", "chapter_id") + + # d. prune chapters that don't become dividers: placeholders / empty ones + # (no anchor), and the unlabeled chapter that would land redundantly at + # page 1 (the series just starts — no divider needed there). + op.execute( + """ + DELETE FROM series_chapter sc + USING ( + SELECT sc2.id + FROM series_chapter sc2 + LEFT JOIN series_page sp ON sp.id = sc2.anchor_page_id + WHERE sc2.anchor_page_id IS NULL + OR (sp.page_number = 1 + AND sc2.title IS NULL + AND sc2.stated_part IS NULL) + ) gone + WHERE sc.id = gone.id + """ + ) + + # e. reshape into the divider model. + op.drop_column("series_chapter", "chapter_number") + op.drop_column("series_chapter", "is_placeholder") + op.drop_column("series_chapter", "stated_page_start") + op.drop_column("series_chapter", "stated_page_end") + op.alter_column("series_chapter", "anchor_page_id", nullable=False) + op.create_unique_constraint( + "uq_series_chapter_anchor_page", "series_chapter", ["anchor_page_id"] + ) + op.create_foreign_key( + "fk_series_chapter_anchor_page", + "series_chapter", + "series_page", + ["anchor_page_id"], + ["id"], + ondelete="CASCADE", + ) + + +def downgrade() -> None: + # Lossy: dividers can't be reconstructed as owning chapters. Collapse back to + # exactly one chapter per series that owns all its pages in order. + op.add_column( + "series_page", sa.Column("chapter_id", sa.Integer(), nullable=True) + ) + op.drop_constraint( + "fk_series_chapter_anchor_page", "series_chapter", type_="foreignkey" + ) + op.drop_constraint( + "uq_series_chapter_anchor_page", "series_chapter", type_="unique" + ) + op.drop_column("series_chapter", "anchor_page_id") + op.add_column( + "series_chapter", + sa.Column( + "chapter_number", sa.Integer(), nullable=False, server_default="1" + ), + ) + op.add_column( + "series_chapter", + sa.Column( + "is_placeholder", sa.Boolean(), nullable=False, + server_default="false", + ), + ) + op.add_column( + "series_chapter", + sa.Column("stated_page_start", sa.Integer(), nullable=True), + ) + op.add_column( + "series_chapter", + sa.Column("stated_page_end", sa.Integer(), nullable=True), + ) + op.execute("DELETE FROM series_chapter") + op.execute( + """ + INSERT INTO series_chapter (series_tag_id, chapter_number) + SELECT DISTINCT series_tag_id, 1 FROM series_page + """ + ) + op.execute( + """ + UPDATE series_page sp + SET chapter_id = sc.id + FROM series_chapter sc + WHERE sc.series_tag_id = sp.series_tag_id + """ + ) + op.alter_column("series_page", "chapter_id", nullable=False) + op.create_foreign_key( + "fk_series_page_chapter", + "series_page", + "series_chapter", + ["chapter_id"], + ["id"], + ondelete="CASCADE", + ) diff --git a/alembic/versions/0048_series_page_pending_status.py b/alembic/versions/0048_series_page_pending_status.py new file mode 100644 index 0000000..25944a5 --- /dev/null +++ b/alembic/versions/0048_series_page_pending_status.py @@ -0,0 +1,45 @@ +"""series_page pending staging: status + nullable page_number (#789 Phase 2) + +Pages added from a post no longer append straight into the run — they land +'pending' with a NULL page_number, staged grouped by their source post so the +operator can drop junk (text-free alts, bumpers) and place the keepers into the +sequence. A page only gets a series-global page_number once it's 'placed'. + +Revision ID: 0048 +Revises: 0047 +Create Date: 2026-06-11 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0048" +down_revision: Union[str, None] = "0047" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "series_page", + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="placed", + ), + ) + op.alter_column( + "series_page", "page_number", + existing_type=sa.Integer(), nullable=True, + ) + + +def downgrade() -> None: + # Lossy: pending pages are unsorted staging rows with no order — drop them. + op.execute("DELETE FROM series_page WHERE status = 'pending'") + op.alter_column( + "series_page", "page_number", + existing_type=sa.Integer(), nullable=False, + ) + op.drop_column("series_page", "status") diff --git a/alembic/versions/0049_external_link_table.py b/alembic/versions/0049_external_link_table.py new file mode 100644 index 0000000..373c807 --- /dev/null +++ b/alembic/versions/0049_external_link_table.py @@ -0,0 +1,90 @@ +"""external_link table — off-platform file-host links found in post bodies + +Creators host the real files on mega.nz / Google Drive / MediaFire / Dropbox / +Pixeldrain and link them in the post text. This table records each such link +(so nothing is silently dropped), and doubles as the dedup + dead-letter ledger +the download worker (a later slice) walks. `url` keeps the FULL link including +the `#fragment` — mega.nz's decryption key lives there; truncating it makes the +file undownloadable. + +CHECK whitelists for host + status include the full enum up front (incl. the +download-worker statuses) so the worker slice needs no constraint migration. + +Revision ID: 0049 +Revises: 0048 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0049" +down_revision: Union[str, None] = "0048" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "external_link", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "post_id", sa.Integer(), + sa.ForeignKey("post.id", ondelete="CASCADE"), nullable=False, + ), + sa.Column( + "artist_id", sa.Integer(), + sa.ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column("host", sa.String(length=16), nullable=False), + sa.Column("url", sa.Text(), nullable=False), + sa.Column("label", sa.Text(), nullable=True), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="pending", + ), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "attachment_id", sa.Integer(), + sa.ForeignKey("post_attachment.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("duration_seconds", sa.Float(), nullable=True), + sa.CheckConstraint( + "host IN ('mega','gdrive','mediafire','dropbox','pixeldrain')", + name="ck_external_link_host", + ), + sa.CheckConstraint( + "status IN ('pending','downloading','downloaded','failed'," + "'skipped','dead')", + name="ck_external_link_status", + ), + ) + op.create_index( + "ix_external_link_post_id", "external_link", ["post_id"], + ) + op.create_index( + "ix_external_link_artist_id", "external_link", ["artist_id"], + ) + op.create_index( + "ix_external_link_status", "external_link", ["status"], + ) + op.create_index( + "uq_external_link_post_url", "external_link", ["post_id", "url"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("uq_external_link_post_url", table_name="external_link") + op.drop_index("ix_external_link_status", table_name="external_link") + op.drop_index("ix_external_link_artist_id", table_name="external_link") + op.drop_index("ix_external_link_post_id", table_name="external_link") + op.drop_table("external_link") diff --git a/alembic/versions/0050_external_link_host_toggles.py b/alembic/versions/0050_external_link_host_toggles.py new file mode 100644 index 0000000..ac78e75 --- /dev/null +++ b/alembic/versions/0050_external_link_host_toggles.py @@ -0,0 +1,38 @@ +"""import_settings: per-host enable toggles for external file-host downloads + +Operator levers (#830): disable a single host (e.g. mega.nz when it's +rate-limiting/banning) without touching the others. The worker reads these via +getattr and defaults to enabled, so the toggles default TRUE (works out of the +box, rule #26). + +Revision ID: 0050 +Revises: 0049 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0050" +down_revision: Union[str, None] = "0049" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_HOSTS = ("mega", "gdrive", "mediafire", "dropbox", "pixeldrain") + + +def upgrade() -> None: + for host in _HOSTS: + op.add_column( + "import_settings", + sa.Column( + f"extdl_{host}_enabled", sa.Boolean(), nullable=False, + server_default=sa.true(), + ), + ) + + +def downgrade() -> None: + for host in _HOSTS: + op.drop_column("import_settings", f"extdl_{host}_enabled") diff --git a/alembic/versions/0051_image_source_provenance.py b/alembic/versions/0051_image_source_provenance.py new file mode 100644 index 0000000..595077d --- /dev/null +++ b/alembic/versions/0051_image_source_provenance.py @@ -0,0 +1,38 @@ +"""image_record: source_url + source_filehash (inline-image localization) + +#830 Phase 2. To render a post body faithfully we serve LOCAL copies of inline +images instead of hotlinking the public CDN. The join key between a body +`` and the local file is the CDN's 32-hex filehash (the same +identity extract_media dedups by). Persist it (indexed) plus the full source +URL for provenance/debugging. Both NULL for filesystem-imported / pre-existing +rows — those fall back to hotlinking until re-downloaded. + +Revision ID: 0051 +Revises: 0050 +Create Date: 2026-06-14 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0051" +down_revision: Union[str, None] = "0050" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("image_record", sa.Column("source_url", sa.Text(), nullable=True)) + op.add_column( + "image_record", sa.Column("source_filehash", sa.String(length=32), nullable=True) + ) + op.create_index( + "ix_image_record_source_filehash", "image_record", ["source_filehash"] + ) + + +def downgrade() -> None: + op.drop_index("ix_image_record_source_filehash", table_name="image_record") + op.drop_column("image_record", "source_filehash") + op.drop_column("image_record", "source_url") diff --git a/alembic/versions/0052_image_duration_seconds.py b/alembic/versions/0052_image_duration_seconds.py new file mode 100644 index 0000000..ec2a180 --- /dev/null +++ b/alembic/versions/0052_image_duration_seconds.py @@ -0,0 +1,32 @@ +"""image_record: duration_seconds (Tier-1 video near-dup key) + +#871. Videos previously deduped on sha256 only (pHash is images-only), so a +different encode/remux of the same video imported as a distinct record. Persist +the container duration so the importer can treat same-artist videos with matching +duration (+ aspect ratio) as the same content and dedup/supersede like images. +NULL for images and for video rows imported before this column existed (a +backfill re-probes those so they participate in dedup). + +Revision ID: 0052 +Revises: 0051 +Create Date: 2026-06-16 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0052" +down_revision: Union[str, None] = "0051" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "image_record", sa.Column("duration_seconds", sa.Float(), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("image_record", "duration_seconds") diff --git a/alembic/versions/0053_ml_settings_video_tagging.py b/alembic/versions/0053_ml_settings_video_tagging.py new file mode 100644 index 0000000..1f192a4 --- /dev/null +++ b/alembic/versions/0053_ml_settings_video_tagging.py @@ -0,0 +1,49 @@ +"""ml_settings: video tagging knobs (cadence sampling + noise floor) + +#747. Video tag quality/perf: sample frames at a fixed cadence (interval) so a +tag's frame-presence reflects real screen time, cap total frames so long videos +stay bounded, and keep a tag only if it appears in >= min_tag_frames sampled +frames. Operator-tunable via Settings → ML (replaces the VIDEO_ML_FRAMES env var). + +Revision ID: 0053 +Revises: 0052 +Create Date: 2026-06-16 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0053" +down_revision: Union[str, None] = "0052" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "video_frame_interval_seconds", sa.Float(), nullable=False, + server_default="4.0", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "video_max_frames", sa.Integer(), nullable=False, server_default="64", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "video_min_tag_frames", sa.Integer(), nullable=False, + server_default="3", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "video_min_tag_frames") + op.drop_column("ml_settings", "video_max_frames") + op.drop_column("ml_settings", "video_frame_interval_seconds") diff --git a/alembic/versions/0054_subscribestar_ledgers.py b/alembic/versions/0054_subscribestar_ledgers.py new file mode 100644 index 0000000..59972ae --- /dev/null +++ b/alembic/versions/0054_subscribestar_ledgers.py @@ -0,0 +1,82 @@ +"""subscribestar_seen_media + subscribestar_failed_media: per-source ledgers + +Revision ID: 0054 +Revises: 0053 +Create Date: 2026-06-17 + +SubscribeStar native ingester (phase 1 of the gallery-dl → native-core +migration). Mirrors the Patreon ledger tables (0037/0038): a seen-ledger so +routine walks skip already-ingested media (recovery bypasses it) and a +dead-letter ledger so persistently-failing media stops re-burning backfill +chunks. `filehash` is a CDN content hash when present, else a synthesized +``:`` key — hence String(128). UNIQUE (source_id, filehash) +is the upsert key on each. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0054" +down_revision: Union[str, None] = "0053" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "subscribestar_seen_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("post_id", sa.String(64), nullable=True), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_subscribestar_seen_media_source_id" + ), + ) + op.create_table( + "subscribestar_failed_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), + sa.Column("last_error", sa.Text, nullable=True), + sa.Column( + "first_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.Column( + "last_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_subscribestar_failed_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("subscribestar_failed_media") + op.drop_table("subscribestar_seen_media") diff --git a/alembic/versions/0055_image_provenance_from_attachment.py b/alembic/versions/0055_image_provenance_from_attachment.py new file mode 100644 index 0000000..8b2566b --- /dev/null +++ b/alembic/versions/0055_image_provenance_from_attachment.py @@ -0,0 +1,55 @@ +"""image_provenance: from_attachment_id (which archive an image was extracted from) + +Milestone #87. When an image is pulled out of a .zip/.rar, record WHICH archive +PostAttachment it came from, so the provenance UI can show the single archive a +file lives inside instead of every attachment on the post. Nullable FK with +ON DELETE SET NULL — a loose (non-archive) download leaves it NULL, and deleting +the archive attachment forgets the linkage without destroying the (image, post) +provenance edge. Existing rows are NULL until the reextract backfill stamps them. + +Revision ID: 0055 +Revises: 0054 +Create Date: 2026-06-22 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0055" +down_revision: Union[str, None] = "0054" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "image_provenance", + sa.Column("from_attachment_id", sa.Integer(), nullable=True), + ) + op.create_index( + "ix_image_provenance_from_attachment_id", + "image_provenance", + ["from_attachment_id"], + ) + op.create_foreign_key( + "fk_image_provenance_from_attachment", + "image_provenance", + "post_attachment", + ["from_attachment_id"], + ["id"], + ondelete="SET NULL", + ) + + +def downgrade() -> None: + op.drop_constraint( + "fk_image_provenance_from_attachment", + "image_provenance", + type_="foreignkey", + ) + op.drop_index( + "ix_image_provenance_from_attachment_id", + table_name="image_provenance", + ) + op.drop_column("image_provenance", "from_attachment_id") diff --git a/alembic/versions/0056_tag_eval_run.py b/alembic/versions/0056_tag_eval_run.py new file mode 100644 index 0000000..7d8e91f --- /dev/null +++ b/alembic/versions/0056_tag_eval_run.py @@ -0,0 +1,43 @@ +"""tag_eval_run: persisted head-vs-centroid tagging eval runs (#1130) + +Milestone #114 slice 1. A long ml-queue eval whose full report must SURVIVE +navigation, so the run + report live in a row the admin card rehydrates from +(mirrors library_audit_run). running -> ready / error. + +Revision ID: 0056 +Revises: 0055 +Create Date: 2026-06-28 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB + +revision: str = "0056" +down_revision: Union[str, None] = "0055" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "tag_eval_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("params", JSONB(), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False, server_default="running"), + sa.Column( + "started_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("report", JSONB(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"]) + + +def downgrade() -> None: + op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run") + op.drop_table("tag_eval_run") diff --git a/alembic/versions/0057_tag_positive_confirmation.py b/alembic/versions/0057_tag_positive_confirmation.py new file mode 100644 index 0000000..92335c2 --- /dev/null +++ b/alembic/versions/0057_tag_positive_confirmation.py @@ -0,0 +1,40 @@ +"""tag_positive_confirmation: operator-affirmed correct positives (#1130) + +Mirror of tag_suggestion_rejection. "Keep" on a doubted positive records here so +the eval's doubts list stops resurfacing confirmed-correct images every run. + +Revision ID: 0057 +Revises: 0056 +Create Date: 2026-06-28 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0057" +down_revision: Union[str, None] = "0056" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "tag_positive_confirmation", + sa.Column( + "image_record_id", sa.Integer(), + sa.ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True, + ), + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True, + ), + sa.Column( + "confirmed_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + ) + + +def downgrade() -> None: + op.drop_table("tag_positive_confirmation") diff --git a/alembic/versions/0058_tag_head.py b/alembic/versions/0058_tag_head.py new file mode 100644 index 0000000..7ff45f6 --- /dev/null +++ b/alembic/versions/0058_tag_head.py @@ -0,0 +1,95 @@ +"""tag_head + head_training_run: production heads that learn from tags (#114) + +The eval (#1130) proved the frozen-embedding + trained-head spine; this lands its +production form. tag_head stores one logistic-regression head per concept (the +new suggestion source, replacing Camie + centroid); head_training_run tracks the +batch that (re)trains them. Adds two head-training tunables to ml_settings. + +Revision ID: 0058 +Revises: 0057 +Create Date: 2026-06-28 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector +from sqlalchemy.dialects.postgresql import JSONB + +revision: str = "0058" +down_revision: Union[str, None] = "0057" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_HEAD_DIM = 1152 + + +def upgrade() -> None: + op.create_table( + "tag_head", + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, + ), + sa.Column("embedding_version", sa.String(length=128), nullable=False), + sa.Column("weights", Vector(_HEAD_DIM), nullable=False), + sa.Column("bias", sa.Float(), nullable=False), + sa.Column("suggest_threshold", sa.Float(), nullable=False), + sa.Column("auto_apply_threshold", sa.Float(), nullable=True), + sa.Column("n_pos", sa.Integer(), nullable=False), + sa.Column("n_neg", sa.Integer(), nullable=False), + sa.Column("ap", sa.Float(), nullable=False), + sa.Column("precision_cv", sa.Float(), nullable=False), + sa.Column("recall", sa.Float(), nullable=False), + sa.Column( + "trained_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("metrics", JSONB(), nullable=True), + ) + + op.create_table( + "head_training_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("params", JSONB(), nullable=False), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="running", + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("n_trained", sa.Integer(), nullable=True), + sa.Column("n_skipped", sa.Integer(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "ix_head_training_run_status", "head_training_run", ["status"], + ) + + # Head-training tunables on the ml_settings singleton. + op.add_column( + "ml_settings", + sa.Column( + "head_min_positives", sa.Integer(), nullable=False, + server_default="8", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "head_auto_apply_precision", sa.Float(), nullable=False, + server_default="0.97", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "head_auto_apply_precision") + op.drop_column("ml_settings", "head_min_positives") + op.drop_index("ix_head_training_run_status", table_name="head_training_run") + op.drop_table("head_training_run") + op.drop_table("tag_head") diff --git a/alembic/versions/0059_head_auto_apply.py b/alembic/versions/0059_head_auto_apply.py new file mode 100644 index 0000000..d0bb9b8 --- /dev/null +++ b/alembic/versions/0059_head_auto_apply.py @@ -0,0 +1,70 @@ +"""head_auto_apply_run + earned-auto-apply settings (#114) + +A graduated head can apply its tag without a human, gated by a master switch + +a support floor. head_auto_apply_run tracks each sweep / dry-run preview. + +Revision ID: 0059 +Revises: 0058 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects.postgresql import JSONB + +revision: str = "0059" +down_revision: Union[str, None] = "0058" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "head_auto_apply_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "dry_run", sa.Boolean(), nullable=False, server_default=sa.false() + ), + sa.Column("params", JSONB(), nullable=False), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="running", + ), + sa.Column( + "started_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("n_applied", sa.Integer(), nullable=True), + sa.Column("report", JSONB(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("last_progress_at", sa.DateTime(timezone=True), nullable=True), + ) + op.create_index( + "ix_head_auto_apply_run_status", "head_auto_apply_run", ["status"], + ) + + op.add_column( + "ml_settings", + sa.Column( + "head_auto_apply_enabled", sa.Boolean(), nullable=False, + server_default=sa.true(), # opt-out: on by default (operator-asked) + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "head_auto_apply_min_positives", sa.Integer(), nullable=False, + server_default="30", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "head_auto_apply_min_positives") + op.drop_column("ml_settings", "head_auto_apply_enabled") + op.drop_index( + "ix_head_auto_apply_run_status", table_name="head_auto_apply_run" + ) + op.drop_table("head_auto_apply_run") diff --git a/alembic/versions/0060_head_metrics.py b/alembic/versions/0060_head_metrics.py new file mode 100644 index 0000000..e94edb8 --- /dev/null +++ b/alembic/versions/0060_head_metrics.py @@ -0,0 +1,74 @@ +"""head_metric + head_metrics_snapshot: auto-apply observability (#114) + +Running misfire/under-fire counters per concept (captured at correction time, +since image_tag.source is lost on delete) + a daily per-concept time-series so +the operator can tune the precision target + support floor from real data. + +Revision ID: 0060 +Revises: 0059 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0060" +down_revision: Union[str, None] = "0059" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "head_metric", + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, + ), + sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"), + sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"), + sa.Column( + "updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + ) + + op.create_table( + "head_metrics_snapshot", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), + ), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column( + "snapshot_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("n_auto_applied", sa.Integer(), nullable=False, server_default="0"), + sa.Column("n_misfires", sa.Integer(), nullable=False, server_default="0"), + sa.Column("n_underfires", sa.Integer(), nullable=False, server_default="0"), + sa.Column("ap", sa.Float(), nullable=True), + sa.Column("precision_cv", sa.Float(), nullable=True), + sa.Column("recall", sa.Float(), nullable=True), + sa.Column("n_pos", sa.Integer(), nullable=True), + ) + op.create_index( + "ix_head_metrics_snapshot_tag_id", "head_metrics_snapshot", ["tag_id"], + ) + op.create_index( + "ix_head_metrics_snapshot_snapshot_at", "head_metrics_snapshot", + ["snapshot_at"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_head_metrics_snapshot_snapshot_at", table_name="head_metrics_snapshot" + ) + op.drop_index( + "ix_head_metrics_snapshot_tag_id", table_name="head_metrics_snapshot" + ) + op.drop_table("head_metrics_snapshot") + op.drop_table("head_metric") diff --git a/alembic/versions/0061_image_region.py b/alembic/versions/0061_image_region.py new file mode 100644 index 0000000..b3af8a9 --- /dev/null +++ b/alembic/versions/0061_image_region.py @@ -0,0 +1,59 @@ +"""image_region: detected/proposed regions + their crop embeddings (#114) + +Storage backbone of the crop pipeline. A region = normalized bbox + the crop's +embedding (CCIP for face/figure → character id; SigLIP for concept regions → +head bag-of-embeddings). Also serves as grounded-tag bbox provenance. + +Revision ID: 0061 +Revises: 0060 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector + +revision: str = "0061" +down_revision: Union[str, None] = "0060" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_CCIP_DIM = 768 +_SIGLIP_DIM = 1152 + + +def upgrade() -> None: + op.create_table( + "image_region", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "image_record_id", sa.Integer(), + sa.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, + ), + sa.Column("kind", sa.String(length=16), nullable=False), + # Video/animated: source frame timestamp (seconds); NULL for stills. + sa.Column("frame_time", sa.Float(), nullable=True), + sa.Column("rx", sa.Float(), nullable=False), + sa.Column("ry", sa.Float(), nullable=False), + sa.Column("rw", sa.Float(), nullable=False), + sa.Column("rh", sa.Float(), nullable=False), + sa.Column("score", sa.Float(), nullable=True), + sa.Column("detector_version", sa.String(length=64), nullable=True), + sa.Column("crop_version", sa.String(length=64), nullable=True), + sa.Column("embedding_version", sa.String(length=128), nullable=True), + sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=True), + sa.Column("siglip_embedding", Vector(_SIGLIP_DIM), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + ) + op.create_index( + "ix_image_region_image_record_id", "image_region", ["image_record_id"], + ) + + +def downgrade() -> None: + op.drop_index("ix_image_region_image_record_id", table_name="image_region") + op.drop_table("image_region") diff --git a/alembic/versions/0062_gpu_job.py b/alembic/versions/0062_gpu_job.py new file mode 100644 index 0000000..a044995 --- /dev/null +++ b/alembic/versions/0062_gpu_job.py @@ -0,0 +1,55 @@ +"""gpu_job: the HTTP-leased GPU work queue for the desktop agent (#114) + +The agent stays HTTP-only — the server enqueues per-(image, task) jobs here and +the agent leases/submits over the web API; Redis/Postgres stay private. + +Revision ID: 0062 +Revises: 0061 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0062" +down_revision: Union[str, None] = "0061" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "gpu_job", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "image_record_id", sa.Integer(), + sa.ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, + ), + sa.Column("task", sa.String(length=32), nullable=False), + sa.Column( + "status", sa.String(length=16), nullable=False, + server_default="pending", + ), + sa.Column("lease_token", sa.String(length=64), nullable=True), + sa.Column("leased_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"), + sa.Column("error", sa.Text(), nullable=True), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + ) + op.create_index("ix_gpu_job_image_record_id", "gpu_job", ["image_record_id"]) + op.create_index("ix_gpu_job_status", "gpu_job", ["status"]) + + +def downgrade() -> None: + op.drop_index("ix_gpu_job_status", table_name="gpu_job") + op.drop_index("ix_gpu_job_image_record_id", table_name="gpu_job") + op.drop_table("gpu_job") diff --git a/alembic/versions/0063_ccip_match_threshold.py b/alembic/versions/0063_ccip_match_threshold.py new file mode 100644 index 0000000..d841398 --- /dev/null +++ b/alembic/versions/0063_ccip_match_threshold.py @@ -0,0 +1,33 @@ +"""ml_settings.ccip_match_threshold — tunable CCIP character-match cut (#114) + +The v1 matcher used a flat 0.75 cosine; live data showed that over-fires (a +high-reference character matched a scatter of images). 0.85 keeps the confident +single-character matches and drops the noise. Tunable from the GPU agent card. + +Revision ID: 0063 +Revises: 0062 +Create Date: 2026-06-29 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0063" +down_revision: Union[str, None] = "0062" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "ccip_match_threshold", sa.Float(), nullable=False, + server_default="0.85", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "ccip_match_threshold") diff --git a/alembic/versions/0064_ccip_auto_apply.py b/alembic/versions/0064_ccip_auto_apply.py new file mode 100644 index 0000000..e5323cf --- /dev/null +++ b/alembic/versions/0064_ccip_auto_apply.py @@ -0,0 +1,42 @@ +"""ml_settings: CCIP auto-apply switch + threshold (#114) + +Confident CCIP character matches auto-tag (source='ccip_auto') on a daily sweep, +so identity tags keep flowing without pressing a button. ON by default (opt-out, +like head auto-apply); the high threshold (0.92, above the 0.85 suggest cut) + +single-character references keep it safe, and every auto-tag is reversible. + +Revision ID: 0064 +Revises: 0063 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0064" +down_revision: Union[str, None] = "0063" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "ccip_auto_apply_enabled", sa.Boolean(), nullable=False, + server_default=sa.true(), + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "ccip_auto_apply_threshold", sa.Float(), nullable=False, + server_default="0.92", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "ccip_auto_apply_threshold") + op.drop_column("ml_settings", "ccip_auto_apply_enabled") diff --git a/alembic/versions/0065_embedder_model_name.py b/alembic/versions/0065_embedder_model_name.py new file mode 100644 index 0000000..0a986b3 --- /dev/null +++ b/alembic/versions/0065_embedder_model_name.py @@ -0,0 +1,35 @@ +"""ml_settings: embedder_model_name (#1190 operator model swap) + +The embedder MODEL VERSION was already a setting (and stamps image_record. +siglip_model_version); the HF model NAME was env-only, so an operator couldn't +actually point the pipeline at a different embedder. Storing the name as a +setting makes the model an operator choice: set name + version → re-embed (the +GPU agent) → retrain heads. Default = the current SigLIP so400m. + +Revision ID: 0065 +Revises: 0064 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0065" +down_revision: Union[str, None] = "0064" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "embedder_model_name", sa.String(length=128), nullable=False, + server_default="google/siglip-so400m-patch14-384", + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "embedder_model_name") diff --git a/alembic/versions/0066_drop_centroids.py b/alembic/versions/0066_drop_centroids.py new file mode 100644 index 0000000..d75a334 --- /dev/null +++ b/alembic/versions/0066_drop_centroids.py @@ -0,0 +1,57 @@ +"""drop the dead per-tag centroid subsystem (#1189 cleanup) + +The v2 pivot replaced per-tag SigLIP centroids with learned heads + CCIP. +Nothing read the centroids anymore — they were recomputed (on merge + a daily +beat) but never consumed for suggestions or auto-apply. Remove the storage + +its two now-unused settings columns. (The recompute tasks, beat, endpoint, +service, and UI card are removed in the same change.) + +Revision ID: 0066 +Revises: 0065 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0066" +down_revision: Union[str, None] = "0065" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_table("tag_reference_embedding") + op.drop_column("ml_settings", "centroid_similarity_threshold") + op.drop_column("ml_settings", "min_reference_images") + + +def downgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "min_reference_images", sa.Integer(), nullable=False, + server_default="5", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "centroid_similarity_threshold", sa.Float(), nullable=False, + server_default="0.55", + ), + ) + op.create_table( + "tag_reference_embedding", + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column("embedding", sa.LargeBinary(), nullable=False), + sa.Column("reference_count", sa.Integer(), nullable=False), + sa.Column("model_version", sa.String(length=128), nullable=False), + sa.Column( + "updated_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False, + ), + sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("tag_id"), + ) diff --git a/alembic/versions/0067_retire_camie_allowlist.py b/alembic/versions/0067_retire_camie_allowlist.py new file mode 100644 index 0000000..e3edd02 --- /dev/null +++ b/alembic/versions/0067_retire_camie_allowlist.py @@ -0,0 +1,66 @@ +"""retire the Camie tagger + allowlist bulk-apply (#1189) + +The v2 pivot made heads + CCIP the tag source and head auto-apply the earned +propagation. The Camie tagger ran only to feed the allowlist bulk-apply (its +predictions had no other consumer), and the allowlist was a second, un-earned +auto-apply path parallel to heads. Both are retired — drop their storage. + +(image_prediction = Camie's per-image predictions; tag_allowlist = the bulk- +apply allowlist. Nothing references INTO these tables, so the drop is clean.) + +Revision ID: 0067 +Revises: 0066 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0067" +down_revision: Union[str, None] = "0066" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_table("image_prediction") + op.drop_table("tag_allowlist") + + +def downgrade() -> None: + op.create_table( + "tag_allowlist", + sa.Column("tag_id", sa.Integer(), nullable=False), + sa.Column( + "min_confidence", sa.Float(), nullable=False, server_default="0.9" + ), + sa.Column( + "created_at", sa.DateTime(timezone=True), + server_default=sa.func.now(), nullable=False, + ), + sa.ForeignKeyConstraint(["tag_id"], ["tag.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("tag_id"), + sa.CheckConstraint( + "min_confidence >= 0 AND min_confidence <= 1", + name="ck_tag_allowlist_confidence_range", + ), + ) + op.create_table( + "image_prediction", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("image_record_id", sa.Integer(), nullable=False), + sa.Column("raw_name", sa.String(length=255), nullable=False), + sa.Column("category", sa.String(length=32), nullable=False), + sa.Column("score", sa.Float(), nullable=False), + sa.ForeignKeyConstraint( + ["image_record_id"], ["image_record.id"], ondelete="CASCADE" + ), + ) + op.create_index( + "ix_image_prediction_image", "image_prediction", ["image_record_id"] + ) + op.create_index( + "ix_image_prediction_name_score", "image_prediction", + ["raw_name", "score"], + ) diff --git a/alembic/versions/0068_drop_dead_tagger_settings.py b/alembic/versions/0068_drop_dead_tagger_settings.py new file mode 100644 index 0000000..770676d --- /dev/null +++ b/alembic/versions/0068_drop_dead_tagger_settings.py @@ -0,0 +1,80 @@ +"""drop dead tagger/suggestion settings + columns left after Camie retirement (#1199) + +Hygiene follow-up to #1189. These were left inert to bound that change; nothing +reads them now: +- ml_settings: tagger_store_floor + tagger_model_version (only the deleted Camie + tagger used them), suggestion_threshold_character/general (already dead pre- + retirement — scoring uses per-head thresholds), video_min_tag_frames (only the + deleted video-prediction aggregator used it). +- image_record: tagger_model_version (no writer now), centroid_scores (long-dead + JSON cache, no reader). + +Revision ID: 0068 +Revises: 0067 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0068" +down_revision: Union[str, None] = "0067" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_column("ml_settings", "suggestion_threshold_character") + op.drop_column("ml_settings", "suggestion_threshold_general") + op.drop_column("ml_settings", "tagger_store_floor") + op.drop_column("ml_settings", "video_min_tag_frames") + op.drop_column("ml_settings", "tagger_model_version") + op.drop_column("image_record", "tagger_model_version") + op.drop_column("image_record", "centroid_scores") + + +def downgrade() -> None: + op.add_column( + "image_record", + sa.Column("centroid_scores", sa.JSON(), nullable=True), + ) + op.add_column( + "image_record", + sa.Column("tagger_model_version", sa.String(length=128), nullable=True), + ) + op.add_column( + "ml_settings", + sa.Column( + "tagger_model_version", sa.String(length=128), nullable=False, + server_default="camie-tagger-v2", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "video_min_tag_frames", sa.Integer(), nullable=False, + server_default="3", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "tagger_store_floor", sa.Float(), nullable=False, + server_default="0.7", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "suggestion_threshold_general", sa.Float(), nullable=False, + server_default="0.7", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "suggestion_threshold_character", sa.Float(), nullable=False, + server_default="0.7", + ), + ) diff --git a/alembic/versions/0069_default_siglip2.py b/alembic/versions/0069_default_siglip2.py new file mode 100644 index 0000000..7bef8b1 --- /dev/null +++ b/alembic/versions/0069_default_siglip2.py @@ -0,0 +1,51 @@ +"""default the embedder to SigLIP 2 — for FRESH installs only (#1203) + +Make SigLIP 2 (so400m, 512px; a 1152-d drop-in) the default embedder. New +installs start on it. An EXISTING library is NOT touched: flipping its stored +embedder version would mark every embedding stale (the scorer is version-gated) +and kill suggestions until a full re-embed+retrain — so an existing instance +switches deliberately via Settings → GPU agent → Embedding model → Re-embed → +Retrain. We detect "fresh" by the absence of any embedded image. + +Revision ID: 0069 +Revises: 0068 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0069" +down_revision: Union[str, None] = "0068" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +_NEW_NAME = "google/siglip2-so400m-patch16-512" +_NEW_VERSION = "siglip2-so400m-patch16-512" +_OLD_NAME = "google/siglip-so400m-patch14-384" +_OLD_VERSION = "siglip-so400m-patch14-384" + + +def upgrade() -> None: + # Fresh install (nothing embedded yet) → adopt SigLIP 2. + op.execute( + f""" + UPDATE ml_settings SET + embedder_model_name = '{_NEW_NAME}', + embedder_model_version = '{_NEW_VERSION}' + WHERE NOT EXISTS ( + SELECT 1 FROM image_record WHERE siglip_embedding IS NOT NULL + ) + """ + ) + op.alter_column("ml_settings", "embedder_model_name", server_default=_NEW_NAME) + op.alter_column( + "ml_settings", "embedder_model_version", server_default=_NEW_VERSION + ) + + +def downgrade() -> None: + op.alter_column("ml_settings", "embedder_model_name", server_default=_OLD_NAME) + op.alter_column( + "ml_settings", "embedder_model_version", server_default=_OLD_VERSION + ) diff --git a/alembic/versions/0070_gpu_job_lease_indexes.py b/alembic/versions/0070_gpu_job_lease_indexes.py new file mode 100644 index 0000000..10ec3f9 --- /dev/null +++ b/alembic/versions/0070_gpu_job_lease_indexes.py @@ -0,0 +1,44 @@ +"""partial indexes so GPU-job leasing stays O(batch), not O(completed) + +The lease claims the lowest-id pending (or expired-leased) jobs. With only a +plain `status` index, `... ORDER BY id LIMIT n` walked the primary-key index from +the start, skipping the entire prefix of already-done/error rows before reaching +pending ones — so leasing slowed to a crawl as `done` piled up (the whole reason +throughput fell off a cliff mid-run and /status stalled). Two partial indexes fix +it: the pending one is id-ordered so the hot path reads just the first n entries, +and the leased-expiry one keeps the crash-recovery reclaim + the orphan sweep +cheap. They cover only the small live slice of the table, so they stay tiny even +as the done/error history grows to millions. + +Revision ID: 0070 +Revises: 0069 +Create Date: 2026-06-30 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0070" +down_revision: Union[str, None] = "0069" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Hot path: lowest-id pending jobs. Index on id, restricted to pending, so + # `WHERE status='pending' ORDER BY id LIMIT n` is a short index-order scan. + op.create_index( + "ix_gpu_job_pending", "gpu_job", ["id"], + postgresql_where=sa.text("status = 'pending'"), + ) + # Crash-recovery: expired leases, for the lease backstop + recover_orphaned. + op.create_index( + "ix_gpu_job_leased_expires", "gpu_job", ["lease_expires_at"], + postgresql_where=sa.text("status = 'leased'"), + ) + + +def downgrade() -> None: + op.drop_index("ix_gpu_job_leased_expires", table_name="gpu_job") + op.drop_index("ix_gpu_job_pending", table_name="gpu_job") diff --git a/alembic/versions/0071_image_record_earliest_post_date.py b/alembic/versions/0071_image_record_earliest_post_date.py new file mode 100644 index 0000000..b2e8f0c --- /dev/null +++ b/alembic/versions/0071_image_record_earliest_post_date.py @@ -0,0 +1,80 @@ +"""image_record.earliest_post_date: original-publish gallery sort key + index + +Revision ID: 0071 +Revises: 0070 +Create Date: 2026-07-01 + +effective_date (0035) keys off the PRIMARY post — which is often the repost / +download the file actually came from — and falls back to created_at, so the +gallery's default order surfaces download dates rather than when content was +first posted (operator-flagged 2026-07-01). Materialize a second sort key, +earliest_post_date = MIN(post_date) across ALL of an image's provenance posts +(every post it appears in), falling back to created_at only when no linked post +carries a date. Indexed (DESC, id DESC) so the "post date" gallery sort is an +index range scan just like effective_date. + +Backfill mirrors 0035: created_at baseline, then override with the MIN over +image_provenance ⋈ post. New rows get the created_at-equivalent server default; +services/importer.py recomputes it whenever a dated post is linked. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0071" +down_revision: Union[str, None] = "0070" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Add nullable first so the backfill can populate before NOT NULL. + op.add_column( + "image_record", + sa.Column("earliest_post_date", sa.DateTime(timezone=True), nullable=True), + ) + # Baseline: download date. Set-based (no per-row binds) → immune to the + # 65535 bind-parameter ceiling regardless of library size. + op.execute( + """ + UPDATE image_record + SET earliest_post_date = created_at + """ + ) + # Override with the earliest post_date across EVERY post the image appears + # in (image_provenance is the many-to-many edge; ignore posts with no date). + op.execute( + """ + UPDATE image_record AS ir + SET earliest_post_date = sub.min_date + FROM ( + SELECT ip.image_record_id AS iid, MIN(p.post_date) AS min_date + FROM image_provenance AS ip + JOIN post AS p ON p.id = ip.post_id + WHERE p.post_date IS NOT NULL + GROUP BY ip.image_record_id + ) AS sub + WHERE ir.id = sub.iid + """ + ) + op.alter_column( + "image_record", + "earliest_post_date", + nullable=False, + server_default=sa.text("now()"), + ) + # DESC/DESC matches the gallery's ORDER BY earliest_post_date DESC, id DESC + # so the "post date" scroll is a forward index scan; raw SQL because + # alembic's column list doesn't express per-column DESC cleanly. + op.execute( + "CREATE INDEX ix_image_record_earliest_post_date " + "ON image_record (earliest_post_date DESC, id DESC)" + ) + + +def downgrade() -> None: + op.drop_index( + "ix_image_record_earliest_post_date", table_name="image_record" + ) + op.drop_column("image_record", "earliest_post_date") diff --git a/alembic/versions/0072_gpu_job_triage_status.py b/alembic/versions/0072_gpu_job_triage_status.py new file mode 100644 index 0000000..1dce875 --- /dev/null +++ b/alembic/versions/0072_gpu_job_triage_status.py @@ -0,0 +1,32 @@ +"""gpu_job.triage_status — the probe's verdict on an errored job's FILE + +Failure triage (#125): a periodic sweep probes each errored image's file +(sha256 + decode, verify_integrity's machinery) exactly once and stores the +verdict here — 'defect' (the file is bad: recovery material, excluded from +/retry_errors) or 'file_ok' (failure was operational, safe to retry). NULL +means not yet probed; selecting on NULL is what makes the sweep resumable. +No index: the errored slice the sweep scans is tiny by design (tombstones). + +Revision ID: 0072 +Revises: 0071 +Create Date: 2026-07-02 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0072" +down_revision: Union[str, None] = "0071" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "gpu_job", sa.Column("triage_status", sa.String(16), nullable=True) + ) + + +def downgrade() -> None: + op.drop_column("gpu_job", "triage_status") diff --git a/alembic/versions/0073_drop_tag_eval_run.py b/alembic/versions/0073_drop_tag_eval_run.py new file mode 100644 index 0000000..4aedb38 --- /dev/null +++ b/alembic/versions/0073_drop_tag_eval_run.py @@ -0,0 +1,46 @@ +"""drop tag_eval_run — the head-vs-centroid eval harness is retired + +The eval (#1130) existed to prove the heads tagging spine on the operator's own +data. It did; the operator accepted the system and retired the harness +(2026-07-02) — card, API, task, model and this table all go. The eval's data +loaders + metric helpers live on in services/ml/training_data.py, where the +production heads trainer uses them nightly. + +Revision ID: 0073 +Revises: 0072 +Create Date: 2026-07-02 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision: str = "0073" +down_revision: Union[str, None] = "0072" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_index("ix_tag_eval_run_status", table_name="tag_eval_run") + op.drop_table("tag_eval_run") + + +def downgrade() -> None: + # Recreates the shape from 0056 (data is not restorable). + op.create_table( + "tag_eval_run", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("params", postgresql.JSONB(), nullable=False), + sa.Column("status", sa.String(length=16), nullable=False, + server_default="running"), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now()), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("report", postgresql.JSONB(), nullable=True), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("last_progress_at", sa.DateTime(timezone=True), + nullable=True), + ) + op.create_index("ix_tag_eval_run_status", "tag_eval_run", ["status"]) diff --git a/alembic/versions/0074_ml_settings_cpu_embed_enabled.py b/alembic/versions/0074_ml_settings_cpu_embed_enabled.py new file mode 100644 index 0000000..48ff8ea --- /dev/null +++ b/alembic/versions/0074_ml_settings_cpu_embed_enabled.py @@ -0,0 +1,35 @@ +"""ml_settings.cpu_embed_enabled — the CPU embed fallback becomes a switch + +B3 (operator 2026-07-02): the ml-worker's only processing role is the CPU +whole-image embed for stacks without a GPU agent. ON by default (a fresh +install works agent-less); agent-equipped stacks that drop the ml-worker +container turn it off so import hooks stop queueing embed work into a queue +nothing consumes — the daily GPU 'embed' backfill covers those images. + +Revision ID: 0074 +Revises: 0073 +Create Date: 2026-07-02 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0074" +down_revision: Union[str, None] = "0073" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "cpu_embed_enabled", sa.Boolean(), nullable=False, + server_default=sa.true(), + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "cpu_embed_enabled") diff --git a/alembic/versions/0075_tag_is_system.py b/alembic/versions/0075_tag_is_system.py new file mode 100644 index 0000000..a6b7e7a --- /dev/null +++ b/alembic/versions/0075_tag_is_system.py @@ -0,0 +1,60 @@ +"""tag.is_system + seed the three hygiene system tags + +Training hygiene (operator 2026-07-03, milestone #128): rough WIPs tagged as a +character poison that character's head and CCIP references; banners/editor +screenshots pollute whole-image similarity. The fix keys on SYSTEM tags the +product ships — not operator configuration — so the seed lives here. + +Seeding ADOPTS an existing same-(name, kind=general) tag (case-insensitive, +matching TagService.rename's collision stance) instead of inserting a +duplicate, so an operator who already tagged `wip` keeps their applications. + +Revision ID: 0075 +Revises: 0074 +Create Date: 2026-07-03 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0075" +down_revision: Union[str, None] = "0074" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +SYSTEM_TAG_NAMES = ("wip", "banner", "editor screenshot") + + +def upgrade() -> None: + op.add_column( + "tag", + sa.Column( + "is_system", sa.Boolean(), nullable=False, + server_default=sa.false(), + ), + ) + conn = op.get_bind() + for name in SYSTEM_TAG_NAMES: + adopted = conn.execute( + sa.text( + "UPDATE tag SET is_system = true " + "WHERE lower(name) = lower(:name) AND kind = 'general'" + ), + {"name": name}, + ) + if adopted.rowcount == 0: + conn.execute( + sa.text( + "INSERT INTO tag (name, kind, is_system) " + "VALUES (:name, 'general', true)" + ), + {"name": name}, + ) + + +def downgrade() -> None: + # The seeded rows survive as ordinary general tags — dropping the flag is + # enough to disarm the mechanism, and deleting rows would orphan any + # operator applications made while the flag existed. + op.drop_column("tag", "is_system") diff --git a/alembic/versions/0076_pixiv_ledgers.py b/alembic/versions/0076_pixiv_ledgers.py new file mode 100644 index 0000000..2655130 --- /dev/null +++ b/alembic/versions/0076_pixiv_ledgers.py @@ -0,0 +1,82 @@ +"""pixiv_seen_media + pixiv_failed_media: per-source ledgers + +Revision ID: 0076 +Revises: 0075 +Create Date: 2026-07-03 + +Pixiv native ingester (milestone #129, gallery-dl → native-core migration). +Mirrors the Patreon (0037/0038) and SubscribeStar (0054) ledger tables: a +seen-ledger so routine walks skip already-ingested media (recovery bypasses +it) and a dead-letter ledger so persistently-failing media stops re-burning +backfill chunks. Pixiv URLs carry no content hash, so `filehash` is always the +synthesized ``:p`` / ``:ugoira`` key — String(128) +matches the siblings. UNIQUE (source_id, filehash) is the upsert key on each. +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0076" +down_revision: Union[str, None] = "0075" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "pixiv_seen_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("post_id", sa.String(64), nullable=True), + sa.Column( + "seen_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_pixiv_seen_media_source_id" + ), + ) + op.create_table( + "pixiv_failed_media", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column( + "source_id", + sa.Integer, + sa.ForeignKey("source.id", ondelete="CASCADE"), + nullable=False, + index=True, + ), + sa.Column("filehash", sa.String(128), nullable=False), + sa.Column("attempts", sa.Integer, nullable=False, server_default="1"), + sa.Column("last_error", sa.Text, nullable=True), + sa.Column( + "first_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.Column( + "last_failed_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("NOW()"), + ), + sa.UniqueConstraint( + "source_id", "filehash", name="uq_pixiv_failed_media_source_id" + ), + ) + + +def downgrade() -> None: + op.drop_table("pixiv_failed_media") + op.drop_table("pixiv_seen_media") diff --git a/alembic/versions/0077_artist_name_not_unique.py b/alembic/versions/0077_artist_name_not_unique.py new file mode 100644 index 0000000..6a09288 --- /dev/null +++ b/alembic/versions/0077_artist_name_not_unique.py @@ -0,0 +1,32 @@ +"""drop uq_artist_name — decouple display name from identity/storage + +Revision ID: 0077 +Revises: 0076 +Create Date: 2026-07-04 + +Artist model fragility fix (milestone #130). One `slug` column was doing +identity + storage-path + display, and BOTH `name` and `slug` were UNIQUE, so +the display name couldn't be edited freely and two genuinely different creators +collided. Decouple: `slug` stays the immutable, unique storage/identity key (the +on-disk path component — untouched here); `name` becomes freely editable, NON- +unique display text. This migration only drops the `uq_artist_name` constraint; +no data moves and no path changes. +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0077" +down_revision: Union[str, None] = "0076" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.drop_constraint("uq_artist_name", "artist", type_="unique") + + +def downgrade() -> None: + # Re-adding the UNIQUE would fail if duplicate names now exist; callers that + # need to reverse this must dedupe names first. + op.create_unique_constraint("uq_artist_name", "artist", ["name"]) diff --git a/alembic/versions/0078_ml_settings_detectors.py b/alembic/versions/0078_ml_settings_detectors.py new file mode 100644 index 0000000..6d04601 --- /dev/null +++ b/alembic/versions/0078_ml_settings_detectors.py @@ -0,0 +1,83 @@ +"""ml_settings crop-proposer / detector config (#134) + +Move the WHERE-to-crop detector config (per-proposer enable + weights + conf, +plus caps + dedupe IoU) into the DB so it's UI-tunable and announced to the GPU +agent in the lease (like the embedder model) — no restart, agent env is now +bootstrap-only. All server_defaults are the working values so existing rows + +fresh installs crop out-of-the-box with all three proposers ON. + +Revision ID: 0078 +Revises: 0077 +Create Date: 2026-07-05 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0078" +down_revision: Union[str, None] = "0077" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +_ANATOMY_DEFAULT = ( + "https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt" +) +_PANEL_DEFAULT = "mosesb/best-comic-panel-detection::best.pt" + + +def upgrade() -> None: + op.add_column("ml_settings", sa.Column( + "detector_person_enabled", sa.Boolean(), nullable=False, + server_default=sa.true())) + op.add_column("ml_settings", sa.Column( + "detector_person_weights", sa.String(512), nullable=False, + server_default="yolo11n.pt")) + op.add_column("ml_settings", sa.Column( + "detector_person_conf", sa.Float(), nullable=False, + server_default=sa.text("0.35"))) + op.add_column("ml_settings", sa.Column( + "detector_anatomy_enabled", sa.Boolean(), nullable=False, + server_default=sa.true())) + op.add_column("ml_settings", sa.Column( + "detector_anatomy_weights", sa.String(512), nullable=False, + server_default=_ANATOMY_DEFAULT)) + op.add_column("ml_settings", sa.Column( + "detector_anatomy_conf", sa.Float(), nullable=False, + server_default=sa.text("0.30"))) + op.add_column("ml_settings", sa.Column( + "detector_panel_enabled", sa.Boolean(), nullable=False, + server_default=sa.true())) + op.add_column("ml_settings", sa.Column( + "detector_panel_weights", sa.String(512), nullable=False, + server_default=_PANEL_DEFAULT)) + op.add_column("ml_settings", sa.Column( + "detector_panel_conf", sa.Float(), nullable=False, + server_default=sa.text("0.30"))) + op.add_column("ml_settings", sa.Column( + "detector_max_figures", sa.Integer(), nullable=False, + server_default=sa.text("8"))) + op.add_column("ml_settings", sa.Column( + "detector_max_components", sa.Integer(), nullable=False, + server_default=sa.text("8"))) + op.add_column("ml_settings", sa.Column( + "detector_max_panels", sa.Integer(), nullable=False, + server_default=sa.text("8"))) + op.add_column("ml_settings", sa.Column( + "detector_max_regions", sa.Integer(), nullable=False, + server_default=sa.text("128"))) + op.add_column("ml_settings", sa.Column( + "detector_dedupe_iou", sa.Float(), nullable=False, + server_default=sa.text("0.85"))) + + +def downgrade() -> None: + for col in ( + "detector_person_enabled", "detector_person_weights", "detector_person_conf", + "detector_anatomy_enabled", "detector_anatomy_weights", "detector_anatomy_conf", + "detector_panel_enabled", "detector_panel_weights", "detector_panel_conf", + "detector_max_figures", "detector_max_components", "detector_max_panels", + "detector_max_regions", "detector_dedupe_iou", + ): + op.drop_column("ml_settings", col) diff --git a/alembic/versions/0079_character_prototypes.py b/alembic/versions/0079_character_prototypes.py new file mode 100644 index 0000000..8ada2f4 --- /dev/null +++ b/alembic/versions/0079_character_prototypes.py @@ -0,0 +1,77 @@ +"""character prototype store (#1317) — precomputed, incremental CCIP references + +New tables character_prototype + ccip_prototype_state, plus MLSettings columns +ccip_ref_signature (cheap global change gate) + ccip_prototype_cap (per-character +reference cap). The reference set the CCIP matcher uses becomes a precomputed +artifact refreshed incrementally off the request path. See milestone 138 / +backend.app.services.ml.character_prototypes. + +Revision ID: 0079 +Revises: 0078 +Create Date: 2026-07-06 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op +from pgvector.sqlalchemy import Vector + +revision: str = "0079" +down_revision: Union[str, None] = "0078" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# Matches models.image_region.CCIP_DIM (the CCIP figure-embedding width). +_CCIP_DIM = 768 + + +def upgrade() -> None: + op.create_table( + "character_prototype", + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, + ), + sa.Column("ccip_embedding", Vector(_CCIP_DIM), nullable=False), + sa.Column( + "region_id", sa.Integer(), + sa.ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True, + ), + ) + op.create_index( + "ix_character_prototype_tag_id", "character_prototype", ["tag_id"] + ) + op.create_table( + "ccip_prototype_state", + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, + ), + sa.Column("fingerprint", sa.String(64), nullable=False), + sa.Column( + "updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + ) + op.add_column( + "ml_settings", + sa.Column("ccip_ref_signature", sa.String(128), nullable=True), + ) + op.add_column( + "ml_settings", + sa.Column( + "ccip_prototype_cap", sa.Integer(), nullable=False, + server_default=sa.text("64"), + ), + ) + + +def downgrade() -> None: + op.drop_column("ml_settings", "ccip_prototype_cap") + op.drop_column("ml_settings", "ccip_ref_signature") + op.drop_table("ccip_prototype_state") + op.drop_index( + "ix_character_prototype_tag_id", table_name="character_prototype" + ) + op.drop_table("character_prototype") diff --git a/alembic/versions/0080_tag_head_train_fingerprint.py b/alembic/versions/0080_tag_head_train_fingerprint.py new file mode 100644 index 0000000..b4bd224 --- /dev/null +++ b/alembic/versions/0080_tag_head_train_fingerprint.py @@ -0,0 +1,31 @@ +"""tag_head.train_fingerprint (#1317 phase 2) — incremental head retraining + +A per-head training-data fingerprint (positive + rejection count/latest-timestamp) +so a manual Retrain refits only the tags whose data changed; the nightly run +ignores it (full reconcile). Nullable — a NULL fingerprint (existing heads) forces +a refit on the first incremental run, then it's stamped. + +Revision ID: 0080 +Revises: 0079 +Create Date: 2026-07-06 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0080" +down_revision: Union[str, None] = "0079" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "tag_head", + sa.Column("train_fingerprint", sa.String(128), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("tag_head", "train_fingerprint") diff --git a/alembic/versions/0081_stricter_auto_apply_defaults.py b/alembic/versions/0081_stricter_auto_apply_defaults.py new file mode 100644 index 0000000..8030eec --- /dev/null +++ b/alembic/versions/0081_stricter_auto_apply_defaults.py @@ -0,0 +1,43 @@ +"""stricter auto-apply defaults (milestone 139) — cut auto-apply misfires + +head_auto_apply_min_positives 30→50 and ccip_auto_apply_threshold 0.92→0.95 +(operator-asked 2026-07-06). The head graduation precision bar stays 0.97 — the +operator confirmed the general-tag confidence was already well tuned; only the +support floor + the CCIP match confidence are raised. The model defaults change +for fresh installs; here we bump the existing singleton row IFF it is still at +the previous default, so a deliberate operator change is NOT clobbered. + +Revision ID: 0081 +Revises: 0080 +Create Date: 2026-07-06 +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0081" +down_revision: Union[str, None] = "0080" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + "UPDATE ml_settings SET head_auto_apply_min_positives = 50 " + "WHERE head_auto_apply_min_positives = 30" + ) + op.execute( + "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.95 " + "WHERE ccip_auto_apply_threshold = 0.92" + ) + + +def downgrade() -> None: + op.execute( + "UPDATE ml_settings SET head_auto_apply_min_positives = 30 " + "WHERE head_auto_apply_min_positives = 50" + ) + op.execute( + "UPDATE ml_settings SET ccip_auto_apply_threshold = 0.92 " + "WHERE ccip_auto_apply_threshold = 0.95" + ) diff --git a/alembic/versions/0082_presentation_auto_hide.py b/alembic/versions/0082_presentation_auto_hide.py new file mode 100644 index 0000000..8dc1f3c --- /dev/null +++ b/alembic/versions/0082_presentation_auto_hide.py @@ -0,0 +1,85 @@ +"""presentation-chrome auto-hide (#141) — settings knobs + review table + +MLSettings gains presentation_auto_apply_enabled / _threshold and +presentation_conflict_threshold: banner + editor-screenshot auto-hide on the +sweep with a FLAT threshold (decoupled from content-head graduation), and a +conflict threshold that flags an auto-hide that "also looks like content". + +New table presentation_review records an auto-hidden chrome image that also +scored high on a content head, surfaced in the Hidden view for a keep-hidden / +un-hide decision. Resolved rows are pruned by retention. + +Revision ID: 0082 +Revises: 0081 +Create Date: 2026-07-07 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0082" +down_revision: Union[str, None] = "0081" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "presentation_auto_apply_enabled", sa.Boolean(), nullable=False, + server_default=sa.text("true"), + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "presentation_auto_apply_threshold", sa.Float(), nullable=False, + server_default=sa.text("0.90"), + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "presentation_conflict_threshold", sa.Float(), nullable=False, + server_default=sa.text("0.50"), + ), + ) + op.create_table( + "presentation_review", + sa.Column( + "image_record_id", sa.Integer(), + sa.ForeignKey("image_record.id", ondelete="CASCADE"), + primary_key=True, + ), + sa.Column( + "tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, + ), + sa.Column( + "conflict_tag_id", sa.Integer(), + sa.ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, + ), + sa.Column("conflict_score", sa.Float(), nullable=False), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.func.now(), + ), + sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True), + ) + # The review list queries the unresolved flags (resolved_at IS NULL). + op.create_index( + "ix_presentation_review_resolved_at", "presentation_review", + ["resolved_at"], + ) + + +def downgrade() -> None: + op.drop_index( + "ix_presentation_review_resolved_at", table_name="presentation_review" + ) + op.drop_table("presentation_review") + op.drop_column("ml_settings", "presentation_conflict_threshold") + op.drop_column("ml_settings", "presentation_auto_apply_threshold") + op.drop_column("ml_settings", "presentation_auto_apply_enabled") diff --git a/alembic/versions/0083_post_translation.py b/alembic/versions/0083_post_translation.py new file mode 100644 index 0000000..d491d03 --- /dev/null +++ b/alembic/versions/0083_post_translation.py @@ -0,0 +1,73 @@ +"""post-text translation via Interpreter (milestone 143) — Post columns + settings + +Post gains the translated title/description + the detected source language, +Interpreter engine_version (cache key), and translated_at — filled by the +translate sweep. ImportSettings gains translation_enabled (OFF by default), +interpreter_base_url (EMPTY — the operator sets their own, behind a reverse +proxy), and translation_target_lang (en). + +Revision ID: 0083 +Revises: 0082 +Create Date: 2026-07-07 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0083" +down_revision: Union[str, None] = "0082" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "post", sa.Column("post_title_translated", sa.Text(), nullable=True) + ) + op.add_column( + "post", sa.Column("description_translated", sa.Text(), nullable=True) + ) + op.add_column( + "post", + sa.Column("translated_source_lang", sa.String(8), nullable=True), + ) + op.add_column( + "post", + sa.Column("translation_engine_version", sa.String(128), nullable=True), + ) + op.add_column( + "post", + sa.Column("translated_at", sa.DateTime(timezone=True), nullable=True), + ) + op.add_column( + "import_settings", + sa.Column( + "translation_enabled", sa.Boolean(), nullable=False, + server_default=sa.text("false"), + ), + ) + op.add_column( + "import_settings", + sa.Column( + "interpreter_base_url", sa.Text(), nullable=False, server_default="", + ), + ) + op.add_column( + "import_settings", + sa.Column( + "translation_target_lang", sa.Text(), nullable=False, + server_default="en", + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "translation_target_lang") + op.drop_column("import_settings", "interpreter_base_url") + op.drop_column("import_settings", "translation_enabled") + op.drop_column("post", "translated_at") + op.drop_column("post", "translation_engine_version") + op.drop_column("post", "translated_source_lang") + op.drop_column("post", "description_translated") + op.drop_column("post", "post_title_translated") diff --git a/alembic/versions/0084_translation_strictness_override.py b/alembic/versions/0084_translation_strictness_override.py new file mode 100644 index 0000000..cd514b4 --- /dev/null +++ b/alembic/versions/0084_translation_strictness_override.py @@ -0,0 +1,51 @@ +"""translation strictness setting + per-post translation override (milestone 155) + +ImportSettings gains ``translation_min_confidence`` (the latin-script acceptance +floor, now operator-tunable in the UI; default 0.9 — stricter than the old +hardcoded 0.8, since Interpreter confidently mis-detects short ASCII English at +~0.86). Post gains ``translation_override`` — a sticky per-post choice of +auto / force / original so the operator can force a skipped translation on, or +knock a wrongly-translated one back to the original, and have it survive a +Re-translate-all. + +Revision ID: 0084 +Revises: 0083 +Create Date: 2026-07-10 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0084" +down_revision: Union[str, None] = "0083" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_settings", + sa.Column( + "translation_min_confidence", sa.Float(), nullable=False, + server_default=sa.text("0.9"), + ), + ) + op.add_column( + "post", + sa.Column( + "translation_override", sa.String(16), nullable=False, + server_default="auto", + ), + ) + op.create_check_constraint( + "ck_post_translation_override", + "post", + "translation_override IN ('auto', 'force', 'original')", + ) + + +def downgrade() -> None: + op.drop_constraint("ck_post_translation_override", "post", type_="check") + op.drop_column("post", "translation_override") + op.drop_column("import_settings", "translation_min_confidence") diff --git a/alembic/versions/0085_wip_title_tagging.py b/alembic/versions/0085_wip_title_tagging.py new file mode 100644 index 0000000..4d260b1 --- /dev/null +++ b/alembic/versions/0085_wip_title_tagging.py @@ -0,0 +1,35 @@ +"""title-based WIP auto-tagging (task #1458) — ImportSettings toggle + +ImportSettings gains wip_title_tagging_enabled (ON by default): when a freshly +imported post's title explicitly declares work-in-progress ("WIP" / "work in +progress"), the importer applies the `wip` system tag to its images. No new +table — the tag itself is the seeded `wip` system tag (migration 0075) and the +application reuses image_tag with source='wip_title'. + +Revision ID: 0085 +Revises: 0084 +Create Date: 2026-07-12 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0085" +down_revision: Union[str, None] = "0084" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_settings", + sa.Column( + "wip_title_tagging_enabled", sa.Boolean(), nullable=False, + server_default=sa.text("true"), + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "wip_title_tagging_enabled") diff --git a/alembic/versions/0086_process_auto_apply_settings.py b/alembic/versions/0086_process_auto_apply_settings.py new file mode 100644 index 0000000..16f03c7 --- /dev/null +++ b/alembic/versions/0086_process_auto_apply_settings.py @@ -0,0 +1,61 @@ +"""process auto-apply settings + review mode (#1464) — system-tag refactor + +The system-tag behavior refactor gives `wip` / `editor screenshot` (the PROCESS +group) their own provisional auto-apply, parallel to the presentation (chrome) +sweep. MLSettings gains three knobs: enabled (OFF by default — a new whole-library +auto-tagger is opt-in), the flat apply threshold, and the ring-loud conflict +threshold. presentation_review gains a `mode` column so one review surface serves +both chrome and process flags (existing rows backfill 'chrome'). server_defaults +so the existing rows fill cleanly. + +Revision ID: 0086 +Revises: 0085 +Create Date: 2026-07-13 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0086" +down_revision: Union[str, None] = "0085" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "ml_settings", + sa.Column( + "process_auto_apply_enabled", sa.Boolean(), nullable=False, + server_default=sa.text("false"), + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "process_auto_apply_threshold", sa.Float(), nullable=False, + server_default="0.90", + ), + ) + op.add_column( + "ml_settings", + sa.Column( + "process_conflict_threshold", sa.Float(), nullable=False, + server_default="0.50", + ), + ) + op.add_column( + "presentation_review", + sa.Column( + "mode", sa.String(16), nullable=False, + server_default="chrome", + ), + ) + + +def downgrade() -> None: + op.drop_column("presentation_review", "mode") + op.drop_column("ml_settings", "process_conflict_threshold") + op.drop_column("ml_settings", "process_auto_apply_threshold") + op.drop_column("ml_settings", "process_auto_apply_enabled") diff --git a/alembic/versions/0087_baseline.py b/alembic/versions/0087_baseline.py deleted file mode 100644 index bf803ae..0000000 --- a/alembic/versions/0087_baseline.py +++ /dev/null @@ -1,872 +0,0 @@ -"""Collapsed baseline — the whole schema in one revision. - -Replaces revisions 0001..0087, which narrated the build-out of this project -and were deleted in milestone 328 step 1. A new install creates the schema in -one step instead of replaying that history. - -WHY THE REVISION ID IS "0087" AND NOT "0001" --------------------------------------------- -It is deliberately the id of the LAST revision this baseline collapses, so an -existing database needs no intervention at all: - - * a fresh install finds current=none, head=0087, runs this file once, and - ends stamped at 0087. - * an existing install is ALREADY at 0087, so `alembic upgrade head` finds - current == head and does nothing. - -The alternative — numbering this 0001 and stamping every existing database — -means running `alembic stamp` against live data, and stamp VALIDATES NOTHING. -It writes a version string whether or not the schema actually matches, so a -wrong baseline would be discovered later, by the next real migration, with no -clean way back. Keeping the id removes that operation instead of making it -safe. Future revisions continue at 0088. - -The one case this makes worse, and it fails LOUDLY rather than silently: a -database still sitting between 0001 and 0086 (i.e. never upgraded to head) -cannot be located in this chain and errors out. Upgrade to 0087 on a -pre-squash build first, then take this one. - -WHAT IS HAND-WRITTEN HERE -------------------------- -Most of this file is `alembic revision --autogenerate` output, but four -things are NOT in SQLAlchemy metadata and the generator cannot produce them. -Each fails differently, and none of them fail at generation time: - - 1. CREATE EXTENSION vector (was 0001) — without it the VECTOR - columns below cannot be created at all. - 2. CREATE EXTENSION tsm_system_rows (was 0004) — used by the random-sample - query path; its absence surfaces only when that query runs. - 3. The HNSW index on image_record.siglip_embedding (was 0036). Raw SQL - because alembic's create_index cannot express `USING hnsw (... - vector_cosine_ops)`. Its absence is the quietest failure of the four: - everything works, similarity search just stops using an index. - 4. `import pgvector.sqlalchemy.vector`. Autogenerate EMITS references to - pgvector.sqlalchemy.vector.VECTOR but does not add the import, so the - generated file dies with NameError on first run. - -The acceptance test for this file is not that it reads correctly — it is -`.forgejo/workflows/baseline.yml`, which builds a database from the old -0001..0087 chain (read out of git) and one from this file, and diffs -pg_dump --schema-only output. That is what proves nothing was missed. - -Revision ID: 0087 -Revises: -Create Date: 2026-08-30 - -""" -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# Autogenerate references pgvector.sqlalchemy.vector.VECTOR without importing -# it. Item 4 above. -import pgvector.sqlalchemy.vector - -revision: str = "0087" -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # Extensions FIRST: the VECTOR columns below cannot be created without - # `vector`, so ordering here is load-bearing, not tidiness. - op.execute("CREATE EXTENSION IF NOT EXISTS vector") - op.execute("CREATE EXTENSION IF NOT EXISTS tsm_system_rows") - - op.create_table('app_setting', - sa.Column('key', sa.String(length=64), nullable=False), - sa.Column('value', sa.Text(), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.PrimaryKeyConstraint('key', name=op.f('pk_app_setting')) - ) - op.create_table('artist', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=255), nullable=False), - sa.Column('slug', sa.String(length=255), nullable=False), - sa.Column('notes', sa.Text(), nullable=True), - sa.Column('is_subscription', sa.Boolean(), nullable=False), - sa.Column('auto_check', sa.Boolean(), nullable=False), - sa.Column('check_interval_seconds', sa.Integer(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.PrimaryKeyConstraint('id', name=op.f('pk_artist')), - sa.UniqueConstraint('slug', name=op.f('uq_artist_slug')) - ) - op.create_table('backup_run', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('kind', sa.String(length=16), nullable=False), - sa.Column('status', sa.String(length=16), nullable=False), - sa.Column('tag', sa.String(length=64), nullable=True), - sa.Column('triggered_by', sa.String(length=32), nullable=False), - sa.Column('started_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('sql_path', sa.Text(), nullable=True), - sa.Column('tar_path', sa.Text(), nullable=True), - sa.Column('size_bytes', sa.BigInteger(), nullable=True), - sa.Column('error', sa.Text(), nullable=True), - sa.Column('manifest', sa.JSON(), server_default='{}', nullable=False), - sa.Column('restored_from_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['restored_from_id'], ['backup_run.id'], name=op.f('fk_backup_run_restored_from_id_backup_run'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_backup_run')) - ) - op.create_index(op.f('ix_backup_run_finished_at'), 'backup_run', ['finished_at'], unique=False) - op.create_index(op.f('ix_backup_run_kind'), 'backup_run', ['kind'], unique=False) - op.create_index(op.f('ix_backup_run_started_at'), 'backup_run', ['started_at'], unique=False) - op.create_index(op.f('ix_backup_run_status'), 'backup_run', ['status'], unique=False) - op.create_index(op.f('ix_backup_run_tag'), 'backup_run', ['tag'], unique=False) - op.create_table('credential', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('platform', sa.String(length=64), nullable=False), - sa.Column('credential_type', sa.String(length=32), nullable=False), - sa.Column('encrypted_blob', sa.LargeBinary(), nullable=False), - sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('expires_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('last_verified', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_credential')), - sa.UniqueConstraint('platform', name=op.f('uq_credential_platform')) - ) - op.create_table('head_auto_apply_run', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('dry_run', sa.Boolean(), nullable=False), - sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column('status', sa.String(length=16), nullable=False), - sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('n_applied', sa.Integer(), nullable=True), - sa.Column('report', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('error', sa.Text(), nullable=True), - sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_head_auto_apply_run')) - ) - op.create_index(op.f('ix_head_auto_apply_run_status'), 'head_auto_apply_run', ['status'], unique=False) - op.create_table('head_training_run', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column('status', sa.String(length=16), nullable=False), - sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('n_trained', sa.Integer(), nullable=True), - sa.Column('n_skipped', sa.Integer(), nullable=True), - sa.Column('error', sa.Text(), nullable=True), - sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_head_training_run')) - ) - op.create_index(op.f('ix_head_training_run_status'), 'head_training_run', ['status'], unique=False) - op.create_table('import_batch', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('triggered_by', sa.String(length=32), nullable=False), - sa.Column('source_path', sa.Text(), nullable=False), - sa.Column('scan_mode', sa.String(length=16), nullable=False), - sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('total_files', sa.Integer(), nullable=False), - sa.Column('imported', sa.Integer(), nullable=False), - sa.Column('skipped', sa.Integer(), nullable=False), - sa.Column('failed', sa.Integer(), nullable=False), - sa.Column('attachments', sa.Integer(), nullable=False), - sa.Column('refreshed', sa.Integer(), nullable=False), - sa.Column('status', sa.String(length=16), nullable=False), - sa.PrimaryKeyConstraint('id', name=op.f('pk_import_batch')) - ) - op.create_index(op.f('ix_import_batch_status'), 'import_batch', ['status'], unique=False) - op.create_table('import_settings', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('import_scan_path', sa.Text(), nullable=False), - sa.Column('min_width', sa.Integer(), nullable=False), - sa.Column('min_height', sa.Integer(), nullable=False), - sa.Column('skip_transparent', sa.Boolean(), nullable=False), - sa.Column('transparency_threshold', sa.Float(), nullable=False), - sa.Column('skip_single_color', sa.Boolean(), nullable=False), - sa.Column('single_color_threshold', sa.Float(), nullable=False), - sa.Column('single_color_tolerance', sa.Integer(), nullable=False), - sa.Column('phash_threshold', sa.Integer(), nullable=False), - sa.Column('download_rate_limit_seconds', sa.Float(), nullable=False), - sa.Column('download_validate_files', sa.Boolean(), nullable=False), - sa.Column('download_schedule_default_seconds', sa.Integer(), nullable=False), - sa.Column('download_event_retention_days', sa.Integer(), nullable=False), - sa.Column('download_failure_warning_threshold', sa.Integer(), nullable=False), - sa.Column('backup_db_nightly_enabled', sa.Boolean(), nullable=False), - sa.Column('backup_db_nightly_hour_utc', sa.Integer(), nullable=False), - sa.Column('backup_db_keep_last_n', sa.Integer(), nullable=False), - sa.Column('backup_images_keep_last_n', sa.Integer(), nullable=False), - sa.Column('series_suggest_enabled', sa.Boolean(), nullable=False), - sa.Column('series_suggest_threshold', sa.Float(), nullable=False), - sa.Column('extdl_mega_enabled', sa.Boolean(), server_default='true', nullable=False), - sa.Column('extdl_gdrive_enabled', sa.Boolean(), server_default='true', nullable=False), - sa.Column('extdl_mediafire_enabled', sa.Boolean(), server_default='true', nullable=False), - sa.Column('extdl_dropbox_enabled', sa.Boolean(), server_default='true', nullable=False), - sa.Column('extdl_pixeldrain_enabled', sa.Boolean(), server_default='true', nullable=False), - sa.Column('translation_enabled', sa.Boolean(), server_default='false', nullable=False), - sa.Column('interpreter_base_url', sa.Text(), server_default='', nullable=False), - sa.Column('translation_target_lang', sa.Text(), server_default='en', nullable=False), - sa.Column('translation_min_confidence', sa.Float(), server_default='0.9', nullable=False), - sa.Column('wip_title_tagging_enabled', sa.Boolean(), server_default='true', nullable=False), - sa.Column('wip_soft_title_tagging_enabled', sa.Boolean(), server_default='false', nullable=False), - sa.CheckConstraint('id = 1', name=op.f('ck_import_settings_singleton')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_import_settings')) - ) - op.create_table('library_audit_run', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('rule', sa.String(length=32), nullable=False), - sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column('status', sa.String(length=16), nullable=False), - sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('scanned_count', sa.Integer(), nullable=False), - sa.Column('matched_count', sa.Integer(), nullable=False), - sa.Column('matched_ids', postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column('error', sa.Text(), nullable=True), - sa.Column('resume_after_id', sa.Integer(), nullable=False), - sa.Column('last_progress_at', sa.DateTime(timezone=True), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_library_audit_run')) - ) - op.create_index(op.f('ix_library_audit_run_rule'), 'library_audit_run', ['rule'], unique=False) - op.create_index(op.f('ix_library_audit_run_status'), 'library_audit_run', ['status'], unique=False) - op.create_table('ml_settings', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('cpu_embed_enabled', sa.Boolean(), nullable=False), - sa.Column('video_frame_interval_seconds', sa.Float(), nullable=False), - sa.Column('video_max_frames', sa.Integer(), nullable=False), - sa.Column('head_min_positives', sa.Integer(), nullable=False), - sa.Column('head_auto_apply_precision', sa.Float(), nullable=False), - sa.Column('head_auto_apply_enabled', sa.Boolean(), nullable=False), - sa.Column('head_auto_apply_min_positives', sa.Integer(), nullable=False), - sa.Column('ccip_match_threshold', sa.Float(), nullable=False), - sa.Column('ccip_auto_apply_enabled', sa.Boolean(), nullable=False), - sa.Column('ccip_auto_apply_threshold', sa.Float(), nullable=False), - sa.Column('presentation_auto_apply_enabled', sa.Boolean(), nullable=False), - sa.Column('presentation_auto_apply_threshold', sa.Float(), nullable=False), - sa.Column('presentation_conflict_threshold', sa.Float(), nullable=False), - sa.Column('process_auto_apply_enabled', sa.Boolean(), nullable=False), - sa.Column('process_auto_apply_threshold', sa.Float(), nullable=False), - sa.Column('process_conflict_threshold', sa.Float(), nullable=False), - sa.Column('embedder_model_version', sa.String(length=128), nullable=False), - sa.Column('embedder_model_name', sa.String(length=128), nullable=False), - sa.Column('detector_person_enabled', sa.Boolean(), nullable=False), - sa.Column('detector_person_weights', sa.String(length=512), nullable=False), - sa.Column('detector_person_conf', sa.Float(), nullable=False), - sa.Column('detector_anatomy_enabled', sa.Boolean(), nullable=False), - sa.Column('detector_anatomy_weights', sa.String(length=512), nullable=False), - sa.Column('detector_anatomy_conf', sa.Float(), nullable=False), - sa.Column('detector_panel_enabled', sa.Boolean(), nullable=False), - sa.Column('detector_panel_weights', sa.String(length=512), nullable=False), - sa.Column('detector_panel_conf', sa.Float(), nullable=False), - sa.Column('detector_max_figures', sa.Integer(), nullable=False), - sa.Column('detector_max_components', sa.Integer(), nullable=False), - sa.Column('detector_max_panels', sa.Integer(), nullable=False), - sa.Column('detector_max_regions', sa.Integer(), nullable=False), - sa.Column('detector_dedupe_iou', sa.Float(), nullable=False), - sa.Column('ccip_ref_signature', sa.String(length=128), nullable=True), - sa.Column('ccip_prototype_cap', sa.Integer(), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.CheckConstraint('id = 1', name=op.f('ck_ml_settings_singleton')), - sa.PrimaryKeyConstraint('id', name=op.f('pk_ml_settings')) - ) - op.create_table('tag', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=255), nullable=False), - sa.Column('kind', sa.Enum('artist', 'character', 'fandom', 'general', 'series', 'archive', 'post', name='tag_kind'), nullable=False), - sa.Column('fandom_id', sa.Integer(), nullable=True), - sa.Column('is_system', sa.Boolean(), server_default=sa.text('false'), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.CheckConstraint("(fandom_id IS NULL) OR (kind = 'character')", name=op.f('ck_tag_ck_tag_fandom_requires_character')), - sa.ForeignKeyConstraint(['fandom_id'], ['tag.id'], name=op.f('fk_tag_fandom_id_tag'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_tag')) - ) - op.create_index(op.f('ix_tag_fandom_id'), 'tag', ['fandom_id'], unique=False) - op.create_table('task_run', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('celery_task_id', sa.String(length=64), nullable=False), - sa.Column('queue', sa.String(length=32), nullable=False), - sa.Column('task_name', sa.String(length=128), nullable=False), - sa.Column('target_id', sa.Integer(), nullable=True), - sa.Column('started_at', sa.DateTime(timezone=True), nullable=False), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('duration_ms', sa.Integer(), nullable=True), - sa.Column('status', sa.String(length=16), nullable=False), - sa.Column('error_type', sa.String(length=128), nullable=True), - sa.Column('error_message', sa.Text(), nullable=True), - sa.Column('retry_count', sa.Integer(), nullable=True), - sa.Column('worker_hostname', sa.String(length=128), nullable=True), - sa.Column('args_summary', sa.String(length=255), nullable=True), - sa.PrimaryKeyConstraint('id', name=op.f('pk_task_run')) - ) - op.create_index(op.f('ix_task_run_celery_task_id'), 'task_run', ['celery_task_id'], unique=False) - op.create_index(op.f('ix_task_run_finished_at'), 'task_run', ['finished_at'], unique=False) - op.create_index(op.f('ix_task_run_queue'), 'task_run', ['queue'], unique=False) - op.create_index(op.f('ix_task_run_started_at'), 'task_run', ['started_at'], unique=False) - op.create_index(op.f('ix_task_run_status'), 'task_run', ['status'], unique=False) - op.create_index(op.f('ix_task_run_task_name'), 'task_run', ['task_name'], unique=False) - op.create_table('artist_visit', - sa.Column('artist_id', sa.Integer(), nullable=False), - sa.Column('last_viewed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_artist_visit_artist_id_artist'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('artist_id', name=op.f('pk_artist_visit')) - ) - op.create_table('ccip_prototype_state', - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('fingerprint', sa.String(length=64), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_ccip_prototype_state_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_ccip_prototype_state')) - ) - op.create_table('head_metric', - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('n_misfires', sa.Integer(), nullable=False), - sa.Column('n_underfires', sa.Integer(), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metric_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_head_metric')) - ) - op.create_table('head_metrics_snapshot', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('name', sa.String(length=255), nullable=False), - sa.Column('snapshot_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('n_auto_applied', sa.Integer(), nullable=False), - sa.Column('n_misfires', sa.Integer(), nullable=False), - sa.Column('n_underfires', sa.Integer(), nullable=False), - sa.Column('ap', sa.Float(), nullable=True), - sa.Column('precision_cv', sa.Float(), nullable=True), - sa.Column('recall', sa.Float(), nullable=True), - sa.Column('n_pos', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_head_metrics_snapshot_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_head_metrics_snapshot')) - ) - op.create_index(op.f('ix_head_metrics_snapshot_snapshot_at'), 'head_metrics_snapshot', ['snapshot_at'], unique=False) - op.create_index(op.f('ix_head_metrics_snapshot_tag_id'), 'head_metrics_snapshot', ['tag_id'], unique=False) - op.create_table('source', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('artist_id', sa.Integer(), nullable=False), - sa.Column('platform', sa.String(length=64), nullable=False), - sa.Column('url', sa.Text(), nullable=False), - sa.Column('enabled', sa.Boolean(), nullable=False), - sa.Column('config_overrides', sa.JSON(), nullable=True), - sa.Column('last_checked_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('last_error', sa.Text(), nullable=True), - sa.Column('error_type', sa.String(length=32), nullable=True), - sa.Column('check_interval_override', sa.Integer(), nullable=True), - sa.Column('consecutive_failures', sa.Integer(), nullable=False), - sa.Column('backfill_runs_remaining', sa.Integer(), server_default='0', nullable=False), - sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_source_artist_id_artist'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_source')) - ) - op.create_index(op.f('ix_source_artist_id'), 'source', ['artist_id'], unique=False) - op.create_index(op.f('ix_source_error_type'), 'source', ['error_type'], unique=False) - op.create_table('tag_alias', - sa.Column('alias_string', sa.String(length=255), nullable=False), - sa.Column('alias_category', sa.String(length=32), nullable=False), - sa.Column('canonical_tag_id', sa.Integer(), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['canonical_tag_id'], ['tag.id'], name=op.f('fk_tag_alias_canonical_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('alias_string', 'alias_category', name=op.f('pk_tag_alias')) - ) - op.create_index(op.f('ix_tag_alias_canonical_tag_id'), 'tag_alias', ['canonical_tag_id'], unique=False) - op.create_table('tag_head', - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('embedding_version', sa.String(length=128), nullable=False), - sa.Column('weights', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=False), - sa.Column('bias', sa.Float(), nullable=False), - sa.Column('suggest_threshold', sa.Float(), nullable=False), - sa.Column('auto_apply_threshold', sa.Float(), nullable=True), - sa.Column('n_pos', sa.Integer(), nullable=False), - sa.Column('n_neg', sa.Integer(), nullable=False), - sa.Column('ap', sa.Float(), nullable=False), - sa.Column('precision_cv', sa.Float(), nullable=False), - sa.Column('recall', sa.Float(), nullable=False), - sa.Column('trained_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('train_fingerprint', sa.String(length=128), nullable=True), - sa.Column('metrics', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_head_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('tag_id', name=op.f('pk_tag_head')) - ) - op.create_table('patreon_failed_media', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=False), - sa.Column('filehash', sa.String(length=128), nullable=False), - sa.Column('attempts', sa.Integer(), nullable=False), - sa.Column('last_error', sa.Text(), nullable=True), - sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_patreon_failed_media_source_id_source'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_patreon_failed_media')), - sa.UniqueConstraint('source_id', 'filehash', name='uq_patreon_failed_media_source_id') - ) - op.create_index(op.f('ix_patreon_failed_media_source_id'), 'patreon_failed_media', ['source_id'], unique=False) - op.create_table('patreon_seen_media', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=False), - sa.Column('filehash', sa.String(length=128), nullable=False), - sa.Column('post_id', sa.String(length=64), nullable=True), - sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_patreon_seen_media_source_id_source'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_patreon_seen_media')), - sa.UniqueConstraint('source_id', 'filehash', name='uq_patreon_seen_media_source_id') - ) - op.create_index(op.f('ix_patreon_seen_media_source_id'), 'patreon_seen_media', ['source_id'], unique=False) - op.create_table('pixiv_failed_media', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=False), - sa.Column('filehash', sa.String(length=128), nullable=False), - sa.Column('attempts', sa.Integer(), nullable=False), - sa.Column('last_error', sa.Text(), nullable=True), - sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_pixiv_failed_media_source_id_source'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_pixiv_failed_media')), - sa.UniqueConstraint('source_id', 'filehash', name='uq_pixiv_failed_media_source_id') - ) - op.create_index(op.f('ix_pixiv_failed_media_source_id'), 'pixiv_failed_media', ['source_id'], unique=False) - op.create_table('pixiv_seen_media', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=False), - sa.Column('filehash', sa.String(length=128), nullable=False), - sa.Column('post_id', sa.String(length=64), nullable=True), - sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_pixiv_seen_media_source_id_source'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_pixiv_seen_media')), - sa.UniqueConstraint('source_id', 'filehash', name='uq_pixiv_seen_media_source_id') - ) - op.create_index(op.f('ix_pixiv_seen_media_source_id'), 'pixiv_seen_media', ['source_id'], unique=False) - op.create_table('post', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=True), - sa.Column('artist_id', sa.Integer(), nullable=False), - sa.Column('external_post_id', sa.String(length=128), nullable=False), - sa.Column('post_url', sa.Text(), nullable=True), - sa.Column('post_title', sa.Text(), nullable=True), - sa.Column('post_date', sa.DateTime(timezone=True), nullable=True), - sa.Column('raw_metadata', sa.JSON(), nullable=True), - sa.Column('description', sa.Text(), nullable=True), - sa.Column('attachment_count', sa.Integer(), nullable=True), - sa.Column('post_title_translated', sa.Text(), nullable=True), - sa.Column('description_translated', sa.Text(), nullable=True), - sa.Column('translated_source_lang', sa.String(length=8), nullable=True), - sa.Column('translation_engine_version', sa.String(length=128), nullable=True), - sa.Column('translated_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('translation_override', sa.String(length=16), server_default='auto', nullable=False), - sa.Column('downloaded_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.CheckConstraint("translation_override IN ('auto', 'force', 'original')", name=op.f('ck_post_ck_post_translation_override')), - sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_artist_id_artist'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_post_source_id_source'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_post')), - sa.UniqueConstraint('source_id', 'external_post_id', name='uq_post_source_external_id') - ) - op.create_index(op.f('ix_post_artist_id'), 'post', ['artist_id'], unique=False) - op.create_index(op.f('ix_post_source_id'), 'post', ['source_id'], unique=False) - op.create_table('subscribestar_failed_media', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=False), - sa.Column('filehash', sa.String(length=128), nullable=False), - sa.Column('attempts', sa.Integer(), nullable=False), - sa.Column('last_error', sa.Text(), nullable=True), - sa.Column('first_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('last_failed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_subscribestar_failed_media_source_id_source'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_subscribestar_failed_media')), - sa.UniqueConstraint('source_id', 'filehash', name='uq_subscribestar_failed_media_source_id') - ) - op.create_index(op.f('ix_subscribestar_failed_media_source_id'), 'subscribestar_failed_media', ['source_id'], unique=False) - op.create_table('subscribestar_seen_media', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=False), - sa.Column('filehash', sa.String(length=128), nullable=False), - sa.Column('post_id', sa.String(length=64), nullable=True), - sa.Column('seen_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_subscribestar_seen_media_source_id_source'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_subscribestar_seen_media')), - sa.UniqueConstraint('source_id', 'filehash', name='uq_subscribestar_seen_media_source_id') - ) - op.create_index(op.f('ix_subscribestar_seen_media_source_id'), 'subscribestar_seen_media', ['source_id'], unique=False) - op.create_table('download_event', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=False), - sa.Column('post_id', sa.Integer(), nullable=True), - sa.Column('status', sa.String(length=32), nullable=False), - sa.Column('started_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('bytes_downloaded', sa.BigInteger(), nullable=False), - sa.Column('files_count', sa.Integer(), nullable=False), - sa.Column('error', sa.Text(), nullable=True), - sa.Column('metadata', postgresql.JSONB(astext_type=sa.Text()), server_default=sa.text("'{}'::jsonb"), nullable=False), - sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_download_event_post_id_post'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_download_event_source_id_source'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_download_event')) - ) - op.create_index(op.f('ix_download_event_post_id'), 'download_event', ['post_id'], unique=False) - op.create_index(op.f('ix_download_event_source_id'), 'download_event', ['source_id'], unique=False) - op.create_table('image_record', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('path', sa.Text(), nullable=False), - sa.Column('sha256', sa.String(length=64), nullable=False), - sa.Column('phash', sa.String(length=32), nullable=True), - sa.Column('size_bytes', sa.BigInteger(), nullable=False), - sa.Column('mime', sa.String(length=64), nullable=False), - sa.Column('width', sa.Integer(), nullable=True), - sa.Column('height', sa.Integer(), nullable=True), - sa.Column('duration_seconds', sa.Float(), nullable=True), - sa.Column('integrity_status', sa.String(length=24), nullable=False), - sa.Column('thumbnail_path', sa.Text(), nullable=True), - sa.Column('source_url', sa.Text(), nullable=True), - sa.Column('source_filehash', sa.String(length=32), nullable=True), - sa.Column('origin', sa.Enum('downloaded', 'imported_filesystem', 'uploaded', name='origin_enum'), nullable=False), - sa.Column('primary_post_id', sa.Integer(), nullable=True), - sa.Column('artist_id', sa.Integer(), nullable=True), - sa.Column('siglip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=True), - sa.Column('siglip_model_version', sa.String(length=128), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('effective_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('earliest_post_date', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_image_record_artist_id_artist'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['primary_post_id'], ['post.id'], name=op.f('fk_image_record_primary_post_id_post'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_image_record')), - sa.UniqueConstraint('path', name=op.f('uq_image_record_path')) - ) - op.create_index(op.f('ix_image_record_artist_id'), 'image_record', ['artist_id'], unique=False) - op.create_index(op.f('ix_image_record_integrity_status'), 'image_record', ['integrity_status'], unique=False) - op.create_index(op.f('ix_image_record_phash'), 'image_record', ['phash'], unique=False) - op.create_index(op.f('ix_image_record_primary_post_id'), 'image_record', ['primary_post_id'], unique=False) - op.create_index(op.f('ix_image_record_sha256'), 'image_record', ['sha256'], unique=True) - op.create_index(op.f('ix_image_record_source_filehash'), 'image_record', ['source_filehash'], unique=False) - op.create_table('post_attachment', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('post_id', sa.Integer(), nullable=True), - sa.Column('artist_id', sa.Integer(), nullable=True), - sa.Column('sha256', sa.String(length=64), nullable=False), - sa.Column('path', sa.Text(), nullable=False), - sa.Column('original_filename', sa.Text(), nullable=False), - sa.Column('ext', sa.String(length=32), nullable=False), - sa.Column('mime', sa.String(length=128), nullable=True), - sa.Column('size_bytes', sa.BigInteger(), nullable=False), - sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_post_attachment_artist_id_artist'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_post_attachment_post_id_post'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_post_attachment')) - ) - op.create_index(op.f('ix_post_attachment_artist_id'), 'post_attachment', ['artist_id'], unique=False) - op.create_index(op.f('ix_post_attachment_post_id'), 'post_attachment', ['post_id'], unique=False) - op.create_index(op.f('ix_post_attachment_sha256'), 'post_attachment', ['sha256'], unique=False) - op.create_index('uq_post_attachment_null_post_sha', 'post_attachment', ['sha256'], unique=True, postgresql_where=sa.text('post_id IS NULL')) - op.create_index('uq_post_attachment_post_sha', 'post_attachment', ['post_id', 'sha256'], unique=True, postgresql_where=sa.text('post_id IS NOT NULL')) - op.create_table('series_suggestion', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('post_id', sa.Integer(), nullable=False), - sa.Column('series_tag_id', sa.Integer(), nullable=False), - sa.Column('score', sa.Float(), nullable=False), - sa.Column('signals', sa.JSON(), nullable=True), - sa.Column('status', sa.String(length=16), server_default='pending', nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_series_suggestion_post_id_post'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_suggestion_series_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_series_suggestion')), - sa.UniqueConstraint('post_id', 'series_tag_id', name='uq_series_suggestion_post_series') - ) - op.create_index(op.f('ix_series_suggestion_post_id'), 'series_suggestion', ['post_id'], unique=False) - op.create_index(op.f('ix_series_suggestion_series_tag_id'), 'series_suggestion', ['series_tag_id'], unique=False) - op.create_index(op.f('ix_series_suggestion_status'), 'series_suggestion', ['status'], unique=False) - op.create_table('external_link', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('post_id', sa.Integer(), nullable=False), - sa.Column('artist_id', sa.Integer(), nullable=True), - sa.Column('host', sa.String(length=16), nullable=False), - sa.Column('url', sa.Text(), nullable=False), - sa.Column('label', sa.Text(), nullable=True), - sa.Column('status', sa.String(length=16), server_default='pending', nullable=False), - sa.Column('attempts', sa.Integer(), server_default=sa.text('0'), nullable=False), - sa.Column('last_error', sa.Text(), nullable=True), - sa.Column('attachment_id', sa.Integer(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('completed_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('duration_seconds', sa.Float(), nullable=True), - sa.ForeignKeyConstraint(['artist_id'], ['artist.id'], name=op.f('fk_external_link_artist_id_artist'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['attachment_id'], ['post_attachment.id'], name=op.f('fk_external_link_attachment_id_post_attachment'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_external_link_post_id_post'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_external_link')) - ) - op.create_index(op.f('ix_external_link_artist_id'), 'external_link', ['artist_id'], unique=False) - op.create_index(op.f('ix_external_link_post_id'), 'external_link', ['post_id'], unique=False) - op.create_index('ix_external_link_status', 'external_link', ['status'], unique=False) - op.create_index('uq_external_link_post_url', 'external_link', ['post_id', 'url'], unique=True) - op.create_table('gpu_job', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('image_record_id', sa.Integer(), nullable=False), - sa.Column('task', sa.String(length=32), nullable=False), - sa.Column('status', sa.String(length=16), nullable=False), - sa.Column('lease_token', sa.String(length=64), nullable=True), - sa.Column('leased_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('lease_expires_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('attempts', sa.Integer(), nullable=False), - sa.Column('error', sa.Text(), nullable=True), - sa.Column('triage_status', sa.String(length=16), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_gpu_job_image_record_id_image_record'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_gpu_job')) - ) - op.create_index(op.f('ix_gpu_job_image_record_id'), 'gpu_job', ['image_record_id'], unique=False) - op.create_index('ix_gpu_job_leased_expires', 'gpu_job', ['lease_expires_at'], unique=False, postgresql_where=sa.text("status = 'leased'")) - op.create_index('ix_gpu_job_pending', 'gpu_job', ['id'], unique=False, postgresql_where=sa.text("status = 'pending'")) - op.create_index(op.f('ix_gpu_job_status'), 'gpu_job', ['status'], unique=False) - op.create_table('image_provenance', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('image_record_id', sa.Integer(), nullable=False), - sa.Column('post_id', sa.Integer(), nullable=False), - sa.Column('source_id', sa.Integer(), nullable=True), - sa.Column('from_attachment_id', sa.Integer(), nullable=True), - sa.Column('captured_metadata', sa.JSON(), nullable=True), - sa.Column('captured_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['from_attachment_id'], ['post_attachment.id'], name=op.f('fk_image_provenance_from_attachment_id_post_attachment'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_provenance_image_record_id_image_record'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['post_id'], ['post.id'], name=op.f('fk_image_provenance_post_id_post'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['source_id'], ['source.id'], name=op.f('fk_image_provenance_source_id_source'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_image_provenance')), - sa.UniqueConstraint('image_record_id', 'post_id', name='uq_image_provenance_image_post') - ) - op.create_index(op.f('ix_image_provenance_from_attachment_id'), 'image_provenance', ['from_attachment_id'], unique=False) - op.create_index(op.f('ix_image_provenance_image_record_id'), 'image_provenance', ['image_record_id'], unique=False) - op.create_index(op.f('ix_image_provenance_post_id'), 'image_provenance', ['post_id'], unique=False) - op.create_index(op.f('ix_image_provenance_source_id'), 'image_provenance', ['source_id'], unique=False) - op.create_table('image_region', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('image_record_id', sa.Integer(), nullable=False), - sa.Column('kind', sa.String(length=16), nullable=False), - sa.Column('frame_time', sa.Float(), nullable=True), - sa.Column('rx', sa.Float(), nullable=False), - sa.Column('ry', sa.Float(), nullable=False), - sa.Column('rw', sa.Float(), nullable=False), - sa.Column('rh', sa.Float(), nullable=False), - sa.Column('score', sa.Float(), nullable=True), - sa.Column('detector_version', sa.String(length=64), nullable=True), - sa.Column('crop_version', sa.String(length=64), nullable=True), - sa.Column('embedding_version', sa.String(length=128), nullable=True), - sa.Column('ccip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=768), nullable=True), - sa.Column('siglip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=1152), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_region_image_record_id_image_record'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_image_region')) - ) - op.create_index(op.f('ix_image_region_image_record_id'), 'image_region', ['image_record_id'], unique=False) - op.create_table('image_tag', - sa.Column('image_record_id', sa.Integer(), nullable=False), - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('source', sa.String(length=32), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_image_tag_image_record_id_image_record'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_image_tag_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_image_tag')) - ) - op.create_table('import_task', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('batch_id', sa.Integer(), nullable=False), - sa.Column('source_path', sa.Text(), nullable=False), - sa.Column('task_type', sa.String(length=16), nullable=False), - sa.Column('status', sa.String(length=16), nullable=False), - sa.Column('recovery_count', sa.Integer(), nullable=False), - sa.Column('refetched', sa.Boolean(), nullable=False), - sa.Column('result_image_id', sa.Integer(), nullable=True), - sa.Column('error', sa.Text(), nullable=True), - sa.Column('size_bytes', sa.BigInteger(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('started_at', sa.DateTime(timezone=True), nullable=True), - sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['batch_id'], ['import_batch.id'], name=op.f('fk_import_task_batch_id_import_batch'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['result_image_id'], ['image_record.id'], name=op.f('fk_import_task_result_image_id_image_record'), ondelete='SET NULL'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_import_task')) - ) - op.create_index(op.f('ix_import_task_batch_id'), 'import_task', ['batch_id'], unique=False) - op.create_index(op.f('ix_import_task_status'), 'import_task', ['status'], unique=False) - op.create_table('presentation_review', - sa.Column('image_record_id', sa.Integer(), nullable=False), - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('conflict_tag_id', sa.Integer(), nullable=True), - sa.Column('conflict_score', sa.Float(), nullable=False), - sa.Column('mode', sa.String(length=16), server_default='chrome', nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('resolved_at', sa.DateTime(timezone=True), nullable=True), - sa.ForeignKeyConstraint(['conflict_tag_id'], ['tag.id'], name=op.f('fk_presentation_review_conflict_tag_id_tag'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_presentation_review_image_record_id_image_record'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_presentation_review_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_presentation_review')) - ) - op.create_table('series_page', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('series_tag_id', sa.Integer(), nullable=False), - sa.Column('image_id', sa.Integer(), nullable=False), - sa.Column('status', sa.String(length=16), server_default='placed', nullable=False), - sa.Column('page_number', sa.Integer(), nullable=True), - sa.Column('stated_page', sa.Integer(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['image_id'], ['image_record.id'], name=op.f('fk_series_page_image_id_image_record'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_page_series_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_series_page')), - sa.UniqueConstraint('image_id', name=op.f('uq_series_page_image_id')) - ) - op.create_index(op.f('ix_series_page_series_tag_id'), 'series_page', ['series_tag_id'], unique=False) - op.create_table('tag_positive_confirmation', - sa.Column('image_record_id', sa.Integer(), nullable=False), - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('confirmed_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_tag_positive_confirmation_image_record_id_image_record'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_positive_confirmation_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_positive_confirmation')) - ) - op.create_index(op.f('ix_tag_positive_confirmation_tag_id'), 'tag_positive_confirmation', ['tag_id'], unique=False) - op.create_table('tag_suggestion_rejection', - sa.Column('image_record_id', sa.Integer(), nullable=False), - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('rejected_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['image_record_id'], ['image_record.id'], name=op.f('fk_tag_suggestion_rejection_image_record_id_image_record'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_tag_suggestion_rejection_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('image_record_id', 'tag_id', name=op.f('pk_tag_suggestion_rejection')) - ) - op.create_index(op.f('ix_tag_suggestion_rejection_tag_id'), 'tag_suggestion_rejection', ['tag_id'], unique=False) - op.create_table('character_prototype', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('tag_id', sa.Integer(), nullable=False), - sa.Column('ccip_embedding', pgvector.sqlalchemy.vector.VECTOR(dim=768), nullable=False), - sa.Column('region_id', sa.Integer(), nullable=True), - sa.ForeignKeyConstraint(['region_id'], ['image_region.id'], name=op.f('fk_character_prototype_region_id_image_region'), ondelete='SET NULL'), - sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name=op.f('fk_character_prototype_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_character_prototype')) - ) - op.create_index(op.f('ix_character_prototype_tag_id'), 'character_prototype', ['tag_id'], unique=False) - op.create_table('series_chapter', - sa.Column('id', sa.Integer(), nullable=False), - sa.Column('series_tag_id', sa.Integer(), nullable=False), - sa.Column('anchor_page_id', sa.Integer(), nullable=False), - sa.Column('title', sa.Text(), nullable=True), - sa.Column('stated_part', sa.Integer(), nullable=True), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False), - sa.ForeignKeyConstraint(['anchor_page_id'], ['series_page.id'], name=op.f('fk_series_chapter_anchor_page_id_series_page'), ondelete='CASCADE'), - sa.ForeignKeyConstraint(['series_tag_id'], ['tag.id'], name=op.f('fk_series_chapter_series_tag_id_tag'), ondelete='CASCADE'), - sa.PrimaryKeyConstraint('id', name=op.f('pk_series_chapter')), - sa.UniqueConstraint('anchor_page_id', name=op.f('uq_series_chapter_anchor_page_id')) - ) - op.create_index(op.f('ix_series_chapter_series_tag_id'), 'series_chapter', ['series_tag_id'], unique=False) - - # The HNSW index, item 3 above. Must match the query's cosine-distance - # operator class or the planner will not use it. - op.execute( - "CREATE INDEX ix_image_record_siglip_hnsw " - "ON image_record USING hnsw (siglip_embedding vector_cosine_ops)" - ) - - -def downgrade() -> None: - # Dropping image_record takes its indexes with it, so the HNSW index needs - # no separate drop. The extensions are deliberately left in place: they are - # database-scoped and something else may be using them. - op.drop_index(op.f('ix_series_chapter_series_tag_id'), table_name='series_chapter') - op.drop_table('series_chapter') - op.drop_index(op.f('ix_character_prototype_tag_id'), table_name='character_prototype') - op.drop_table('character_prototype') - op.drop_index(op.f('ix_tag_suggestion_rejection_tag_id'), table_name='tag_suggestion_rejection') - op.drop_table('tag_suggestion_rejection') - op.drop_index(op.f('ix_tag_positive_confirmation_tag_id'), table_name='tag_positive_confirmation') - op.drop_table('tag_positive_confirmation') - op.drop_index(op.f('ix_series_page_series_tag_id'), table_name='series_page') - op.drop_table('series_page') - op.drop_table('presentation_review') - op.drop_index(op.f('ix_import_task_status'), table_name='import_task') - op.drop_index(op.f('ix_import_task_batch_id'), table_name='import_task') - op.drop_table('import_task') - op.drop_table('image_tag') - op.drop_index(op.f('ix_image_region_image_record_id'), table_name='image_region') - op.drop_table('image_region') - op.drop_index(op.f('ix_image_provenance_source_id'), table_name='image_provenance') - op.drop_index(op.f('ix_image_provenance_post_id'), table_name='image_provenance') - op.drop_index(op.f('ix_image_provenance_image_record_id'), table_name='image_provenance') - op.drop_index(op.f('ix_image_provenance_from_attachment_id'), table_name='image_provenance') - op.drop_table('image_provenance') - op.drop_index(op.f('ix_gpu_job_status'), table_name='gpu_job') - op.drop_index('ix_gpu_job_pending', table_name='gpu_job', postgresql_where=sa.text("status = 'pending'")) - op.drop_index('ix_gpu_job_leased_expires', table_name='gpu_job', postgresql_where=sa.text("status = 'leased'")) - op.drop_index(op.f('ix_gpu_job_image_record_id'), table_name='gpu_job') - op.drop_table('gpu_job') - op.drop_index('uq_external_link_post_url', table_name='external_link') - op.drop_index('ix_external_link_status', table_name='external_link') - op.drop_index(op.f('ix_external_link_post_id'), table_name='external_link') - op.drop_index(op.f('ix_external_link_artist_id'), table_name='external_link') - op.drop_table('external_link') - op.drop_index(op.f('ix_series_suggestion_status'), table_name='series_suggestion') - op.drop_index(op.f('ix_series_suggestion_series_tag_id'), table_name='series_suggestion') - op.drop_index(op.f('ix_series_suggestion_post_id'), table_name='series_suggestion') - op.drop_table('series_suggestion') - op.drop_index('uq_post_attachment_post_sha', table_name='post_attachment', postgresql_where=sa.text('post_id IS NOT NULL')) - op.drop_index('uq_post_attachment_null_post_sha', table_name='post_attachment', postgresql_where=sa.text('post_id IS NULL')) - op.drop_index(op.f('ix_post_attachment_sha256'), table_name='post_attachment') - op.drop_index(op.f('ix_post_attachment_post_id'), table_name='post_attachment') - op.drop_index(op.f('ix_post_attachment_artist_id'), table_name='post_attachment') - op.drop_table('post_attachment') - op.drop_index(op.f('ix_image_record_source_filehash'), table_name='image_record') - op.drop_index(op.f('ix_image_record_sha256'), table_name='image_record') - op.drop_index(op.f('ix_image_record_primary_post_id'), table_name='image_record') - op.drop_index(op.f('ix_image_record_phash'), table_name='image_record') - op.drop_index(op.f('ix_image_record_integrity_status'), table_name='image_record') - op.drop_index(op.f('ix_image_record_artist_id'), table_name='image_record') - op.drop_table('image_record') - op.drop_index(op.f('ix_download_event_source_id'), table_name='download_event') - op.drop_index(op.f('ix_download_event_post_id'), table_name='download_event') - op.drop_table('download_event') - op.drop_index(op.f('ix_subscribestar_seen_media_source_id'), table_name='subscribestar_seen_media') - op.drop_table('subscribestar_seen_media') - op.drop_index(op.f('ix_subscribestar_failed_media_source_id'), table_name='subscribestar_failed_media') - op.drop_table('subscribestar_failed_media') - op.drop_index(op.f('ix_post_source_id'), table_name='post') - op.drop_index(op.f('ix_post_artist_id'), table_name='post') - op.drop_table('post') - op.drop_index(op.f('ix_pixiv_seen_media_source_id'), table_name='pixiv_seen_media') - op.drop_table('pixiv_seen_media') - op.drop_index(op.f('ix_pixiv_failed_media_source_id'), table_name='pixiv_failed_media') - op.drop_table('pixiv_failed_media') - op.drop_index(op.f('ix_patreon_seen_media_source_id'), table_name='patreon_seen_media') - op.drop_table('patreon_seen_media') - op.drop_index(op.f('ix_patreon_failed_media_source_id'), table_name='patreon_failed_media') - op.drop_table('patreon_failed_media') - op.drop_table('tag_head') - op.drop_index(op.f('ix_tag_alias_canonical_tag_id'), table_name='tag_alias') - op.drop_table('tag_alias') - op.drop_index(op.f('ix_source_error_type'), table_name='source') - op.drop_index(op.f('ix_source_artist_id'), table_name='source') - op.drop_table('source') - op.drop_index(op.f('ix_head_metrics_snapshot_tag_id'), table_name='head_metrics_snapshot') - op.drop_index(op.f('ix_head_metrics_snapshot_snapshot_at'), table_name='head_metrics_snapshot') - op.drop_table('head_metrics_snapshot') - op.drop_table('head_metric') - op.drop_table('ccip_prototype_state') - op.drop_table('artist_visit') - op.drop_index(op.f('ix_task_run_task_name'), table_name='task_run') - op.drop_index(op.f('ix_task_run_status'), table_name='task_run') - op.drop_index(op.f('ix_task_run_started_at'), table_name='task_run') - op.drop_index(op.f('ix_task_run_queue'), table_name='task_run') - op.drop_index(op.f('ix_task_run_finished_at'), table_name='task_run') - op.drop_index(op.f('ix_task_run_celery_task_id'), table_name='task_run') - op.drop_table('task_run') - op.drop_index(op.f('ix_tag_fandom_id'), table_name='tag') - op.drop_table('tag') - op.drop_table('ml_settings') - op.drop_index(op.f('ix_library_audit_run_status'), table_name='library_audit_run') - op.drop_index(op.f('ix_library_audit_run_rule'), table_name='library_audit_run') - op.drop_table('library_audit_run') - op.drop_table('import_settings') - op.drop_index(op.f('ix_import_batch_status'), table_name='import_batch') - op.drop_table('import_batch') - op.drop_index(op.f('ix_head_training_run_status'), table_name='head_training_run') - op.drop_table('head_training_run') - op.drop_index(op.f('ix_head_auto_apply_run_status'), table_name='head_auto_apply_run') - op.drop_table('head_auto_apply_run') - op.drop_table('credential') - op.drop_index(op.f('ix_backup_run_tag'), table_name='backup_run') - op.drop_index(op.f('ix_backup_run_status'), table_name='backup_run') - op.drop_index(op.f('ix_backup_run_started_at'), table_name='backup_run') - op.drop_index(op.f('ix_backup_run_kind'), table_name='backup_run') - op.drop_index(op.f('ix_backup_run_finished_at'), table_name='backup_run') - op.drop_table('backup_run') - op.drop_table('artist') - op.drop_table('app_setting') diff --git a/alembic/versions/0087_wip_soft_title_tagging.py b/alembic/versions/0087_wip_soft_title_tagging.py new file mode 100644 index 0000000..58478e1 --- /dev/null +++ b/alembic/versions/0087_wip_soft_title_tagging.py @@ -0,0 +1,33 @@ +"""soft WIP title tier toggle (#1474) — ImportSettings.wip_soft_title_tagging_enabled + +The soft tier also tags sketch/doodle/scribble titles, but with a provisional source +that never trains the head. OFF by default (a lower-precision tier is opt-in). +server_default so the existing singleton row (id=1) fills cleanly. + +Revision ID: 0087 +Revises: 0086 +Create Date: 2026-07-13 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +revision: str = "0087" +down_revision: Union[str, None] = "0086" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column( + "import_settings", + sa.Column( + "wip_soft_title_tagging_enabled", sa.Boolean(), nullable=False, + server_default=sa.text("false"), + ), + ) + + +def downgrade() -> None: + op.drop_column("import_settings", "wip_soft_title_tagging_enabled") diff --git a/backend/app/utils/artist_backfill.py b/backend/app/utils/artist_backfill.py new file mode 100644 index 0000000..3979b24 --- /dev/null +++ b/backend/app/utils/artist_backfill.py @@ -0,0 +1,44 @@ +"""Literal SQL for the FC-2d-vii-c artist backfill / artist-tag delete. + +Intentionally pure string constants — NO model/slug imports, NO logic — +so migration 0008 and its test share one drift-proof source of truth. +Backfill steps are ordered primary -> provenance -> artist-tag and each +only touches rows still NULL (idempotent, first match wins). The +artist-tag step matches Artist.name = Tag.name: the importer always +created both from the same artist_name string. +""" + +BACKFILL_PRIMARY_SQL = """ +UPDATE image_record AS ir +SET artist_id = s.artist_id +FROM post p +JOIN source s ON s.id = p.source_id +WHERE ir.primary_post_id = p.id + AND ir.artist_id IS NULL +""" + +BACKFILL_PROVENANCE_SQL = """ +UPDATE image_record AS ir +SET artist_id = s.artist_id +FROM ( + SELECT DISTINCT ON (ip.image_record_id) + ip.image_record_id, src.artist_id + FROM image_provenance ip + JOIN source src ON src.id = ip.source_id + ORDER BY ip.image_record_id, ip.id +) AS s +WHERE ir.id = s.image_record_id + AND ir.artist_id IS NULL +""" + +BACKFILL_TAG_SQL = """ +UPDATE image_record AS ir +SET artist_id = a.id +FROM image_tag it +JOIN tag t ON t.id = it.tag_id AND t.kind = 'artist' +JOIN artist a ON a.name = t.name +WHERE it.image_record_id = ir.id + AND ir.artist_id IS NULL +""" + +DELETE_ARTIST_TAGS_SQL = "DELETE FROM tag WHERE kind = 'artist'" diff --git a/tests/test_migration_0002.py b/tests/test_migration_0002.py new file mode 100644 index 0000000..1786731 --- /dev/null +++ b/tests/test_migration_0002.py @@ -0,0 +1,58 @@ +"""Smoke test for migration 0002: confirms model classes import and the +tag-kind uniqueness rule shape is correct. +""" + +from backend.app.models import ( + Base, + ImportBatch, + ImportSettings, + ImportTask, + Tag, + TagKind, +) + + +def test_new_tables_registered(): + expected = {"import_batch", "import_task", "import_settings"} + assert expected.issubset(Base.metadata.tables.keys()) + + +def test_tag_has_kind_and_fandom_id(): + cols = {c.name for c in Tag.__table__.columns} + assert "kind" in cols + assert "fandom_id" in cols + assert "namespace" not in cols + + +def test_tag_kind_enum_values(): + # Current TagKind enum after alembic 0023 dropped meta + rating + # (operator-retired 2026-05-26). `artist` is still in the enum + # for backward-compat with historical rows, though new artist + # tags don't get created (Artist row is canonical per FC-2d-vii-c). + expected = { + "artist", + "character", + "fandom", + "general", + "series", + "archive", + "post", + } + assert {k.value for k in TagKind} == expected + + +def test_image_record_has_integrity_status(): + from backend.app.models import ImageRecord + cols = {c.name for c in ImageRecord.__table__.columns} + assert "integrity_status" in cols + + +def test_import_task_has_state_columns(): + cols = {c.name for c in ImportTask.__table__.columns} + for required in ("batch_id", "source_path", "task_type", "status", "result_image_id"): + assert required in cols + + +def test_import_settings_singleton_constraint(): + constraints = {c.name for c in ImportSettings.__table__.constraints} + assert "ck_import_settings_singleton" in constraints diff --git a/tests/test_migration_0003.py b/tests/test_migration_0003.py new file mode 100644 index 0000000..1752120 --- /dev/null +++ b/tests/test_migration_0003.py @@ -0,0 +1,46 @@ +"""Smoke test for migration 0003: model classes import, schema shape correct.""" + +from backend.app.models import ( + Base, + ImageRecord, + MLSettings, + TagAlias, + TagSuggestionRejection, +) + + +def test_new_tables_registered(): + expected = { + "tag_suggestion_rejection", + "tag_alias", + "ml_settings", + } + assert expected.issubset(Base.metadata.tables.keys()) + + +def test_image_record_columns_renamed(): + cols = {c.name for c in ImageRecord.__table__.columns} + # Legacy tagger columns are all gone: tagger_predictions/wd14_* dropped in + # 0046, tagger_model_version + centroid_scores dropped in 0068 (#1199, Camie + # retirement). The SigLIP embedding columns are the live ML fields. + assert "siglip_embedding" in cols + assert "siglip_model_version" in cols + assert "tagger_model_version" not in cols + assert "centroid_scores" not in cols + assert "tagger_predictions" not in cols + assert "wd14_predictions" not in cols + + +def test_tag_alias_composite_pk(): + pk_cols = {c.name for c in TagAlias.__table__.primary_key.columns} + assert pk_cols == {"alias_string", "alias_category"} + + +def test_ml_settings_singleton_constraint(): + names = {c.name for c in MLSettings.__table__.constraints} + assert "ck_ml_settings_singleton" in names + + +def test_tag_suggestion_rejection_pk(): + pk_cols = {c.name for c in TagSuggestionRejection.__table__.primary_key.columns} + assert pk_cols == {"image_record_id", "tag_id"} diff --git a/tests/test_migration_0004.py b/tests/test_migration_0004.py new file mode 100644 index 0000000..7975d0c --- /dev/null +++ b/tests/test_migration_0004.py @@ -0,0 +1,27 @@ +"""Integration: the tsm_system_rows extension is installed by migration 0004. + +Needs a real Postgres (CI does not provision one), so integration-marked. +""" + +import pytest +from sqlalchemy import text + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_tsm_system_rows_extension_present(db): + row = ( + await db.execute( + text("SELECT 1 FROM pg_extension WHERE extname = 'tsm_system_rows'") + ) + ).first() + assert row is not None + + +@pytest.mark.asyncio +async def test_system_rows_sampling_is_usable(db): + # Should parse and execute even on an empty table. + await db.execute( + text("SELECT * FROM image_record TABLESAMPLE SYSTEM_ROWS(1)") + ) diff --git a/tests/test_migration_0007.py b/tests/test_migration_0007.py new file mode 100644 index 0000000..6f716f4 --- /dev/null +++ b/tests/test_migration_0007.py @@ -0,0 +1,48 @@ +"""FC-2d-iv: post.description + post.attachment_count round-trip.""" + +from datetime import UTC, datetime + +import pytest + +from backend.app.models import Artist, Post, Source + +pytestmark = pytest.mark.integration + + +async def _post(db, **post_kwargs): + artist = Artist(name="Nadia", slug="nadia") + db.add(artist) + await db.flush() + src = Source(artist_id=artist.id, platform="web", url="http://x") + db.add(src) + await db.flush() + post = Post( + source_id=src.id, artist_id=artist.id, external_post_id="p1", + post_date=datetime(2026, 3, 1, tzinfo=UTC), + **post_kwargs, + ) + db.add(post) + await db.flush() + return post.id + + +def test_post_has_new_columns(): + cols = {c.name for c in Post.__table__.columns} + assert "description" in cols + assert "attachment_count" in cols + + +@pytest.mark.asyncio +async def test_description_and_attachment_count_round_trip(db): + pid = await _post(db, description="

hi

", attachment_count=3) + row = await db.get(Post, pid) + assert row.description == "

hi

" + assert row.attachment_count == 3 + + +@pytest.mark.asyncio +async def test_new_fields_default_null(db): + pid = await _post(db) + row = await db.get(Post, pid) + assert row.description is None + assert row.attachment_count is None diff --git a/tests/test_migration_0008.py b/tests/test_migration_0008.py new file mode 100644 index 0000000..0d7552a --- /dev/null +++ b/tests/test_migration_0008.py @@ -0,0 +1,137 @@ +"""FC-2d-vii-c: image_record.artist_id + backfill + artist-tag delete.""" + +from datetime import UTC, datetime, timedelta + +import pytest +from sqlalchemy import func, select, text + +from backend.app.models import ( + Artist, + ImageProvenance, + ImageRecord, + Post, + Source, + Tag, + TagKind, +) +from backend.app.models.tag import image_tag +from backend.app.utils.artist_backfill import ( + BACKFILL_PRIMARY_SQL, + BACKFILL_PROVENANCE_SQL, + BACKFILL_TAG_SQL, + DELETE_ARTIST_TAGS_SQL, +) + +pytestmark = pytest.mark.integration + + +def test_image_record_has_artist_id_column(): + assert "artist_id" in {c.name for c in ImageRecord.__table__.columns} + + +async def _img(db, n): + rec = ImageRecord( + path=f"/images/bf/{n}.jpg", sha256=f"bf{n:062d}", + size_bytes=1, mime="image/jpeg", width=1, height=1, + origin="imported_filesystem", integrity_status="unknown", + ) + rec.created_at = datetime.now(UTC) - timedelta(minutes=n) + db.add(rec) + await db.flush() + return rec + + +async def _artist_source(db, name, slug): + a = Artist(name=name, slug=slug) + db.add(a) + await db.flush() + s = Source(artist_id=a.id, platform="patreon", + url=f"https://p.test/{slug}") + db.add(s) + await db.flush() + return a, s + + +async def _run_backfill(db): + await db.execute(text(BACKFILL_PRIMARY_SQL)) + await db.execute(text(BACKFILL_PROVENANCE_SQL)) + await db.execute(text(BACKFILL_TAG_SQL)) + + +@pytest.mark.asyncio +async def test_backfill_primary_post(db): + rec = await _img(db, 1) + a, s = await _artist_source(db, "Alice", "alice") + post = Post(source_id=s.id, artist_id=a.id, external_post_id="1") + db.add(post) + await db.flush() + rec.primary_post_id = post.id + await db.flush() + await _run_backfill(db) + got = await db.scalar( + select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) + ) + assert got == a.id + + +@pytest.mark.asyncio +async def test_backfill_provenance_fallback(db): + rec = await _img(db, 1) + a, s = await _artist_source(db, "Bob", "bob") + post = Post(source_id=s.id, artist_id=a.id, external_post_id="2") + db.add(post) + await db.flush() + db.add(ImageProvenance(image_record_id=rec.id, post_id=post.id, + source_id=s.id)) + await db.flush() + await _run_backfill(db) + got = await db.scalar( + select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) + ) + assert got == a.id + + +@pytest.mark.asyncio +async def test_backfill_artist_tag_by_name(db): + rec = await _img(db, 1) + a = Artist(name="Carol", slug="carol") + db.add(a) + await db.flush() + tag = Tag(name="Carol", kind=TagKind.artist) + db.add(tag) + await db.flush() + await db.execute(image_tag.insert().values( + image_record_id=rec.id, tag_id=tag.id, source="auto")) + await db.flush() + await _run_backfill(db) + got = await db.scalar( + select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) + ) + assert got == a.id + + +@pytest.mark.asyncio +async def test_no_signal_stays_null(db): + rec = await _img(db, 1) + await _run_backfill(db) + got = await db.scalar( + select(ImageRecord.artist_id).where(ImageRecord.id == rec.id) + ) + assert got is None + + +@pytest.mark.asyncio +async def test_delete_removes_only_artist_tags(db): + artist_tag = Tag(name="Dave", kind=TagKind.artist) + general_tag = Tag(name="forest", kind=TagKind.general) + db.add_all([artist_tag, general_tag]) + await db.flush() + await db.execute(text(DELETE_ARTIST_TAGS_SQL)) + remaining = await db.scalar( + select(func.count()).select_from(Tag).where(Tag.kind == TagKind.artist) + ) + assert remaining == 0 + survived = await db.scalar( + select(func.count()).select_from(Tag).where(Tag.kind == TagKind.general) + ) + assert survived >= 1 diff --git a/tests/test_migration_0009.py b/tests/test_migration_0009.py new file mode 100644 index 0000000..f2a0d1f --- /dev/null +++ b/tests/test_migration_0009.py @@ -0,0 +1,37 @@ +"""FC-2d-iii: post_attachment table + import_batch.attachments column.""" + +import pytest + +from backend.app.models import ImportBatch, PostAttachment + +pytestmark = pytest.mark.integration + + +def test_post_attachment_columns(): + cols = {c.name for c in PostAttachment.__table__.columns} + assert { + "id", "post_id", "artist_id", "sha256", "path", + "original_filename", "ext", "mime", "size_bytes", "captured_at", + } <= cols + + +def test_import_batch_has_attachments_counter(): + assert "attachments" in {c.name for c in ImportBatch.__table__.columns} + + +@pytest.mark.asyncio +async def test_post_attachment_roundtrip(db): + from backend.app.models import Artist + + a = Artist(name="Zed", slug="zed") + db.add(a) + await db.flush() + att = PostAttachment( + post_id=None, artist_id=a.id, sha256="z" + "0" * 63, + path="/images/attachments/z00/z.zip", original_filename="pack.zip", + ext=".zip", mime="application/zip", size_bytes=123, + ) + db.add(att) + await db.flush() + got = await db.get(PostAttachment, att.id) + assert got.original_filename == "pack.zip" and got.post_id is None diff --git a/tests/test_migration_0010.py b/tests/test_migration_0010.py new file mode 100644 index 0000000..a261e6c --- /dev/null +++ b/tests/test_migration_0010.py @@ -0,0 +1,35 @@ +import pytest +from sqlalchemy.exc import IntegrityError + +from backend.app.models import Artist, Source + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_duplicate_artist_platform_url_rejected(db): + artist = Artist(name="Alice", slug="alice") + db.add(artist) + await db.flush() + db.add(Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", enabled=True, + )) + await db.flush() + db.add(Source( + artist_id=artist.id, platform="patreon", + url="https://patreon.com/alice", enabled=True, + )) + with pytest.raises(IntegrityError): + await db.flush() + + +@pytest.mark.asyncio +async def test_same_url_under_different_artist_ok(db): + a = Artist(name="A", slug="a") + b = Artist(name="B", slug="b") + db.add_all([a, b]) + await db.flush() + db.add(Source(artist_id=a.id, platform="patreon", url="https://x/y", enabled=True)) + db.add(Source(artist_id=b.id, platform="patreon", url="https://x/y", enabled=True)) + await db.flush() # must NOT raise diff --git a/tests/test_migration_0011.py b/tests/test_migration_0011.py new file mode 100644 index 0000000..17e9702 --- /dev/null +++ b/tests/test_migration_0011.py @@ -0,0 +1,32 @@ +import pytest +from sqlalchemy import inspect, text + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_credential_has_credential_type_not_kind(db): + cols = (await db.run_sync( + lambda sync_session: [c["name"] for c in inspect(sync_session.bind).get_columns("credential")] + )) + assert "credential_type" in cols + assert "kind" not in cols + assert "status" not in cols + assert "last_verified" in cols + + +@pytest.mark.asyncio +async def test_credential_round_trip(db): + from backend.app.models import Credential + + db.add(Credential( + platform="patreon", + credential_type="cookies", + encrypted_blob=b"\x00\x01\x02", + )) + await db.flush() + row = (await db.execute( + text("SELECT credential_type, last_verified FROM credential WHERE platform='patreon'") + )).one() + assert row.credential_type == "cookies" + assert row.last_verified is None diff --git a/tests/test_migration_0012.py b/tests/test_migration_0012.py new file mode 100644 index 0000000..a993ab4 --- /dev/null +++ b/tests/test_migration_0012.py @@ -0,0 +1,32 @@ +import pytest +from sqlalchemy import select + +from backend.app.models import AppSetting + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_app_setting_table_round_trip(db): + db.add(AppSetting(key="extension_api_key", value="abc123")) + await db.flush() + row = (await db.execute( + select(AppSetting).where(AppSetting.key == "extension_api_key") + )).scalar_one() + assert row.value == "abc123" + assert row.updated_at is not None + + +@pytest.mark.asyncio +async def test_app_setting_upsert(db): + db.add(AppSetting(key="k", value="v1")) + await db.flush() + row = (await db.execute( + select(AppSetting).where(AppSetting.key == "k") + )).scalar_one() + row.value = "v2" + await db.flush() + again = (await db.execute( + select(AppSetting.value).where(AppSetting.key == "k") + )).scalar_one() + assert again == "v2" diff --git a/tests/test_migration_0013.py b/tests/test_migration_0013.py new file mode 100644 index 0000000..1a18384 --- /dev/null +++ b/tests/test_migration_0013.py @@ -0,0 +1,31 @@ +import pytest +from sqlalchemy import inspect, select + +from backend.app.models import ImportSettings + +pytestmark = pytest.mark.integration + + +@pytest.mark.asyncio +async def test_download_event_has_metadata(db): + cols = await db.run_sync( + lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("download_event")} + ) + assert "metadata" in cols + assert cols["metadata"]["nullable"] is False + + +@pytest.mark.asyncio +async def test_import_settings_has_downloader_fields(db): + cols = await db.run_sync( + lambda s: {c["name"]: c for c in inspect(s.bind).get_columns("import_settings")} + ) + assert "download_rate_limit_seconds" in cols + assert "download_validate_files" in cols + + +@pytest.mark.asyncio +async def test_import_settings_defaults(db): + row = (await db.execute(select(ImportSettings).where(ImportSettings.id == 1))).scalar_one() + assert row.download_rate_limit_seconds == 3.0 + assert row.download_validate_files is True -- 2.54.0 From 98b56330d0133dc48076d989112cb5ce681a6fd5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 14:35:02 -0400 Subject: [PATCH 09/17] ci: emit the chain schema dump for local reconciliation work (#3275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reconciling the models against the deployed schema needs the actual pg_dump, not an inference from the unified diff. Parsing table context out of diff hunks drops every table whose CREATE TABLE line falls outside a hunk — it under-reported 81 columns across 13 tables when the real figure spans more, missing artist, gpu_job, download_event and external_link entirely. Same checksummed-base64 transport as the candidate baseline, for the same reason: a plain cat of a file this size was silently truncated mid-line by the runner on run 4964. --- .forgejo/workflows/baseline.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index d7b4b66..e7df685 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -107,6 +107,23 @@ jobs: docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \ --no-owner --no-privileges -d fc_chain > chain.sql wc -l chain.sql + # Emit the dump itself, checksummed, for local analysis. Reconciling + # the models against the deployed schema (#3275) needs the ACTUAL + # schema, not an inference from a diff — parsing table context out of + # unified-diff hunks drops every table whose CREATE TABLE line falls + # outside a hunk, which silently under-reports. + # + # base64 + sha256 for the same reason as the candidate: a plain cat + # of a file this size was truncated mid-line by the runner with the + # step still green (run 4964). + set +x + B64=$(base64 -w 120 chain.sql) + echo "===== BEGIN CHAIN SCHEMA (base64) =====" + echo "$B64" + echo "===== END CHAIN SCHEMA =====" + echo "chain-sha256: $(sha256sum chain.sql | cut -d' ' -f1)" + echo "chain-bytes: $(wc -c < chain.sql)" + set -x # A candidate baseline, autogenerated from the models against an EMPTY # database so every table shows up as a create. Printed for a human to -- 2.54.0 From 5e1996e77f91d32cb05a137e6d0f2a2d63a036e9 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 14:42:30 -0400 Subject: [PATCH 10/17] db: reconcile the models with the deployed schema (#3275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Milestone 328's acceptance test compared a database built by the real 0001..0087 chain against one built from the models, and found ~130 places where they disagree. This closes them. Almost all were the MODEL being wrong, so almost all of this is model edits with no DDL — the database already had these things, nothing in it changes, and no deploy is needed for this part: * 92 columns gained server_default. The models carried Python-side `default=` only, so the ORM filled the value and the column had no database default. Anything inserting outside the ORM behaved differently from production. * Eleven indexes that existed only in migrations are now declared: the three backup_run reporting indexes, the two date-ordered image_record browse indexes, import_task and presentation_review, and the three task_run history indexes. All use text() for their DESC ordering and postgresql_where for the partial one. * Two UNIQUE indexes that autogenerate silently proposed DROPPING, because neither is expressible as a UniqueConstraint: uq_tag_name_kind_fandom — an EXPRESSION index over (name, kind, COALESCE(fandom_id, 0)) uq_post_artist_external_id_null_source — PARTIAL, WHERE source_id IS NULL post.py already had a comment describing the second one. The comment was right; nothing declared it. * The two external_link enum CHECKs (host, status) — rule 36 territory, and absent from the model entirely. * Two indexes were named explicitly. A bare index=True generated ix_tag_alias_canonical_tag_id where the database has ix_tag_alias_canonical, so autogenerate proposed a drop+create of an index that was already there under another name. Same for tag_suggestion_rejection. Only ONE thing needed DDL, as 0088: tag.fandom_id is declared index=True but no migration ever created that index. Deliberately NOT here: image_record.sha256. The model says unique=True; 0001 created a plain index. Duplicates are possible today and the ORM believes otherwise. The fix depends on whether duplicates already exist — if they do, that is a dedupe decision, not a constraint — so it waits on an answer about live data. The real severity of #3275 is not the squash. It is that --autogenerate has been unsafe on this project: run against the old models it would have proposed dropping eleven indexes and two uniqueness guarantees. --- .../0088_reconcile_models_with_schema.py | 48 ++++++++++ backend/app/models/artist.py | 4 +- backend/app/models/backup_run.py | 10 +- backend/app/models/download_event.py | 4 +- backend/app/models/external_link.py | 11 +++ backend/app/models/gpu_job.py | 5 +- backend/app/models/head_auto_apply_run.py | 5 +- backend/app/models/head_metric.py | 4 +- backend/app/models/head_metrics_snapshot.py | 6 +- backend/app/models/head_training_run.py | 3 +- backend/app/models/image_record.py | 11 ++- backend/app/models/import_batch.py | 14 +-- backend/app/models/import_settings.py | 39 +++++--- backend/app/models/import_task.py | 12 ++- backend/app/models/library_audit_run.py | 13 ++- backend/app/models/ml_settings.py | 95 +++++++++++++------ backend/app/models/patreon_failed_media.py | 2 +- backend/app/models/pixiv_failed_media.py | 2 +- backend/app/models/post.py | 8 +- backend/app/models/presentation_review.py | 6 +- backend/app/models/source.py | 6 +- .../app/models/subscribestar_failed_media.py | 2 +- backend/app/models/tag.py | 9 +- backend/app/models/tag_alias.py | 11 ++- .../app/models/tag_suggestion_rejection.py | 9 +- backend/app/models/task_run.py | 10 +- 26 files changed, 259 insertions(+), 90 deletions(-) create mode 100644 alembic/versions/0088_reconcile_models_with_schema.py diff --git a/alembic/versions/0088_reconcile_models_with_schema.py b/alembic/versions/0088_reconcile_models_with_schema.py new file mode 100644 index 0000000..6117dd3 --- /dev/null +++ b/alembic/versions/0088_reconcile_models_with_schema.py @@ -0,0 +1,48 @@ +"""Reconcile the database with what the models have always claimed (#3275). + +Milestone 328 discovered ~130 places where the ORM models and the deployed +schema disagreed. Almost all of them were the MODEL being wrong — missing +`server_default`s, indexes and CHECK constraints that only ever existed in a +migration — and those are fixed in the model files with no DDL at all, because +the database already had them. + +This migration carries the remainder: the one case where the MODEL was right +and the database was missing something. + +`tag.fandom_id` is declared `index=True` on the model, but no migration ever +created that index. Every autogenerate run since would have proposed adding +it; nobody ran one, so the model and the database simply drifted apart and +stayed that way. + +Deliberately NOT in this migration: making `image_record.sha256` unique. The +model says `unique=True` and `0001` created a plain, non-unique index, so +duplicates are possible today and the ORM believes they are not. Adding the +constraint is a real change that FAILS if duplicates already exist, and if +they do exist the right response is a dedupe decision rather than a constraint +— so it needs an answer about live data before it is written, not after. +Tracked in #3275. + +Revision ID: 0088 +Revises: 0087 +Create Date: 2026-08-30 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0088" +down_revision: Union[str, None] = "0087" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # IF NOT EXISTS because the index is what the model already asks for: any + # database built from metadata rather than from this chain will have it, + # and this migration must be a no-op there rather than an error. + op.execute("CREATE INDEX IF NOT EXISTS ix_tag_fandom_id ON tag (fandom_id)") + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_tag_fandom_id") diff --git a/backend/app/models/artist.py b/backend/app/models/artist.py index e7ca894..fb3401c 100644 --- a/backend/app/models/artist.py +++ b/backend/app/models/artist.py @@ -27,10 +27,10 @@ class Artist(Base): notes: Mapped[str | None] = mapped_column(Text, nullable=True) # True once a Source is attached; flips false if all sources removed. - is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_subscription: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") # Per-artist scheduling overrides; null means "use global default". - auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + auto_check: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true") check_interval_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) created_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/models/backup_run.py b/backend/app/models/backup_run.py index 1717aea..ad7b4f0 100644 --- a/backend/app/models/backup_run.py +++ b/backend/app/models/backup_run.py @@ -20,7 +20,7 @@ feedback_check_existing_enums): from datetime import datetime -from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, Text, text from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -29,10 +29,18 @@ from .base import Base class BackupRun(Base): __tablename__ = "backup_run" + + __table_args__ = ( + # alembic 0017: reporting indexes, never declared on the model (#3275). + Index("ix_backup_run_kind_started", "kind", text("started_at DESC")), + Index("ix_backup_run_status_finished", "status", text("finished_at DESC")), + Index("ix_backup_run_tag_partial", "tag", postgresql_where=text("tag IS NOT NULL")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True) status: Mapped[str] = mapped_column( String(16), nullable=False, default="pending", index=True, + server_default="pending", ) tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) triggered_by: Mapped[str] = mapped_column(String(32), nullable=False) diff --git a/backend/app/models/download_event.py b/backend/app/models/download_event.py index 7fe5b37..3ec00ae 100644 --- a/backend/app/models/download_event.py +++ b/backend/app/models/download_event.py @@ -25,8 +25,8 @@ class DownloadEvent(Base): DateTime(timezone=True), nullable=False, server_default=func.now() ) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0) - files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + bytes_downloaded: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0, server_default="0") + files_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") error: Mapped[str | None] = mapped_column(Text, nullable=True) metadata_: Mapped[dict] = mapped_column( "metadata", JSONB, nullable=False, default=dict, diff --git a/backend/app/models/external_link.py b/backend/app/models/external_link.py index 0902e28..dcf8ee8 100644 --- a/backend/app/models/external_link.py +++ b/backend/app/models/external_link.py @@ -16,6 +16,7 @@ doesn't delete the link record). from datetime import datetime from sqlalchemy import ( + CheckConstraint, DateTime, Float, ForeignKey, @@ -38,6 +39,16 @@ STATUSES = ("pending", "downloading", "downloaded", "failed", "skipped", "dead") class ExternalLink(Base): __tablename__ = "external_link" __table_args__ = ( + # alembic 0028 enum CHECKs. Rule 36 territory: a new host or status value + # needs its constraint swapped in the same migration (#3275). + CheckConstraint( + "host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')", + name="ck_external_link_host", + ), + CheckConstraint( + "status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')", + name="ck_external_link_status", + ), # One row per (post, url). The full url (incl. #fragment) is the identity # — the same file linked twice in a post collapses to one row. Index("uq_external_link_post_url", "post_id", "url", unique=True), diff --git a/backend/app/models/gpu_job.py b/backend/app/models/gpu_job.py index dba5997..931b455 100644 --- a/backend/app/models/gpu_job.py +++ b/backend/app/models/gpu_job.py @@ -50,7 +50,8 @@ class GpuJob(Base): # What to compute, e.g. 'ccip' (detect figures + CCIP-embed) or 'siglip_region'. task: Mapped[str] = mapped_column(String(32), nullable=False) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="pending", index=True + String(16), nullable=False, default="pending", index=True, + server_default="pending", ) # pending | leased | done | error lease_token: Mapped[str | None] = mapped_column(String(64), nullable=True) @@ -60,7 +61,7 @@ class GpuJob(Base): lease_expires_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True ) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") error: Mapped[str | None] = mapped_column(Text, nullable=True) # Triage verdict for an ERRORED job (#125): NULL = not yet probed; # 'defect' = the integrity probe says the FILE itself is bad (surfaced for diff --git a/backend/app/models/head_auto_apply_run.py b/backend/app/models/head_auto_apply_run.py index 08c359a..031109b 100644 --- a/backend/app/models/head_auto_apply_run.py +++ b/backend/app/models/head_auto_apply_run.py @@ -24,10 +24,11 @@ class HeadAutoApplyRun(Base): id: Mapped[int] = mapped_column(Integer, primary_key=True) # dry_run=True is a PREVIEW: scores + counts what WOULD apply, writes nothing # (preview/apply parity, rule 93). - dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + dry_run: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="running", index=True + String(16), nullable=False, default="running", index=True, + server_default="running", ) # running | ready | error started_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/models/head_metric.py b/backend/app/models/head_metric.py index a034e51..7afc104 100644 --- a/backend/app/models/head_metric.py +++ b/backend/app/models/head_metric.py @@ -24,9 +24,9 @@ class HeadMetric(Base): ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True ) # An auto-applied (source='head_auto') tag the operator later REMOVED. - n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # A tag with a head that the operator added by HAND (the head missed it). - n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() ) diff --git a/backend/app/models/head_metrics_snapshot.py b/backend/app/models/head_metrics_snapshot.py index a9ec7ac..651344e 100644 --- a/backend/app/models/head_metrics_snapshot.py +++ b/backend/app/models/head_metrics_snapshot.py @@ -28,9 +28,9 @@ class HeadMetricsSnapshot(Base): DateTime(timezone=True), nullable=False, server_default=func.now(), index=True ) # Current count of source='head_auto' applications still standing. - n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + n_auto_applied: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + n_misfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + n_underfires: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # The head's measured quality at snapshot time (null if no head exists). ap: Mapped[float | None] = mapped_column(Float, nullable=True) precision_cv: Mapped[float | None] = mapped_column(Float, nullable=True) diff --git a/backend/app/models/head_training_run.py b/backend/app/models/head_training_run.py index fd5858e..21c4ac7 100644 --- a/backend/app/models/head_training_run.py +++ b/backend/app/models/head_training_run.py @@ -24,7 +24,8 @@ class HeadTrainingRun(Base): # Training parameters: {min_positives, neg_ratio, precision_target, ...}. params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="running", index=True + String(16), nullable=False, default="running", index=True, + server_default="running", ) # running | ready | error started_at: Mapped[datetime] = mapped_column( diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index 32d6eaa..be273ce 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -14,10 +14,12 @@ from sqlalchemy import ( Enum, Float, ForeignKey, + Index, Integer, String, Text, func, + text, ) from sqlalchemy.orm import Mapped, mapped_column @@ -29,6 +31,12 @@ ORIGIN_CHOICES = ("downloaded", "imported_filesystem", "uploaded") class ImageRecord(Base): __tablename__ = "image_record" + + __table_args__ = ( + # alembic 0035/0071: the date-ordered browse indexes (#3275). + Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")), + Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) # On-disk identity @@ -47,7 +55,8 @@ class ImageRecord(Base): # Integrity verification status. FC-2e populates this; FC-2a leaves rows at 'unknown'. # Values: 'unknown' (default), 'ok', 'corrupt', 'failed_verification'. integrity_status: Mapped[str] = mapped_column( - String(24), nullable=False, default="unknown", index=True + String(24), nullable=False, default="unknown", index=True, + server_default="unknown", ) # Thumbnail (populated by FC-2) diff --git a/backend/app/models/import_batch.py b/backend/app/models/import_batch.py index 474f111..8d8fa61 100644 --- a/backend/app/models/import_batch.py +++ b/backend/app/models/import_batch.py @@ -21,17 +21,17 @@ class ImportBatch(Base): ) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) - total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + total_files: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + imported: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + skipped: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + attachments: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # Deep-scan only: count of already-imported files whose sidecar metadata # got re-applied this run (post/source/provenance upsert). Stays 0 on # quick-scan batches. See `Importer.import_one(deep_scan=True)`. - refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + refreshed: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") - status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="running", index=True, server_default="running") # running | complete | cancelled tasks = relationship("ImportTask", back_populates="batch", cascade="all, delete-orphan") diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index f4b8937..74ac13f 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -17,60 +17,71 @@ class ImportSettings(Base): __table_args__ = (CheckConstraint("id = 1", name="singleton"),) id: Mapped[int] = mapped_column(Integer, primary_key=True) - import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import") + import_scan_path: Mapped[str] = mapped_column(Text, nullable=False, default="/import", server_default="/import") - min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + min_width: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + min_height: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") - skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9) + skip_transparent: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") + transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9") - skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95) - single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30) + skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") + single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95") + single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30") - phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10) + phash_threshold: Mapped[int] = mapped_column(Integer, nullable=False, default=10, server_default="10") # FC-3c downloader knobs download_rate_limit_seconds: Mapped[float] = mapped_column( - Float, nullable=False, default=3.0 + Float, nullable=False, default=3.0, + server_default="3", ) download_validate_files: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) # FC-3d scheduling knobs download_schedule_default_seconds: Mapped[int] = mapped_column( - Integer, nullable=False, default=28800 + Integer, nullable=False, default=28800, + server_default="28800", ) download_event_retention_days: Mapped[int] = mapped_column( - Integer, nullable=False, default=90 + Integer, nullable=False, default=90, + server_default="90", ) download_failure_warning_threshold: Mapped[int] = mapped_column( - Integer, nullable=False, default=5 + Integer, nullable=False, default=5, + server_default="5", ) # FC-3h backup knobs. backup_db_nightly_enabled: Mapped[bool] = mapped_column( Boolean, nullable=False, default=False, + server_default="false", ) backup_db_nightly_hour_utc: Mapped[int] = mapped_column( Integer, nullable=False, default=3, + server_default="3", ) backup_db_keep_last_n: Mapped[int] = mapped_column( Integer, nullable=False, default=14, + server_default="14", ) backup_images_keep_last_n: Mapped[int] = mapped_column( Integer, nullable=False, default=3, + server_default="3", ) # FC-6.3 series continuation matcher. enabled gates the rescan; threshold is # the weighted-score cut-off (0..1) above which a pending suggestion is made. series_suggest_enabled: Mapped[bool] = mapped_column( Boolean, nullable=False, default=True, + server_default="true", ) series_suggest_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.5, + server_default="0.5", ) # #830 off-platform file-host downloads — per-host enable lever (default on, diff --git a/backend/app/models/import_task.py b/backend/app/models/import_task.py index 3c947c1..c3d9f11 100644 --- a/backend/app/models/import_task.py +++ b/backend/app/models/import_task.py @@ -13,10 +13,12 @@ from sqlalchemy import ( Boolean, DateTime, ForeignKey, + Index, Integer, String, Text, func, + text, ) from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -26,6 +28,10 @@ from .base import Base class ImportTask(Base): __tablename__ = "import_task" + + __table_args__ = ( + Index("ix_import_task_created_at_desc", text("created_at DESC")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) batch_id: Mapped[int] = mapped_column( ForeignKey("import_batch.id", ondelete="CASCADE"), nullable=False, index=True @@ -33,14 +39,14 @@ class ImportTask(Base): source_path: Mapped[str] = mapped_column(Text, nullable=False) task_type: Mapped[str] = mapped_column(String(16), nullable=False) # media|archive - status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="pending", index=True, server_default="pending") # Poison-pill circuit breaker (alembic 0026). recovery_count tracks # how many times the stuck-task sweep has re-queued this row; after # the cap it's failed with a diagnostic instead of looping. refetched # bounds the one-shot re-download remediation to a single attempt. - recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + recovery_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + refetched: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false") result_image_id: Mapped[int | None] = mapped_column( ForeignKey("image_record.id", ondelete="SET NULL"), nullable=True diff --git a/backend/app/models/library_audit_run.py b/backend/app/models/library_audit_run.py index a2d4bc2..6ac9bb2 100644 --- a/backend/app/models/library_audit_run.py +++ b/backend/app/models/library_audit_run.py @@ -8,7 +8,7 @@ reads it and routes through cleanup_service.delete_images. from datetime import datetime from typing import Any -from sqlalchemy import DateTime, Integer, String, Text, func +from sqlalchemy import DateTime, Integer, String, Text, func, text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Mapped, mapped_column @@ -23,6 +23,7 @@ class LibraryAuditRun(Base): params: Mapped[dict[str, Any]] = mapped_column(JSONB, nullable=False) status: Mapped[str] = mapped_column( String(16), nullable=False, default="running", index=True, + server_default="running", ) # running | ready | applied | cancelled | error started_at: Mapped[datetime] = mapped_column( @@ -31,14 +32,16 @@ class LibraryAuditRun(Base): finished_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, ) - scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - matched_ids: Mapped[list[int]] = mapped_column(JSONB, nullable=False, default=list) + scanned_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + matched_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") + matched_ids: Mapped[list[int]] = mapped_column( + JSONB, nullable=False, default=list, server_default=text("'[]'::jsonb") + ) error: Mapped[str | None] = mapped_column(Text, nullable=True) # Chunked-scan state (alembic 0039): keyset cursor the next chunk resumes # from, and the last time a chunk made progress (so the recovery sweep can # tell a progressing multi-chunk audit from a stuck one). - resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + resume_after_id: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") last_progress_at: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, ) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 72da17b..3e1570a 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -31,17 +31,20 @@ class MLSettings(Base): # queueing embed work nothing will consume (the daily GPU 'embed' backfill # covers those images instead). cpu_embed_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) # Video embedding (#747). Sample one frame every N seconds (fixed CADENCE, not # a fixed count) so coverage reflects real screen time regardless of length; # cap the total so a long video can't explode into hundreds of embeds. The # per-frame SigLIP embeddings are mean-pooled. Operator-tunable. video_frame_interval_seconds: Mapped[float] = mapped_column( - Float, nullable=False, default=4.0 + Float, nullable=False, default=4.0, + server_default="4", ) video_max_frames: Mapped[int] = mapped_column( - Integer, nullable=False, default=64 + Integer, nullable=False, default=64, + server_default="64", ) # Tagging-v2 head training (#114). The head is the suggestion source that # LEARNS from the operator's tags (replacing Camie + centroid). A concept @@ -49,10 +52,12 @@ class MLSettings(Base): # head_auto_apply_precision is the precision bar a head must clear (at some # operating point) to "graduate" into earned auto-apply. Operator-tunable. head_min_positives: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) head_auto_apply_precision: Mapped[float] = mapped_column( - Float, nullable=False, default=0.97 + Float, nullable=False, default=0.97, + server_default="0.97", ) # Earned auto-apply (#114). A graduated head fires (tags images without a # human) when this master switch is on AND the head has at least @@ -61,29 +66,34 @@ class MLSettings(Base): # default (operator-asked 2026-06-29: opt-OUT, not opt-in); the support + # measured-precision gates keep it safe, and every auto-tag is reversible. head_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) head_auto_apply_min_positives: Mapped[int] = mapped_column( # Support floor raised 30→50 (operator-asked 2026-07-06): a head needs # more human labels before it may fire without a human. - Integer, nullable=False, default=50 + Integer, nullable=False, default=50, + server_default="30", ) # CCIP character-match cosine cut (#114). 0.85 default — the v1 flat 0.75 # over-fired (high-reference characters matched a scatter of images); 0.85 # keeps the confident single-character matches. Tunable from the agent card. ccip_match_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.85 + Float, nullable=False, default=0.85, + server_default="0.85", ) # CCIP auto-apply (#114). Confident matches (>= ccip_auto_apply_threshold, # above the suggest cut) auto-tag on a daily sweep. ON by default (opt-out); # single-character references + the high bar keep it safe, every tag reversible. ccip_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) ccip_auto_apply_threshold: Mapped[float] = mapped_column( # Raised 0.92→0.95 (operator-asked 2026-07-06) so only very confident # character matches auto-tag. - Float, nullable=False, default=0.95 + Float, nullable=False, default=0.95, + server_default="0.92", ) # -- Presentation chrome auto-hide (#141) ------------------------------- # `banner` (chrome — clusters on UI, not content) auto-applies on the sweep @@ -95,13 +105,16 @@ class MLSettings(Base): # (opt-out); every auto-tag is reversible. NOTE (#1464): `wip` + `editor # screenshot` are no longer chrome — they went to the PROCESS path below. presentation_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) presentation_auto_apply_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.90 + Float, nullable=False, default=0.90, + server_default="0.90", ) presentation_conflict_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.50 + Float, nullable=False, default=0.50, + server_default="0.50", ) # -- Process auto-apply (#1464) ---------------------------------------- # `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program @@ -115,24 +128,29 @@ class MLSettings(Base): # (PresentationReview, mode='process') rather than silently marked. OFF by # default — a new whole-library auto-tagger is opt-in; every auto-tag reversible. process_auto_apply_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False + Boolean, nullable=False, default=False, + server_default="false", ) process_auto_apply_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.90 + Float, nullable=False, default=0.90, + server_default="0.9", ) process_conflict_threshold: Mapped[float] = mapped_column( - Float, nullable=False, default=0.50 + Float, nullable=False, default=0.50, + server_default="0.5", ) # Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069); # existing libraries keep their stored value until the operator re-embeds. embedder_model_version: Mapped[str] = mapped_column( - String(128), nullable=False, default="siglip2-so400m-patch16-512" + String(128), nullable=False, default="siglip2-so400m-patch16-512", + server_default="siglip2-so400m-patch16-512", ) # The HF model NAME the embedder loads (server CPU embed + announced to the # GPU agent in the lease). Operator-settable so the embedder is a choice, not # a hardcode (#1190): set name + version together, then re-embed + retrain. embedder_model_name: Mapped[str] = mapped_column( - String(128), nullable=False, default="google/siglip2-so400m-patch16-512" + String(128), nullable=False, default="google/siglip2-so400m-patch16-512", + server_default="google/siglip2-so400m-patch16-512", ) # -- Crop proposers / detectors (#1202, #134) -------------------------- # WHERE-to-crop YOLO detectors feeding the crop→SigLIP bag + CCIP. Config @@ -145,20 +163,24 @@ class MLSettings(Base): # person: general COCO figure detector for Western/realistic art the anime # person-detector misses → NMS-merged with imgutils → CCIP + concept. detector_person_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) detector_person_weights: Mapped[str] = mapped_column( - String(512), nullable=False, default="yolo11n.pt" + String(512), nullable=False, default="yolo11n.pt", + server_default="yolo11n.pt", ) detector_person_conf: Mapped[float] = mapped_column( - Float, nullable=False, default=0.35 + Float, nullable=False, default=0.35, + server_default="0.35", ) # anatomy: booru_yolo anime/furry/NSFW torso components → concept crops. # Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the # upstream repo so the URL resolves. License UNSTATED — fine for a private # homelab (operator accepted #1202). detector_anatomy_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) detector_anatomy_weights: Mapped[str] = mapped_column( String(512), nullable=False, @@ -166,37 +188,47 @@ class MLSettings(Base): "https://github.com/aperveyev/booru_yolo/raw/main/models/" "yolov11m_aa22.pt" ), + server_default="https://github.com/aperveyev/booru_yolo/raw/main/models/yolov11m_aa22.pt", ) detector_anatomy_conf: Mapped[float] = mapped_column( - Float, nullable=False, default=0.30 + Float, nullable=False, default=0.30, + server_default="0.30", ) # panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x). detector_panel_enabled: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True + Boolean, nullable=False, default=True, + server_default="true", ) detector_panel_weights: Mapped[str] = mapped_column( String(512), nullable=False, default="mosesb/best-comic-panel-detection::best.pt", + server_default="mosesb/best-comic-panel-detection::best.pt", ) detector_panel_conf: Mapped[float] = mapped_column( - Float, nullable=False, default=0.30 + Float, nullable=False, default=0.30, + server_default="0.30", ) # Per-frame caps bound the crop→embed explosion; max_regions is the hard # per-job backstop; dedupe_iou drops near-duplicate crops before the embed. detector_max_figures: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) detector_max_components: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) detector_max_panels: Mapped[int] = mapped_column( - Integer, nullable=False, default=8 + Integer, nullable=False, default=8, + server_default="8", ) detector_max_regions: Mapped[int] = mapped_column( - Integer, nullable=False, default=128 + Integer, nullable=False, default=128, + server_default="128", ) detector_dedupe_iou: Mapped[float] = mapped_column( - Float, nullable=False, default=0.85 + Float, nullable=False, default=0.85, + server_default="0.85", ) # -- CCIP character prototypes (#1317) --------------------------------- # The per-character reference set is precomputed + refreshed INCREMENTALLY @@ -208,7 +240,8 @@ class MLSettings(Base): String(128), nullable=True ) ccip_prototype_cap: Mapped[int] = mapped_column( - Integer, nullable=False, default=64 + Integer, nullable=False, default=64, + server_default="64", ) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/patreon_failed_media.py b/backend/app/models/patreon_failed_media.py index 79976ef..26557fe 100644 --- a/backend/app/models/patreon_failed_media.py +++ b/backend/app/models/patreon_failed_media.py @@ -35,7 +35,7 @@ class PatreonFailedMedia(Base): ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True ) filehash: Mapped[str] = mapped_column(String(128), nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") last_error: Mapped[str | None] = mapped_column(Text, nullable=True) first_failed_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/pixiv_failed_media.py b/backend/app/models/pixiv_failed_media.py index a33d15e..7737594 100644 --- a/backend/app/models/pixiv_failed_media.py +++ b/backend/app/models/pixiv_failed_media.py @@ -35,7 +35,7 @@ class PixivFailedMedia(Base): ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True ) filehash: Mapped[str] = mapped_column(String(128), nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") last_error: Mapped[str | None] = mapped_column(Text, nullable=True) first_failed_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/post.py b/backend/app/models/post.py index 4183330..1cc15a4 100644 --- a/backend/app/models/post.py +++ b/backend/app/models/post.py @@ -9,15 +9,17 @@ artist-filter queries don't depend on the Source detour). from datetime import datetime from sqlalchemy import ( - JSON, CheckConstraint, DateTime, ForeignKey, + Index, Integer, + JSON, String, Text, UniqueConstraint, func, + text, ) from sqlalchemy.orm import Mapped, mapped_column @@ -27,6 +29,10 @@ from .base import Base class Post(Base): __tablename__ = "post" __table_args__ = ( + # alembic 0030. The comment above described this index; nothing declared + # it, so autogenerate proposed dropping it (#3275). + Index("uq_post_artist_external_id_null_source", "artist_id", "external_post_id", + unique=True, postgresql_where=text("source_id IS NULL")), # Source-bound dedup. Postgres treats NULL != NULL so rows # with source_id IS NULL aren't deduped by this constraint; # the partial unique index `uq_post_artist_external_id_null_source` diff --git a/backend/app/models/presentation_review.py b/backend/app/models/presentation_review.py index 73da13f..e18e298 100644 --- a/backend/app/models/presentation_review.py +++ b/backend/app/models/presentation_review.py @@ -11,7 +11,7 @@ are pruned by retention. from datetime import datetime -from sqlalchemy import DateTime, Float, ForeignKey, String, func +from sqlalchemy import DateTime, Float, ForeignKey, Index, String, func from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -20,6 +20,10 @@ from .base import Base class PresentationReview(Base): __tablename__ = "presentation_review" + + __table_args__ = ( + Index("ix_presentation_review_resolved_at", "resolved_at"), + ) image_record_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True ) diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 1bf6c67..1c29e6f 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -5,7 +5,7 @@ Multiple sources per artist support creators with cross-platform presence. from datetime import datetime -from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, JSON, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship from .base import Base @@ -20,7 +20,7 @@ class Source(Base): ) platform: Mapped[str] = mapped_column(String(64), nullable=False) url: Mapped[str] = mapped_column(Text, nullable=False) - enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true") config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True) @@ -32,7 +32,7 @@ class Source(Base): # by _update_source_health alongside last_error; cleared on 'ok'. error_type: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) check_interval_override: Mapped[int | None] = mapped_column(Integer, nullable=True) - consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default="0") # alembic 0031: sticky deep-scan budget. When > 0, the next N download # runs use gallery-dl's full-walk config (skip: True + 1800s timeout); diff --git a/backend/app/models/subscribestar_failed_media.py b/backend/app/models/subscribestar_failed_media.py index 9201aa7..d12ff73 100644 --- a/backend/app/models/subscribestar_failed_media.py +++ b/backend/app/models/subscribestar_failed_media.py @@ -34,7 +34,7 @@ class SubscribeStarFailedMedia(Base): ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True ) filehash: Mapped[str] = mapped_column(String(128), nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1") last_error: Mapped[str | None] = mapped_column(Text, nullable=True) first_failed_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index 8d5a256..d3107d0 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -15,11 +15,13 @@ from sqlalchemy import ( Column, DateTime, ForeignKey, + Index, Integer, String, Table, false, func, + text, ) from sqlalchemy import ( Enum as SQLEnum, @@ -67,7 +69,7 @@ image_tag = Table( primary_key=True, ), Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True), - Column("source", String(32), nullable=False, default="manual"), + Column("source", String(32), nullable=False, default="manual", server_default="manual"), Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), ) @@ -75,6 +77,10 @@ image_tag = Table( class Tag(Base): __tablename__ = "tag" __table_args__ = ( + # alembic 0002. An EXPRESSION index — COALESCE cannot be expressed as a + # UniqueConstraint, which is why it only ever existed in a migration (#3275). + Index("uq_tag_name_kind_fandom", "name", "kind", text("COALESCE(fandom_id, 0)"), + unique=True), CheckConstraint( "(fandom_id IS NULL) OR (kind = 'character')", name="ck_tag_fandom_requires_character", @@ -87,6 +93,7 @@ class Tag(Base): SQLEnum(TagKind, name="tag_kind", values_callable=lambda e: [m.value for m in e]), nullable=False, default=TagKind.general, + server_default="general", ) fandom_id: Mapped[int | None] = mapped_column( ForeignKey("tag.id", ondelete="SET NULL"), nullable=True, index=True diff --git a/backend/app/models/tag_alias.py b/backend/app/models/tag_alias.py index 93f4755..533cec3 100644 --- a/backend/app/models/tag_alias.py +++ b/backend/app/models/tag_alias.py @@ -5,7 +5,7 @@ in image_prediction stay unmolested. from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, String, func +from sqlalchemy import DateTime, ForeignKey, Index, String, func from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -14,10 +14,17 @@ from .base import Base class TagAlias(Base): __tablename__ = "tag_alias" + + __table_args__ = ( + # Named explicitly: the database calls this ix_tag_alias_canonical, while + # a bare index=True on the column would generate ix_tag_alias_canonical_tag_id + # and silently propose a drop+create on the next autogenerate (#3275). + Index("ix_tag_alias_canonical", "canonical_tag_id"), + ) alias_string: Mapped[str] = mapped_column(String(255), primary_key=True) alias_category: Mapped[str] = mapped_column(String(32), primary_key=True) canonical_tag_id: Mapped[int] = mapped_column( - ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True + ForeignKey("tag.id", ondelete="CASCADE"), nullable=False ) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/tag_suggestion_rejection.py b/backend/app/models/tag_suggestion_rejection.py index 8a834fe..02756de 100644 --- a/backend/app/models/tag_suggestion_rejection.py +++ b/backend/app/models/tag_suggestion_rejection.py @@ -5,7 +5,7 @@ Prevents re-suggestion AND prevents allowlist auto-apply on that image. from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, func +from sqlalchemy import DateTime, ForeignKey, Index, func from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -14,11 +14,16 @@ from .base import Base class TagSuggestionRejection(Base): __tablename__ = "tag_suggestion_rejection" + + __table_args__ = ( + # Named explicitly; see tag_alias for why (#3275). + Index("ix_tag_suggestion_rejection_tag", "tag_id"), + ) image_record_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True ) tag_id: Mapped[int] = mapped_column( - ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True, index=True + ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True ) rejected_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() diff --git a/backend/app/models/task_run.py b/backend/app/models/task_run.py index e8eac05..c24ed46 100644 --- a/backend/app/models/task_run.py +++ b/backend/app/models/task_run.py @@ -15,7 +15,7 @@ backend.app.tasks.maintenance.recover_stalled_task_runs (Beat 5 min). from datetime import datetime -from sqlalchemy import DateTime, Integer, String, Text +from sqlalchemy import DateTime, Index, Integer, String, Text, text from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -24,6 +24,13 @@ from .base import Base class TaskRun(Base): __tablename__ = "task_run" + + __table_args__ = ( + # alembic 0016: the three task-history indexes (#3275). + Index("ix_task_run_name_started", "task_name", text("started_at DESC")), + Index("ix_task_run_queue_started", "queue", text("started_at DESC")), + Index("ix_task_run_status_started", "status", text("started_at DESC")), + ) id: Mapped[int] = mapped_column(Integer, primary_key=True) celery_task_id: Mapped[str] = mapped_column( String(64), nullable=False, index=True, @@ -40,6 +47,7 @@ class TaskRun(Base): duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) status: Mapped[str] = mapped_column( String(16), nullable=False, default="running", index=True, + server_default="running", ) error_type: Mapped[str | None] = mapped_column(String(128), nullable=True) error_message: Mapped[str | None] = mapped_column(Text, nullable=True) -- 2.54.0 From ed2b1adc2e53a542baa69cb05bec6ee227b43bf1 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 14:43:26 -0400 Subject: [PATCH 11/17] ci: compare the schema the MODELS produce against the migrations (#3275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit baseline.yml only ever compared migrations against migrations. The question #3275 exists because nobody had ever asked the other one: does a database built from the MODELS match the one the chain produces? `mode: models` answers it. It applies the candidate autogenerated from the models instead of this tree's revisions, and diffs that against the chain. A clean run means --autogenerate is trustworthy again, which it demonstrably has not been: against the pre-reconciliation models it would have proposed dropping eleven indexes and two uniqueness guarantees. The two extensions are created by hand in that mode. They are database objects rather than table metadata, so no model can carry them — their absence is outside what this comparison asks about, and silently tolerating it is correct rather than a filter that hides a defect. Also declares the HNSW index on the ImageRecord model. SQLAlchemy can express an hnsw access method with an operator class (postgresql_using + postgresql_ops), so there was never a reason for it to live only in 0036. That removes the last item from the list of things a generated baseline cannot reproduce, leaving only the two extensions. --- .forgejo/workflows/baseline.yml | 40 +++++++++++++++++++++++++----- backend/app/models/image_record.py | 11 ++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index e7df685..14643f8 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -37,6 +37,10 @@ on: description: 'Commit/tag that still carries the full 0001..0087 chain' type: string default: '0a5bbe8' + mode: + description: 'chain = compare against this tree''s migrations; models = compare against a schema built from the MODELS' + type: string + default: 'chain' jobs: compare: @@ -174,20 +178,44 @@ jobs: echo "candidate-bytes: $(wc -c < "$F")" echo "candidate-b64-lines: $(echo "$B64" | wc -l)" set -x + mkdir -p /tmp/candidate + cp alembic/versions/*.py /tmp/candidate/ # Put the tree back exactly as it was; this job never mutates state. rm -f alembic/versions/*.py mv /tmp/versions_held/*.py alembic/versions/ 2>/dev/null || true - # DB 2: whatever the CURRENT tree's alembic/versions produces. Before the - # squash that is the same 87 revisions and the diff is trivially clean — - # which is worth running once as a control, so a clean diff after the - # squash means something. + # DB 2: what the CURRENT tree produces. + # + # `mode: models` applies the candidate autogenerated from the MODELS + # instead, which is what answers "do the models describe the schema?" — + # the question #3275 exists because nobody had ever asked it. Under that + # mode a clean diff means autogenerate is trustworthy again. + # + # The two extensions are created by hand first. They are database + # objects, not table metadata, so no model can carry them and their + # absence is not a model defect — it is simply outside what this + # comparison is asking about. - name: Build the schema the CURRENT tree produces + env: + MODE: ${{ github.event.inputs.mode }} run: | set -eux docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_base - ls alembic/versions/*.py | wc -l - DB_NAME=fc_base alembic upgrade head + if [ "${MODE:-chain}" = "models" ]; then + docker exec "$PG_CONTAINER" psql -U fabledcurator -d fc_base \ + -c "CREATE EXTENSION IF NOT EXISTS vector" \ + -c "CREATE EXTENSION IF NOT EXISTS tsm_system_rows" + mkdir -p /tmp/held + mv alembic/versions/*.py /tmp/held/ + cp /tmp/candidate/*.py alembic/versions/ + ls alembic/versions/*.py + DB_NAME=fc_base alembic upgrade head + rm -f alembic/versions/*.py + mv /tmp/held/*.py alembic/versions/ + else + ls alembic/versions/*.py | wc -l + DB_NAME=fc_base alembic upgrade head + fi docker exec "$PG_CONTAINER" pg_dump -U fabledcurator --schema-only \ --no-owner --no-privileges -d fc_base > baseline.sql wc -l baseline.sql diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index be273ce..3de0a3f 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -33,6 +33,17 @@ class ImageRecord(Base): __table_args__ = ( + # alembic 0036, and the last thing in this schema that lived only in a + # migration. SQLAlchemy CAN express an hnsw index with an operator + # class, so there is no reason for it to be invisible to the models — + # and its absence was the quietest failure of the lot: everything + # works, similarity search just silently stops using an index. + Index( + "ix_image_record_siglip_hnsw", + "siglip_embedding", + postgresql_using="hnsw", + postgresql_ops={"siglip_embedding": "vector_cosine_ops"}, + ), # alembic 0035/0071: the date-ordered browse indexes (#3275). Index("ix_image_record_effective_date", text("effective_date DESC"), text("id DESC")), Index("ix_image_record_earliest_post_date", text("earliest_post_date DESC"), text("id DESC")), -- 2.54.0 From d044e93bdbbaeb80d3fb798cccb6058d1ad52ee5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Sun, 30 Aug 2026 14:47:29 -0400 Subject: [PATCH 12/17] ci: repair autogenerate's missing pgvector import before applying (#3275) mode: models applies the raw autogenerated candidate, and it cannot run: sa.Column('weights', pgvector.sqlalchemy.vector.VECTOR(dim=1152), ...) NameError: name 'pgvector' is not defined Alembic emits the qualified reference without emitting the import. Observed on run 4988, which turns this from a thing I predicted by reading the candidate into a thing demonstrated by executing it. Repaired in the workflow rather than counted as a schema difference: the comparison asks whether the MODELS describe the schema, and this is a defect in the generator. The same fixup has to be applied by hand to any baseline generated this way, which is why it is item 4 on the collapsed baseline's hand-written list. --- .forgejo/workflows/baseline.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index 14643f8..7d36ac4 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -208,6 +208,15 @@ jobs: mkdir -p /tmp/held mv alembic/versions/*.py /tmp/held/ cp /tmp/candidate/*.py alembic/versions/ + # Autogenerate EMITS pgvector.sqlalchemy.vector.VECTOR(...) without + # importing it, so the file it writes cannot run: + # NameError: name 'pgvector' is not defined + # Observed on run 4988, which is the proof rather than the theory. + # This is a defect in the GENERATOR, not in the models, so it is + # repaired here rather than counted as a schema difference — the + # comparison is about whether the models describe the schema. + sed -i '0,/^import sqlalchemy as sa$/s//import sqlalchemy as sa\nimport pgvector.sqlalchemy.vector/' alembic/versions/*.py + grep -n 'import pgvector' alembic/versions/*.py ls alembic/versions/*.py DB_NAME=fc_base alembic upgrade head rm -f alembic/versions/*.py -- 2.54.0 From 573228b9da2bb08584d6d1a9704ecc3d04c64a9d Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 31 Aug 2026 00:24:00 -0400 Subject: [PATCH 13/17] db: finish reconciling the models with the deployed schema (#3275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the residue the first reconciliation pass left, and corrects a factual error I put into the record. sha256 was NOT missing a uniqueness guarantee. I read `op.create_index("ix_image_record_sha256", ...)` at 0001 line 151 and concluded duplicates were possible, without reading line 149 two lines above it: sa.UniqueConstraint("sha256", name="uq_image_record_sha256"), Uniqueness has held since the initial schema. The database expresses it as a CONSTRAINT plus a separate non-unique lookup index; the model said `unique=True, index=True`, which is one UNIQUE index under a different name. Same guarantee, different objects — which is exactly why the two schemas did not line up. The model now declares both objects. No DDL. 0088's docstring, which repeated the claim, is corrected in place. Two real divergences, both the MODEL over-claiming: * source: uq_source_artist_platform_url (alembic 0010) was declared nowhere in the models — source.py had no __table_args__ at all — so autogenerate would have proposed DROPPING it. * head_metrics_snapshot.tag_id: model said NOT NULL, 0060 created it nullable. Left nullable; the FK already cascades. Seven constraints renamed to what the chain actually created, rather than what base.py's naming convention renders: uq_series_page_image, uq_series_chapter_anchor_page, fk_series_chapter_anchor_page, fk_image_record_artist_id, fk_image_provenance_from_attachment, and the two hand-shortened fk_tsr_* names from 0003. Float server_defaults now mirror their own migration, per column. The chain is MIXED: a plain string renders DEFAULT '0.90'::double precision, sa.text() renders DEFAULT 0.90, and the migrations used both. Seven columns take text(); the rest stay strings. Two literals also disagreed outright — process_{auto_apply,conflict}_threshold said 0.9/0.5 against the migration's 0.90/0.50. baseline.yml gains two things. A repair for a SECOND generator defect in the same class as the missing pgvector import: base.py's ck convention contains %(constraint_name)s, so it applies even to a NAMED CheckConstraint — autogenerate writes the already-rendered name into the migration and running it applies the convention again, yielding ck_ml_settings_ck_ml_settings_singleton. That is round-tripping damage, not a claim the models make, so it is undone rather than counted. And the diff now runs twice. Column ORDER differs permanently between a schema built by 87 ADD COLUMNs and one built in a single shot — the operator's database keeps chain order forever, a fresh install gets model order — so a check that failed on it could never pass. The second pass SORTS column lines within each CREATE TABLE instead of deleting them, which cannot hide a column present on one side only, or one whose type, nullability or default differs. Ordered diff is reported as information; the order-insensitive one is the verdict. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017QHszn9H8VBvx5Ke8x1hvw --- .forgejo/workflows/baseline.yml | 102 +++++++++++++++++- .../0088_reconcile_models_with_schema.py | 21 ++-- backend/app/models/head_metrics_snapshot.py | 10 +- backend/app/models/image_provenance.py | 9 +- backend/app/models/image_record.py | 20 +++- backend/app/models/import_settings.py | 14 ++- backend/app/models/ml_settings.py | 22 ++-- backend/app/models/series_chapter.py | 25 ++++- backend/app/models/series_page.py | 19 +++- backend/app/models/source.py | 25 ++++- .../app/models/tag_suggestion_rejection.py | 12 ++- 11 files changed, 247 insertions(+), 32 deletions(-) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index 7d36ac4..4681a35 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -217,6 +217,42 @@ jobs: # comparison is about whether the models describe the schema. sed -i '0,/^import sqlalchemy as sa$/s//import sqlalchemy as sa\nimport pgvector.sqlalchemy.vector/' alembic/versions/*.py grep -n 'import pgvector' alembic/versions/*.py + # Second generator defect, same class as the missing import. + # + # base.py's naming convention includes %(constraint_name)s for ck, + # which — unlike uq/fk/ix — means the convention is applied even to + # a CheckConstraint that HAS a name. So a model declaring + # name="singleton" correctly becomes ck_ml_settings_singleton in + # the metadata. Autogenerate then writes that RENDERED name into + # the migration, and running the migration applies the convention a + # SECOND time: ck_ml_settings_ck_ml_settings_singleton. + # + # That is round-tripping damage done by the generator, not a claim + # the models make, so it is repaired here rather than counted as a + # schema difference. Undone by removing the ck_
_ prefix the + # convention will re-add — the exact inverse, and it only fires on + # a name that actually carries its own table's prefix. + python3 - alembic/versions/*.py <<'PYEOF' + import re, sys + + table = None + for path in sys.argv[1:]: + out = [] + for line in open(path): + m = re.search(r"op\.create_table\(\s*[\"']([A-Za-z0-9_]+)[\"']", line) + if m: + table = m.group(1) + if table and "CheckConstraint" in line: + prefix = f"ck_{table}_" + line = re.sub( + r"(name=[\"'])" + re.escape(prefix), + r"\1", + line, + ) + out.append(line) + open(path, "w").writelines(out) + PYEOF + grep -n 'CheckConstraint' alembic/versions/*.py || true ls alembic/versions/*.py DB_NAME=fc_base alembic upgrade head rm -f alembic/versions/*.py @@ -245,6 +281,25 @@ jobs: # these two lines and nothing else. That control is what licenses this # filter — it was observed to be the only false positive, rather than # assumed to be one. + # Column ORDER inside a CREATE TABLE is compared separately from column + # CONTENT, and only content is fatal. + # + # A table built by 87 migrations has its columns in ADD COLUMN order; the + # same table built in one shot has them in declaration order. That is a + # real and permanent difference which no baseline can erase — the + # operator's existing database keeps chain order forever, a fresh install + # gets model order — so a check that fails on it would never pass and + # would teach nothing. FC reaches every column through the ORM by name, + # and `SELECT *` ordering is not depended on anywhere. + # + # So the second pass SORTS the column lines within each CREATE TABLE + # rather than DELETING them. That distinction is the whole point: sorting + # cannot hide a column that exists on one side only, or one whose type, + # nullability or default differs — those still land in the diff. A filter + # could have hidden all three. + # + # Both diffs are reported. The ordered one is informational; the + # order-insensitive one is the verdict. - name: Diff run: | set -eu @@ -256,11 +311,54 @@ jobs: norm chain.sql > a.txt norm baseline.sql > b.txt echo "normalised: chain=$(wc -l < a.txt) lines, current=$(wc -l < b.txt) lines" + + sort_table_columns() { + python3 - "$1" <<'PYEOF' + import re, sys + + lines = open(sys.argv[1]).read().splitlines() + out, block = [], None + for line in lines: + if block is not None: + # ');' on its own closes the CREATE TABLE body. + if line.strip() == ");": + out.extend(sorted(block)) + out.append(line) + block = None + else: + # Drop the list comma before sorting. Only the LAST + # column lacks one, so keeping it would make every + # reordering look like a content change as well — the + # comma is punctuation, and carries no schema meaning. + block.append(line.rstrip().rstrip(",")) + continue + out.append(line) + if re.match(r"CREATE TABLE .*\($", line): + block = [] + if block is not None: # unterminated body: emit it rather than drop it + out.extend(block) + print("\n".join(out)) + PYEOF + } + sort_table_columns a.txt > a.sorted.txt + sort_table_columns b.txt > b.sorted.txt + test "$(wc -l < a.sorted.txt)" = "$(wc -l < a.txt)" + test "$(wc -l < b.sorted.txt)" = "$(wc -l < b.txt)" + if diff -u a.txt b.txt > schema.diff; then - echo "SCHEMAS IDENTICAL — the collapsed chain reproduces the old one." + echo "ORDERED DIFF: identical, column order included." else - echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' schema.diff) changed lines:" + echo "ORDERED DIFF: $(grep -cE '^[+-]' schema.diff) changed lines (informational):" cat schema.diff + fi + echo + echo "================================================================" + echo + if diff -u a.sorted.txt b.sorted.txt > sorted.diff; then + echo "SCHEMAS MATCH — every difference above is column ORDER alone." + else + echo "SCHEMAS DIFFER — $(grep -cE '^[+-]' sorted.diff) changed lines that are NOT ordering:" + cat sorted.diff echo echo "The baseline is wrong, not the database. Do not stamp." exit 1 diff --git a/alembic/versions/0088_reconcile_models_with_schema.py b/alembic/versions/0088_reconcile_models_with_schema.py index 6117dd3..a286c6c 100644 --- a/alembic/versions/0088_reconcile_models_with_schema.py +++ b/alembic/versions/0088_reconcile_models_with_schema.py @@ -14,13 +14,20 @@ created that index. Every autogenerate run since would have proposed adding it; nobody ran one, so the model and the database simply drifted apart and stayed that way. -Deliberately NOT in this migration: making `image_record.sha256` unique. The -model says `unique=True` and `0001` created a plain, non-unique index, so -duplicates are possible today and the ORM believes they are not. Adding the -constraint is a real change that FAILS if duplicates already exist, and if -they do exist the right response is a dedupe decision rather than a constraint -— so it needs an answer about live data before it is written, not after. -Tracked in #3275. +Deliberately NOT in this migration: anything about `image_record.sha256`. An +earlier draft of this file claimed sha256 was not unique in the database and +that duplicate rows were therefore possible. That was WRONG, and it was wrong +because it was read off `op.create_index("ix_image_record_sha256", ...)` at +0001 line 151 without reading line 149 two lines above it: + + sa.UniqueConstraint("sha256", name="uq_image_record_sha256"), + +Uniqueness has been enforced since the initial schema. The database simply +expresses it as a CONSTRAINT plus a separate non-unique lookup index, where +the model expressed it as one `unique=True, index=True` column — the same +guarantee built from different objects, which is why the two schemas did not +line up. The model now declares the constraint and the plain index separately, +so it describes what is actually there. No DDL is needed for it. Revision ID: 0088 Revises: 0087 diff --git a/backend/app/models/head_metrics_snapshot.py b/backend/app/models/head_metrics_snapshot.py index 651344e..dfde05e 100644 --- a/backend/app/models/head_metrics_snapshot.py +++ b/backend/app/models/head_metrics_snapshot.py @@ -19,8 +19,14 @@ class HeadMetricsSnapshot(Base): __tablename__ = "head_metrics_snapshot" id: Mapped[int] = mapped_column(Integer, primary_key=True) - tag_id: Mapped[int] = mapped_column( - ForeignKey("tag.id", ondelete="CASCADE"), index=True + # Nullable, matching alembic 0060, which declared this column without + # `nullable=False`. The model had it as `Mapped[int]` — NOT NULL — which + # was simply never true of the database (#3275). Left nullable rather than + # tightened: a snapshot of a tag that is later hard-deleted is a row worth + # keeping, and the FK is ON DELETE CASCADE, so tightening it would only + # change behaviour, not correct a bug. + tag_id: Mapped[int | None] = mapped_column( + ForeignKey("tag.id", ondelete="CASCADE"), nullable=True, index=True ) # Denormalized so a snapshot stays readable even if the tag is later renamed. name: Mapped[str] = mapped_column(String(255), nullable=False) diff --git a/backend/app/models/image_provenance.py b/backend/app/models/image_provenance.py index fb18178..2ce95de 100644 --- a/backend/app/models/image_provenance.py +++ b/backend/app/models/image_provenance.py @@ -47,8 +47,15 @@ class ImageProvenance(Base): # attachment on the post. NULL for loose downloads and pre-backfill rows. # SET NULL so deleting the archive attachment never destroys the (image, # post) edge — it just forgets which archive it came from. + # FK named explicitly: the convention renders this + # `fk_image_provenance_from_attachment_id_post_attachment`, but alembic + # 0055 created it as `fk_image_provenance_from_attachment` (#3275). from_attachment_id: Mapped[int | None] = mapped_column( - ForeignKey("post_attachment.id", ondelete="SET NULL"), + ForeignKey( + "post_attachment.id", + ondelete="SET NULL", + name="fk_image_provenance_from_attachment", + ), nullable=True, index=True, ) captured_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index 3de0a3f..0fae950 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -18,6 +18,7 @@ from sqlalchemy import ( Integer, String, Text, + UniqueConstraint, func, text, ) @@ -33,6 +34,12 @@ class ImageRecord(Base): __table_args__ = ( + # alembic 0001. The database enforces sha256 uniqueness with a + # CONSTRAINT and carries a SEPARATE non-unique btree index; the model + # said `unique=True, index=True`, which collapses both into a single + # UNIQUE index under a different name. Same guarantee either way, but + # not the same objects, so autogenerate saw a drop and an add (#3275). + UniqueConstraint("sha256", name="uq_image_record_sha256"), # alembic 0036, and the last thing in this schema that lived only in a # migration. SQLAlchemy CAN express an hnsw index with an operator # class, so there is no reason for it to be invisible to the models — @@ -52,7 +59,9 @@ class ImageRecord(Base): # On-disk identity path: Mapped[str] = mapped_column(Text, nullable=False, unique=True) - sha256: Mapped[str] = mapped_column(String(64), nullable=False, unique=True, index=True) + # index=True only: the UNIQUE half is the named constraint in + # __table_args__ above, matching what 0001 actually created. + sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True) phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) mime: Mapped[str] = mapped_column(String(64), nullable=False) @@ -92,8 +101,15 @@ class ImageRecord(Base): ) # FC-2d-vii-c: canonical per-image artist (the single source of truth # for attribution; provenance posts remain lineage detail). + # FK named explicitly: the naming convention renders this + # `fk_image_record_artist_id_artist`, but alembic 0008 created it as + # `fk_image_record_artist_id` (#3275). artist_id: Mapped[int | None] = mapped_column( - ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True + ForeignKey( + "artist.id", ondelete="SET NULL", name="fk_image_record_artist_id" + ), + nullable=True, + index=True, ) # ML fields (populated by the ml-worker / GPU agent). 1152 = SigLIP-so400m diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index 74ac13f..78c0fab 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -4,7 +4,15 @@ Enforced as a single row via a CHECK (id = 1) constraint. The application always SELECTs id=1 and never inserts/deletes after the initial migration. """ -from sqlalchemy import Boolean, CheckConstraint, Float, Integer, Text, select +from sqlalchemy import ( + Boolean, + CheckConstraint, + Float, + Integer, + Text, + select, + text, +) from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -124,7 +132,9 @@ class ImportSettings(Base): # English (e.g. "… WIP Part 1") as a European language at ~0.86. CJK stays # trusted regardless (script-detected). Per-post overrides handle the misses. translation_min_confidence: Mapped[float] = mapped_column( - Float, nullable=False, default=0.9, server_default="0.9", + # text() because alembic 0084 used sa.text(); see ml_settings for why + # the form matters and why it is per-column (#3275). + Float, nullable=False, default=0.9, server_default=text("0.9"), ) # Title-based WIP auto-tagging (task #1458). When a freshly-imported post's diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index 3e1570a..d70449c 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -11,6 +11,7 @@ from sqlalchemy import ( String, func, select, + text, ) from sqlalchemy.orm import Mapped, mapped_column @@ -110,11 +111,16 @@ class MLSettings(Base): ) presentation_auto_apply_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.90, - server_default="0.90", + # text(), not a string, because alembic 0082 used sa.text(): a bare + # string renders DEFAULT '0.90'::double precision while text() renders + # DEFAULT 0.90, and the chain is MIXED — some migrations used one, + # some the other. Same value, different stored expression, so each + # column here mirrors whichever form its own migration used (#3275). + server_default=text("0.90"), ) presentation_conflict_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.50, - server_default="0.50", + server_default=text("0.50"), ) # -- Process auto-apply (#1464) ---------------------------------------- # `wip` / `editor screenshot` are PROCESS art — unfinished pieces + program @@ -133,11 +139,11 @@ class MLSettings(Base): ) process_auto_apply_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.90, - server_default="0.9", + server_default="0.90", ) process_conflict_threshold: Mapped[float] = mapped_column( Float, nullable=False, default=0.50, - server_default="0.5", + server_default="0.50", ) # Default = SigLIP 2 (so400m, 512px) for new installs (migration 0069); # existing libraries keep their stored value until the operator re-embeds. @@ -172,7 +178,7 @@ class MLSettings(Base): ) detector_person_conf: Mapped[float] = mapped_column( Float, nullable=False, default=0.35, - server_default="0.35", + server_default=text("0.35"), ) # anatomy: booru_yolo anime/furry/NSFW torso components → concept crops. # Default = yolov11m_aa22 (26 classes, best mAP50-95 0.96), committed in the @@ -192,7 +198,7 @@ class MLSettings(Base): ) detector_anatomy_conf: Mapped[float] = mapped_column( Float, nullable=False, default=0.30, - server_default="0.30", + server_default=text("0.30"), ) # panel: comic page → panel regions → concept crops (Apache-2.0, YOLOv12x). detector_panel_enabled: Mapped[bool] = mapped_column( @@ -206,7 +212,7 @@ class MLSettings(Base): ) detector_panel_conf: Mapped[float] = mapped_column( Float, nullable=False, default=0.30, - server_default="0.30", + server_default=text("0.30"), ) # Per-frame caps bound the crop→embed explosion; max_regions is the hard # per-job backstop; dedupe_iou drops near-duplicate crops before the embed. @@ -228,7 +234,7 @@ class MLSettings(Base): ) detector_dedupe_iou: Mapped[float] = mapped_column( Float, nullable=False, default=0.85, - server_default="0.85", + server_default=text("0.85"), ) # -- CCIP character prototypes (#1317) --------------------------------- # The per-character reference set is precomputed + refreshed INCREMENTALLY diff --git a/backend/app/models/series_chapter.py b/backend/app/models/series_chapter.py index ff7698a..87fc5c2 100644 --- a/backend/app/models/series_chapter.py +++ b/backend/app/models/series_chapter.py @@ -16,7 +16,14 @@ title is the optional chapter name; stated_part is the optional operator-facing from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, Integer, Text, func +from sqlalchemy import ( + DateTime, + ForeignKey, + Integer, + Text, + UniqueConstraint, + func, +) from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -25,14 +32,26 @@ from .base import Base class SeriesChapter(Base): __tablename__ = "series_chapter" + __table_args__ = ( + # alembic 0047 named the UNIQUE `uq_series_chapter_anchor_page`, not + # the `uq_series_chapter_anchor_page_id` a bare `unique=True` would + # render (#3275). + UniqueConstraint("anchor_page_id", name="uq_series_chapter_anchor_page"), + ) + id: Mapped[int] = mapped_column(Integer, primary_key=True) series_tag_id: Mapped[int] = mapped_column( ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True ) + # Both the UNIQUE (above) and the FK carry the names 0047 gave them; the + # convention would render the FK `fk_series_chapter_anchor_page_id_series_page`. anchor_page_id: Mapped[int] = mapped_column( - ForeignKey("series_page.id", ondelete="CASCADE"), + ForeignKey( + "series_page.id", + ondelete="CASCADE", + name="fk_series_chapter_anchor_page", + ), nullable=False, - unique=True, ) title: Mapped[str | None] = mapped_column(Text, nullable=True) stated_part: Mapped[int | None] = mapped_column(Integer, nullable=True) diff --git a/backend/app/models/series_page.py b/backend/app/models/series_page.py index 0bb0b40..22e19e1 100644 --- a/backend/app/models/series_page.py +++ b/backend/app/models/series_page.py @@ -14,7 +14,14 @@ number parsed from the source post, nullable when unknown. from datetime import datetime -from sqlalchemy import DateTime, ForeignKey, Integer, String, func +from sqlalchemy import ( + DateTime, + ForeignKey, + Integer, + String, + UniqueConstraint, + func, +) from sqlalchemy.orm import Mapped, mapped_column from .base import Base @@ -23,14 +30,22 @@ from .base import Base class SeriesPage(Base): __tablename__ = "series_page" + __table_args__ = ( + # alembic 0005 named this `uq_series_page_image`; a bare `unique=True` + # on the column renders `uq_series_page_image_id` under the naming + # convention, which is a different object from the one the database + # has (#3275). + UniqueConstraint("image_id", name="uq_series_page_image"), + ) + id: Mapped[int] = mapped_column(Integer, primary_key=True) series_tag_id: Mapped[int] = mapped_column( ForeignKey("tag.id", ondelete="CASCADE"), nullable=False, index=True ) + # UNIQUE lives in __table_args__ above, under the name 0005 gave it. image_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), nullable=False, - unique=True, ) # 'placed' = in the series-global run (page_number set); 'pending' = staged # from a post awaiting the operator's sort (page_number NULL). (#789 P2) diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 1c29e6f..8538bda 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -5,7 +5,16 @@ Multiple sources per artist support creators with cross-platform presence. from datetime import datetime -from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, JSON, String, Text +from sqlalchemy import ( + Boolean, + DateTime, + ForeignKey, + Integer, + JSON, + String, + Text, + UniqueConstraint, +) from sqlalchemy.orm import Mapped, mapped_column, relationship from .base import Base @@ -14,6 +23,20 @@ from .base import Base class Source(Base): __tablename__ = "source" + __table_args__ = ( + # alembic 0010. One row per (artist, platform, url): re-adding a source + # the artist already has is an update, not a second row. The model had + # never declared it (#3275), so autogenerate would have proposed + # DROPPING it — the guarantee existed only in the migration chain. + # + # Named explicitly because the naming convention would render this + # `uq_source_artist_id` (uq keys off column_0_name), which is both + # wrong about the shape and not what the database actually has. + UniqueConstraint( + "artist_id", "platform", "url", name="uq_source_artist_platform_url" + ), + ) + id: Mapped[int] = mapped_column(Integer, primary_key=True) artist_id: Mapped[int] = mapped_column( ForeignKey("artist.id", ondelete="CASCADE"), nullable=False, index=True diff --git a/backend/app/models/tag_suggestion_rejection.py b/backend/app/models/tag_suggestion_rejection.py index 02756de..341f67f 100644 --- a/backend/app/models/tag_suggestion_rejection.py +++ b/backend/app/models/tag_suggestion_rejection.py @@ -19,11 +19,19 @@ class TagSuggestionRejection(Base): # Named explicitly; see tag_alias for why (#3275). Index("ix_tag_suggestion_rejection_tag", "tag_id"), ) + # Both FKs named explicitly. alembic 0003 used a hand-shortened `tsr` + # prefix; the convention would render the full table name (#3275). image_record_id: Mapped[int] = mapped_column( - ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True + ForeignKey( + "image_record.id", + ondelete="CASCADE", + name="fk_tsr_image_record_id_image_record", + ), + primary_key=True, ) tag_id: Mapped[int] = mapped_column( - ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True + ForeignKey("tag.id", ondelete="CASCADE", name="fk_tsr_tag_id_tag"), + primary_key=True, ) rejected_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, server_default=func.now() -- 2.54.0 From b979062dd7fdb7678d3a439640b400cda13917a5 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 31 Aug 2026 00:29:57 -0400 Subject: [PATCH 14/17] db: rename the four double-prefixed CHECK constraints (#3275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 5026 got the models-vs-chain diff to 7 lines. Three findings, and one of them reverses an assumption I made in the previous commit. The doubled CHECK names are what the DATABASE has, not what the generator invented. base.py's convention is ck_%(table_name)s_%(constraint_name)s, which — unlike uq/fk/ix — applies even to a constraint that already has a name, so four migrations that passed an already-prefixed name got it prefixed twice: ck_import_settings_ck_import_settings_singleton ck_ml_settings_ck_ml_settings_singleton ck_post_ck_post_translation_override ck_tag_ck_tag_fandom_requires_character The workflow repair added last commit is still correct and still needed — autogenerate really does re-double a name on the round trip — but it was making the MODELS side clean against a chain that is dirty. The comment in ml_settings.py claiming its bare name "matches migration 0003" was simply false; 0003 produces the doubled form. Nothing reads a CHECK constraint by name, so this has never done harm. But it is precisely the development-era residue the collapsed baseline exists to leave behind, and a public schema should not ship it — so 0088 renames the deployed constraints and all six models now declare bare names. RENAME CONSTRAINT is catalog-only: no scan, no rewrite, no revalidation, which is why this is safe on post and tag. Guarded on pg_constraint scoped by conrelid, so it is a no-op on a database built from the models. ix_tag_fandom_id showed as a difference only because chain_ref was pinned to 0a5bbe8, which predates 0088 — the comparison was measuring the models against a chain missing the migration that closes the gap. chain_ref now defaults to blank, meaning "the chain in this ref". Pin it to a commit only after the collapse, when the tree no longer carries the revisions. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017QHszn9H8VBvx5Ke8x1hvw --- .forgejo/workflows/baseline.yml | 11 +++- .../0088_reconcile_models_with_schema.py | 65 ++++++++++++++++++- backend/app/models/external_link.py | 8 ++- backend/app/models/import_settings.py | 5 ++ backend/app/models/ml_settings.py | 5 +- backend/app/models/post.py | 6 +- backend/app/models/tag.py | 6 +- 7 files changed, 96 insertions(+), 10 deletions(-) diff --git a/.forgejo/workflows/baseline.yml b/.forgejo/workflows/baseline.yml index 4681a35..7b49421 100644 --- a/.forgejo/workflows/baseline.yml +++ b/.forgejo/workflows/baseline.yml @@ -34,9 +34,9 @@ on: workflow_dispatch: inputs: chain_ref: - description: 'Commit/tag that still carries the full 0001..0087 chain' + description: 'Commit/tag carrying the full chain; blank = this ref (use a pinned commit only AFTER the collapse)' type: string - default: '0a5bbe8' + default: '' mode: description: 'chain = compare against this tree''s migrations; models = compare against a schema built from the MODELS' type: string @@ -100,10 +100,15 @@ jobs: - name: Build the schema the OLD chain produces env: CHAIN_REF: ${{ github.event.inputs.chain_ref }} + THIS_SHA: ${{ github.sha }} run: | set -eux docker exec "$PG_CONTAINER" createdb -U fabledcurator fc_chain - git worktree add /tmp/chain "$CHAIN_REF" + # Blank means "the chain in this ref", which is what you want while + # the chain is still intact — comparing the models against a PINNED + # older commit reports every migration written since as a difference. + # Pin it only after the collapse, when the tree no longer has them. + git worktree add /tmp/chain "${CHAIN_REF:-$THIS_SHA}" ls /tmp/chain/alembic/versions/*.py | wc -l cd /tmp/chain DB_NAME=fc_chain alembic upgrade head diff --git a/alembic/versions/0088_reconcile_models_with_schema.py b/alembic/versions/0088_reconcile_models_with_schema.py index a286c6c..30020dc 100644 --- a/alembic/versions/0088_reconcile_models_with_schema.py +++ b/alembic/versions/0088_reconcile_models_with_schema.py @@ -6,8 +6,8 @@ schema disagreed. Almost all of them were the MODEL being wrong — missing migration — and those are fixed in the model files with no DDL at all, because the database already had them. -This migration carries the remainder: the one case where the MODEL was right -and the database was missing something. +This migration carries the remainder — the two places where DDL is actually +needed, because the database is what is wrong. `tag.fandom_id` is declared `index=True` on the model, but no migration ever created that index. Every autogenerate run since would have proposed adding @@ -29,6 +29,29 @@ guarantee built from different objects, which is why the two schemas did not line up. The model now declares the constraint and the plain index separately, so it describes what is actually there. No DDL is needed for it. +Also here: four CHECK constraints whose names carry their table prefix TWICE. + +`base.py`'s naming convention is `ck_%(table_name)s_%(constraint_name)s`, and +unlike the uq/fk/ix entries it applies even to a constraint that already has a +name. Four migrations passed an already-prefixed name, so the convention +prefixed it again: + + ck_import_settings_ck_import_settings_singleton + ck_ml_settings_ck_ml_settings_singleton + ck_post_ck_post_translation_override + ck_tag_ck_tag_fandom_requires_character + +Nothing reads a CHECK constraint by name, so this has never done any harm — +but it is exactly the development-era residue the collapsed baseline exists to +leave behind, and a public schema should not ship it. The models now declare +bare names, which the convention renders into the single-prefix form; this +renames the deployed constraints to match. + +RENAME CONSTRAINT is a catalog-only operation: no table scan, no rewrite, no +validation of existing rows. It takes a brief ACCESS EXCLUSIVE lock and +returns. That is why this is safe to do on `post` and `tag`, which are the two +large tables in the schema. + Revision ID: 0088 Revises: 0087 Create Date: 2026-08-30 @@ -43,6 +66,38 @@ down_revision: Union[str, None] = "0087" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None +# (table, doubled name, single-prefix name) +DOUBLED_CHECKS = ( + ("import_settings", "ck_import_settings_ck_import_settings_singleton", + "ck_import_settings_singleton"), + ("ml_settings", "ck_ml_settings_ck_ml_settings_singleton", + "ck_ml_settings_singleton"), + ("post", "ck_post_ck_post_translation_override", + "ck_post_translation_override"), + ("tag", "ck_tag_ck_tag_fandom_requires_character", + "ck_tag_fandom_requires_character"), +) + + +def _rename_check(table: str, old: str, new: str) -> None: + # Guarded on pg_constraint rather than run bare: a database built from the + # models (a fresh install, or the CI integration schema) already has the + # single-prefix name, and this migration must be a no-op there rather than + # an error. Same reasoning as the CREATE INDEX IF NOT EXISTS below. + op.execute( + f""" + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = '{old}' AND conrelid = '{table}'::regclass + ) THEN + ALTER TABLE {table} RENAME CONSTRAINT {old} TO {new}; + END IF; + END $$; + """ + ) + def upgrade() -> None: # IF NOT EXISTS because the index is what the model already asks for: any @@ -50,6 +105,12 @@ def upgrade() -> None: # and this migration must be a no-op there rather than an error. op.execute("CREATE INDEX IF NOT EXISTS ix_tag_fandom_id ON tag (fandom_id)") + for table, old, new in DOUBLED_CHECKS: + _rename_check(table, old, new) + def downgrade() -> None: + for table, old, new in DOUBLED_CHECKS: + _rename_check(table, new, old) + op.execute("DROP INDEX IF EXISTS ix_tag_fandom_id") diff --git a/backend/app/models/external_link.py b/backend/app/models/external_link.py index dcf8ee8..b06bf9b 100644 --- a/backend/app/models/external_link.py +++ b/backend/app/models/external_link.py @@ -43,11 +43,15 @@ class ExternalLink(Base): # needs its constraint swapped in the same migration (#3275). CheckConstraint( "host IN ('mega', 'gdrive', 'mediafire', 'dropbox', 'pixeldrain')", - name="ck_external_link_host", + # Bare name: Base.metadata's naming convention prepends + # ck_
_. Pre-prefixing it here doubles the prefix — see + # alembic 0088, which renames the four constraints that shipped + # that way (#3275). + name="host", ), CheckConstraint( "status IN ('pending', 'downloading', 'downloaded', 'failed', 'skipped', 'dead')", - name="ck_external_link_status", + name="status", ), # One row per (post, url). The full url (incl. #fragment) is the identity # — the same file linked twice in a post collapses to one row. diff --git a/backend/app/models/import_settings.py b/backend/app/models/import_settings.py index 78c0fab..83ae9c2 100644 --- a/backend/app/models/import_settings.py +++ b/backend/app/models/import_settings.py @@ -22,6 +22,11 @@ class ImportSettings(Base): __tablename__ = "import_settings" # Bare constraint name — Base.metadata's naming convention applies the # ck_
_ prefix, producing the final ck_import_settings_singleton. + # Bare name — Base.metadata's naming convention prepends ck_
_, + # producing ck_import_settings_singleton. The chain shipped the DOUBLED + # ck_import_settings_ck_import_settings_singleton, because the migration + # pre-prefixed the name and the convention prefixed it again; alembic + # 0088 renames it to what this line has always produced (#3275). __table_args__ = (CheckConstraint("id = 1", name="singleton"),) id: Mapped[int] = mapped_column(Integer, primary_key=True) diff --git a/backend/app/models/ml_settings.py b/backend/app/models/ml_settings.py index d70449c..4705d1a 100644 --- a/backend/app/models/ml_settings.py +++ b/backend/app/models/ml_settings.py @@ -21,7 +21,10 @@ from .base import Base class MLSettings(Base): __tablename__ = "ml_settings" # Bare name — Base.metadata's naming convention prepends ck_
_, - # producing the final ck_ml_settings_singleton (matches migration 0003). + # producing ck_ml_settings_singleton. The chain shipped the DOUBLED + # ck_ml_settings_ck_ml_settings_singleton, because the migration + # pre-prefixed the name and the convention prefixed it again; alembic + # 0088 renames it to what this line has always produced (#3275). __table_args__ = (CheckConstraint("id = 1", name="singleton"),) id: Mapped[int] = mapped_column(Integer, primary_key=True) diff --git a/backend/app/models/post.py b/backend/app/models/post.py index 1cc15a4..6b4ec33 100644 --- a/backend/app/models/post.py +++ b/backend/app/models/post.py @@ -41,7 +41,11 @@ class Post(Base): UniqueConstraint("source_id", "external_post_id", name="uq_post_source_external_id"), CheckConstraint( "translation_override IN ('auto', 'force', 'original')", - name="ck_post_translation_override", + # Bare name: Base.metadata's naming convention prepends + # ck_
_. Pre-prefixing it here doubles the prefix — see + # alembic 0088, which renames the four constraints that shipped + # that way (#3275). + name="translation_override", ), ) diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index d3107d0..cc23686 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -83,7 +83,11 @@ class Tag(Base): unique=True), CheckConstraint( "(fandom_id IS NULL) OR (kind = 'character')", - name="ck_tag_fandom_requires_character", + # Bare name: Base.metadata's naming convention prepends + # ck_
_. Pre-prefixing it here doubles the prefix — see + # alembic 0088, which renames the four constraints that shipped + # that way (#3275). + name="fandom_requires_character", ), ) -- 2.54.0 From 389afe2f7bab59eabe8cb9addd40b5c4419b473c Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 31 Aug 2026 00:33:50 -0400 Subject: [PATCH 15/17] db: the doubled CHECK list was six, not four (#3275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run 5029 confirmed the four renames landed and surfaced two I had missed: external_link's host and status CHECKs are doubled the same way. They did not show in run 5026's diff because BOTH sides produced the doubled form back then — external_link.py pre-prefixed its names, so the models matched the chain's mistake. Switching all six models to bare names is what exposed the two the migration did not cover. The list in the file now comes from matching ck_(\w+?)_ck_\1_ against the chain's own pg_dump, rather than from reading migrations by eye. Reading by eye is what missed these, in the same way it earlier missed a UNIQUE constraint sitting two lines above the index being looked at. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017QHszn9H8VBvx5Ke8x1hvw --- .../0088_reconcile_models_with_schema.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/alembic/versions/0088_reconcile_models_with_schema.py b/alembic/versions/0088_reconcile_models_with_schema.py index 30020dc..d5b84c2 100644 --- a/alembic/versions/0088_reconcile_models_with_schema.py +++ b/alembic/versions/0088_reconcile_models_with_schema.py @@ -29,13 +29,15 @@ guarantee built from different objects, which is why the two schemas did not line up. The model now declares the constraint and the plain index separately, so it describes what is actually there. No DDL is needed for it. -Also here: four CHECK constraints whose names carry their table prefix TWICE. +Also here: six CHECK constraints whose names carry their table prefix TWICE. `base.py`'s naming convention is `ck_%(table_name)s_%(constraint_name)s`, and unlike the uq/fk/ix entries it applies even to a constraint that already has a -name. Four migrations passed an already-prefixed name, so the convention +name. Six migrations passed an already-prefixed name, so the convention prefixed it again: + ck_external_link_ck_external_link_host + ck_external_link_ck_external_link_status ck_import_settings_ck_import_settings_singleton ck_ml_settings_ck_ml_settings_singleton ck_post_ck_post_translation_override @@ -67,7 +69,17 @@ branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None # (table, doubled name, single-prefix name) +# +# Six, not the four a first read of the migrations turned up. The list that +# settles it is the one extracted from the chain's pg_dump by matching +# `ck_(\w+?)_ck_\1_` — reading the migrations by eye missed external_link +# twice over, in the same way an earlier pass missed a UNIQUE constraint two +# lines above the index it was looking at (see the sha256 note above). DOUBLED_CHECKS = ( + ("external_link", "ck_external_link_ck_external_link_host", + "ck_external_link_host"), + ("external_link", "ck_external_link_ck_external_link_status", + "ck_external_link_status"), ("import_settings", "ck_import_settings_ck_import_settings_singleton", "ck_import_settings_singleton"), ("ml_settings", "ck_ml_settings_ck_ml_settings_singleton", -- 2.54.0 From b1bd2531add8f7468adf4856c495fa9f03742d21 Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 31 Aug 2026 00:38:46 -0400 Subject: [PATCH 16/17] style: sort JSON first in three sqlalchemy import blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ruff's isort runs with order-by-type, which sorts ALL_CAPS names ahead of CamelCase ones, so `JSON` belongs at the head of the list rather than between `Integer` and `String`. Two of these (backup_run.py, post.py) have been failing lint since 5e1996e — I did not check the push CI after that commit, only the baseline workflow I had dispatched, so ci.yml has been red on dev across 5e1996e, ed2b1ad and d044e93. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017QHszn9H8VBvx5Ke8x1hvw --- backend/app/models/backup_run.py | 2 +- backend/app/models/post.py | 2 +- backend/app/models/source.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/app/models/backup_run.py b/backend/app/models/backup_run.py index ad7b4f0..787e78c 100644 --- a/backend/app/models/backup_run.py +++ b/backend/app/models/backup_run.py @@ -20,7 +20,7 @@ feedback_check_existing_enums): from datetime import datetime -from sqlalchemy import BigInteger, DateTime, ForeignKey, Index, Integer, JSON, String, Text, text +from sqlalchemy import JSON, BigInteger, DateTime, ForeignKey, Index, Integer, String, Text, text from sqlalchemy.orm import Mapped, mapped_column from .base import Base diff --git a/backend/app/models/post.py b/backend/app/models/post.py index 6b4ec33..e3f8e5e 100644 --- a/backend/app/models/post.py +++ b/backend/app/models/post.py @@ -9,12 +9,12 @@ artist-filter queries don't depend on the Source detour). from datetime import datetime from sqlalchemy import ( + JSON, CheckConstraint, DateTime, ForeignKey, Index, Integer, - JSON, String, Text, UniqueConstraint, diff --git a/backend/app/models/source.py b/backend/app/models/source.py index 8538bda..3501505 100644 --- a/backend/app/models/source.py +++ b/backend/app/models/source.py @@ -6,11 +6,11 @@ Multiple sources per artist support creators with cross-platform presence. from datetime import datetime from sqlalchemy import ( + JSON, Boolean, DateTime, ForeignKey, Integer, - JSON, String, Text, UniqueConstraint, -- 2.54.0 From 08418d54a3270bd0dea0e91430fda4ea0e059bdd Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Mon, 31 Aug 2026 08:25:41 -0400 Subject: [PATCH 17/17] db: index the seven unindexed FKs, drop the seven redundant ones (#3300, #3301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A structural sweep of the deployed schema, run AFTER 0088 got the models and the chain to exact agreement. That agreement is what 0088 achieved, and it is worth naming what it does not prove: a models-vs-chain diff shows the two describe the same schema, not that the schema is right. Everything here was wrong in BOTH. The one that matters: image_tag has PRIMARY KEY (image_record_id, tag_id) and no other index, so tag_id is unindexed. That is the gallery's tag filter (tag_query.py builds `image_tag.c.tag_id == tid`) and the ON DELETE CASCADE from tag, both scanning the largest table in the schema. Six more FKs were unindexed on smaller tables; presentation_review.tag_id also CASCADEs. Dropped, on the other side: ix_image_record_sha256 was an exact duplicate of the index uq_image_record_sha256 already builds — two btrees on the same column of the highest-insert-rate table. The other six are single-column indexes a later composite superseded without the narrow one being retired; a btree on (a,b) already serves lookups on a. 0088 deliberately taught the models to declare BOTH sha256 indexes so they would describe reality. This changes the reality instead, and the models change with it — otherwise the next baseline.yml run reintroduces exactly the drift 0088 removed. CONCURRENTLY throughout, so building the image_tag index does not hold an ACCESS EXCLUSIVE lock over every write for the duration. The cost is that the migration cannot run in a transaction and so is not atomic: every statement is IF NOT EXISTS / IF EXISTS, making a re-run after a partial failure safe. The docstring carries the query for finding an INVALID index left by an interrupted CONCURRENTLY build. What the sweep found clean, for the record: all 43 tables have a primary key; all 51 FKs declare an explicit ON DELETE, so none silently blocks a delete; the three enum CHECKs match the code that writes them (rule 36). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017QHszn9H8VBvx5Ke8x1hvw --- alembic/versions/0089_index_hygiene.py | 120 ++++++++++++++++++++++ backend/app/models/backup_run.py | 11 +- backend/app/models/character_prototype.py | 4 +- backend/app/models/external_link.py | 6 +- backend/app/models/image_record.py | 8 +- backend/app/models/import_task.py | 2 + backend/app/models/presentation_review.py | 4 + backend/app/models/tag.py | 6 ++ backend/app/models/task_run.py | 9 +- 9 files changed, 159 insertions(+), 11 deletions(-) create mode 100644 alembic/versions/0089_index_hygiene.py diff --git a/alembic/versions/0089_index_hygiene.py b/alembic/versions/0089_index_hygiene.py new file mode 100644 index 0000000..0b22b5c --- /dev/null +++ b/alembic/versions/0089_index_hygiene.py @@ -0,0 +1,120 @@ +"""Index the seven unindexed FKs; drop the seven redundant indexes (#3300, #3301). + +Found by a structural sweep of the deployed schema done AFTER 0088 brought the +models and the migration chain into exact agreement. That agreement is what +0088 achieved, and it is worth being precise about what it does NOT prove: a +models-vs-chain diff shows the two describe the same schema. It says nothing +about whether that schema is right. Everything here was wrong in BOTH, which is +exactly the class of problem the reconciliation could not see. + +## Added: seven FK indexes + +`image_tag.tag_id` is the one that matters. The table's only index is +PRIMARY KEY (image_record_id, tag_id), which leads with the wrong column for +the two hottest things done with it: + + * the gallery's tag filter — services/tag_query.py builds + `image_tag.c.tag_id == tid` (and `.in_(tids)`) on every tag-scoped browse; + * ON DELETE CASCADE from `tag` — deleting or merging a tag makes Postgres + find that tag's rows before it can remove them. + +Both had to scan the largest table in the schema. The other six are the same +shape on much smaller tables; `presentation_review.tag_id` is the notable one, +since it also CASCADEs. + +## Dropped: seven redundant indexes + +`ix_image_record_sha256` was an exact duplicate. A UNIQUE constraint builds its +own index, so `uq_image_record_sha256` already covered the column and +`image_record` carried two btrees on `sha256` — on the highest-insert-rate +table in the system. + +The other six are single-column indexes that a later composite superseded +without the narrow one being retired. A btree on (a, b) already serves lookups +on `a`, so each was pure write amplification. `task_run` and `backup_run` are +append-heavy operational logs, which is where that cost lands hardest. + +Note for anyone reading 0088 next to this: 0088 deliberately taught the models +to declare BOTH sha256 indexes, so they would describe reality. That was right. +This migration changes the reality instead, and the models change with it. + +## CONCURRENTLY, and why this migration has no transaction + +`CREATE INDEX` takes an ACCESS EXCLUSIVE lock for the whole build, which on +`image_tag` means stalling every write for as long as it takes. CONCURRENTLY +builds without blocking writers, at the cost of two table passes and an +inability to run inside a transaction — hence `autocommit_block()`. + +The consequence to know about: this migration is NOT atomic. If it fails +partway, the work already done stays done. Every statement is therefore written +IF NOT EXISTS / IF EXISTS so that re-running it after a failure is safe rather +than an error. + +A failed CONCURRENTLY build also leaves an INVALID index behind — it is not +used by the planner and not repaired automatically. Find them with: + + SELECT c.relname FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + WHERE NOT i.indisvalid; + +Drop what that returns and re-run; nothing else is needed. + +Revision ID: 0089 +Revises: 0088 +Create Date: 2026-08-31 + +""" +from typing import Sequence, Union + +from alembic import op + +revision: str = "0089" +down_revision: Union[str, None] = "0088" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +# (index name, table, column) — names match what the models render under +# base.py's naming convention, so autogenerate stays quiet after this. +MISSING_FK_INDEXES = ( + ("ix_image_tag_tag_id", "image_tag", "tag_id"), + ("ix_presentation_review_tag_id", "presentation_review", "tag_id"), + ("ix_presentation_review_conflict_tag_id", "presentation_review", "conflict_tag_id"), + ("ix_import_task_result_image_id", "import_task", "result_image_id"), + ("ix_external_link_attachment_id", "external_link", "attachment_id"), + ("ix_character_prototype_region_id", "character_prototype", "region_id"), + ("ix_backup_run_restored_from_id", "backup_run", "restored_from_id"), +) + +# (index name, table, column) — redundant; the second element of each pair in +# the docstring is what still covers the column after the drop. +REDUNDANT_INDEXES = ( + ("ix_image_record_sha256", "image_record", "sha256"), + ("ix_backup_run_kind", "backup_run", "kind"), + ("ix_backup_run_status", "backup_run", "status"), + ("ix_task_run_queue", "task_run", "queue"), + ("ix_task_run_status", "task_run", "status"), + ("ix_task_run_task_name", "task_run", "task_name"), + ("ix_external_link_post_id", "external_link", "post_id"), +) + + +def upgrade() -> None: + with op.get_context().autocommit_block(): + for name, table, column in MISSING_FK_INDEXES: + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} " + f"ON {table} ({column})" + ) + for name, _table, _column in REDUNDANT_INDEXES: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") + + +def downgrade() -> None: + with op.get_context().autocommit_block(): + for name, table, column in REDUNDANT_INDEXES: + op.execute( + f"CREATE INDEX CONCURRENTLY IF NOT EXISTS {name} " + f"ON {table} ({column})" + ) + for name, _table, _column in MISSING_FK_INDEXES: + op.execute(f"DROP INDEX CONCURRENTLY IF EXISTS {name}") diff --git a/backend/app/models/backup_run.py b/backend/app/models/backup_run.py index 787e78c..d5e54cd 100644 --- a/backend/app/models/backup_run.py +++ b/backend/app/models/backup_run.py @@ -37,9 +37,12 @@ class BackupRun(Base): Index("ix_backup_run_tag_partial", "tag", postgresql_where=text("tag IS NOT NULL")), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) - kind: Mapped[str] = mapped_column(String(16), nullable=False, index=True) + # No index=True: ix_backup_run_kind_started (above) already leads with + # `kind`, so a single-column index on it was pure write cost (#3301). + kind: Mapped[str] = mapped_column(String(16), nullable=False) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="pending", index=True, + # No index=True — ix_backup_run_status_finished leads with `status`. + String(16), nullable=False, default="pending", server_default="pending", ) tag: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) @@ -57,7 +60,9 @@ class BackupRun(Base): manifest: Mapped[dict] = mapped_column( JSON, nullable=False, default=dict, server_default="{}", ) + # Self-referential FK, unindexed until 0089 (#3300): SET NULL has to find + # the rows pointing at a deleted run before it can null them. restored_from_id: Mapped[int | None] = mapped_column( ForeignKey("backup_run.id", ondelete="SET NULL"), - nullable=True, + nullable=True, index=True, ) diff --git a/backend/app/models/character_prototype.py b/backend/app/models/character_prototype.py index 191a29d..f281251 100644 --- a/backend/app/models/character_prototype.py +++ b/backend/app/models/character_prototype.py @@ -40,8 +40,10 @@ class CharacterPrototype(Base): ) # Provenance: the region this vector was copied from. SET NULL so pruning a # region doesn't delete the prototype mid-cycle (the next refresh reconciles). + # index=True added in 0089 — the FK was unindexed (#3300). region_id: Mapped[int | None] = mapped_column( - ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True + ForeignKey("image_region.id", ondelete="SET NULL"), nullable=True, + index=True, ) diff --git a/backend/app/models/external_link.py b/backend/app/models/external_link.py index b06bf9b..4882125 100644 --- a/backend/app/models/external_link.py +++ b/backend/app/models/external_link.py @@ -57,11 +57,15 @@ class ExternalLink(Base): # — the same file linked twice in a post collapses to one row. Index("uq_external_link_post_url", "post_id", "url", unique=True), Index("ix_external_link_status", "status"), + # Unindexed FK (#3300). + Index("ix_external_link_attachment_id", "attachment_id"), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) + # No index=True: uq_external_link_post_url (post_id, url) already leads + # with post_id (#3301). post_id: Mapped[int] = mapped_column( - ForeignKey("post.id", ondelete="CASCADE"), nullable=False, index=True + ForeignKey("post.id", ondelete="CASCADE"), nullable=False ) artist_id: Mapped[int | None] = mapped_column( ForeignKey("artist.id", ondelete="SET NULL"), nullable=True, index=True diff --git a/backend/app/models/image_record.py b/backend/app/models/image_record.py index 0fae950..f5f4050 100644 --- a/backend/app/models/image_record.py +++ b/backend/app/models/image_record.py @@ -59,9 +59,11 @@ class ImageRecord(Base): # On-disk identity path: Mapped[str] = mapped_column(Text, nullable=False, unique=True) - # index=True only: the UNIQUE half is the named constraint in - # __table_args__ above, matching what 0001 actually created. - sha256: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + # Neither unique= nor index=: uq_image_record_sha256 in __table_args__ + # above creates its own index, and the separate ix_image_record_sha256 + # that 0001 also built was an exact duplicate of it — dropped in 0089 + # (#3301). Lookups by sha256 use the constraint's index. + sha256: Mapped[str] = mapped_column(String(64), nullable=False) phash: Mapped[str | None] = mapped_column(String(32), nullable=True, index=True) size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) mime: Mapped[str] = mapped_column(String(64), nullable=False) diff --git a/backend/app/models/import_task.py b/backend/app/models/import_task.py index c3d9f11..e7921e6 100644 --- a/backend/app/models/import_task.py +++ b/backend/app/models/import_task.py @@ -31,6 +31,8 @@ class ImportTask(Base): __table_args__ = ( Index("ix_import_task_created_at_desc", text("created_at DESC")), + # Unindexed FK (#3300). + Index("ix_import_task_result_image_id", "result_image_id"), ) id: Mapped[int] = mapped_column(Integer, primary_key=True) batch_id: Mapped[int] = mapped_column( diff --git a/backend/app/models/presentation_review.py b/backend/app/models/presentation_review.py index e18e298..3e83b21 100644 --- a/backend/app/models/presentation_review.py +++ b/backend/app/models/presentation_review.py @@ -23,6 +23,10 @@ class PresentationReview(Base): __table_args__ = ( Index("ix_presentation_review_resolved_at", "resolved_at"), + # Both FKs to tag were unindexed (#3300); tag_id CASCADEs, so a tag + # delete had to scan this table to find its rows. + Index("ix_presentation_review_tag_id", "tag_id"), + Index("ix_presentation_review_conflict_tag_id", "conflict_tag_id"), ) image_record_id: Mapped[int] = mapped_column( ForeignKey("image_record.id", ondelete="CASCADE"), primary_key=True diff --git a/backend/app/models/tag.py b/backend/app/models/tag.py index cc23686..1b85936 100644 --- a/backend/app/models/tag.py +++ b/backend/app/models/tag.py @@ -71,6 +71,12 @@ image_tag = Table( Column("tag_id", ForeignKey("tag.id", ondelete="CASCADE"), primary_key=True), Column("source", String(32), nullable=False, default="manual", server_default="manual"), Column("created_at", DateTime(timezone=True), nullable=False, server_default=func.now()), + # The PK is (image_record_id, tag_id), which leads with the WRONG column + # for the two things that matter most here (#3300): the gallery's tag + # filter (tag_query.py builds `image_tag.c.tag_id == tid`) and the + # ON DELETE CASCADE from tag, which has to find a tag's rows to remove + # them. Without this index both scan the largest table in the schema. + Index("ix_image_tag_tag_id", "tag_id"), ) diff --git a/backend/app/models/task_run.py b/backend/app/models/task_run.py index c24ed46..2cd8b70 100644 --- a/backend/app/models/task_run.py +++ b/backend/app/models/task_run.py @@ -35,8 +35,10 @@ class TaskRun(Base): celery_task_id: Mapped[str] = mapped_column( String(64), nullable=False, index=True, ) - queue: Mapped[str] = mapped_column(String(32), nullable=False, index=True) - task_name: Mapped[str] = mapped_column(String(128), nullable=False, index=True) + # Neither carries index=True: ix_task_run_queue_started and + # ix_task_run_name_started already lead with these columns (#3301). + queue: Mapped[str] = mapped_column(String(32), nullable=False) + task_name: Mapped[str] = mapped_column(String(128), nullable=False) target_id: Mapped[int | None] = mapped_column(Integer, nullable=True) started_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), nullable=False, index=True, @@ -46,7 +48,8 @@ class TaskRun(Base): ) duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True) status: Mapped[str] = mapped_column( - String(16), nullable=False, default="running", index=True, + # No index=True — ix_task_run_status_started leads with `status`. + String(16), nullable=False, default="running", server_default="running", ) error_type: Mapped[str | None] = mapped_column(String(128), nullable=True) -- 2.54.0