Compare commits
41
Commits
+240
-83
@@ -142,6 +142,16 @@ concurrency:
|
||||
# Deriving it per job invites the two halves to disagree: sign-extension would
|
||||
# derive dev's extension version while build-web bundled main's, and the
|
||||
# release download would 404 on a version that exists perfectly well.
|
||||
#
|
||||
# On every other trigger it is the COMMIT that fired (`github.sha`), never the
|
||||
# branch name. A branch is re-resolved by each job's checkout when that job
|
||||
# starts, so a push landing mid-run moved the later jobs onto the new tip: run
|
||||
# 7499 signed 423275a's extension, then build-web checked out 83e1382 (pushed
|
||||
# while 7499 ran), derived a version nobody had signed, and 404'd on the
|
||||
# download (#4427). The guard failed closed that time; a job without one would
|
||||
# have published a commit the run's own lanes never tested — the lanes check
|
||||
# out `github.sha` by default, so pinning here makes the publish build exactly
|
||||
# what they passed.
|
||||
# IS THIS A BASE REFRESH? Asked in five places and previously spelled five
|
||||
# ways — `github.event_name == 'schedule'` in an `if:`, `$GITHUB_EVENT_NAME` in
|
||||
# one shell, an `EVENT:` env passed into another, and a bare expression on
|
||||
@@ -178,7 +188,16 @@ concurrency:
|
||||
# where a step-level `if:` needs the answer before any shell runs.
|
||||
env:
|
||||
IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }}
|
||||
BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.ref }}
|
||||
BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.sha }}
|
||||
# What the six LANES check out. Empty — the checkout default, the triggering
|
||||
# commit (or a PR's merge ref) — on every trigger but the refresh, where it is
|
||||
# `main`: the refresh publishes main, so the gate has to test main (#4430). It
|
||||
# is not BUILD_REF itself because a pull_request run's `github.sha` is a merge
|
||||
# commit the default checkout reaches through its ref, not by sha. Each job
|
||||
# still resolves `main` when it starts, so a merge to main during the ~5 min of
|
||||
# a Sunday-06:00 refresh could put the lanes and the build one commit apart;
|
||||
# the build jobs' own guards assert the branch, not the commit.
|
||||
LANE_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || '' }}
|
||||
|
||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||
@@ -219,6 +238,8 @@ jobs:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
- name: Ruff lint
|
||||
# agent/ included so the GPU-agent is linted before its image is built
|
||||
# (build.yml only `docker build`s it — this is where it gets checked).
|
||||
@@ -242,7 +263,7 @@ jobs:
|
||||
# rather than left running beside the new mechanism (rule 22).
|
||||
#
|
||||
# Two things are still worth asserting, and this is the only lane that can:
|
||||
# the extension.yml suite runs on node:24-slim, which is exactly why
|
||||
# the extension-test lane runs on node:24-slim, which is exactly why
|
||||
# version.spec.js sticks to packaging.sh's git-free subcommands.
|
||||
# 1. the derivation actually resolves on this commit
|
||||
# 2. the derived string is one AMO will accept, checked against Mozilla's
|
||||
@@ -265,6 +286,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# The derivation needs real history: a depth-1 clone sees one commit
|
||||
# and produces a wrong, too-low value RATHER THAN FAILING. Checking
|
||||
# that here is half the point of the lane.
|
||||
@@ -320,6 +342,7 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# Full history for tests/test_artifact_identity.py, which derives
|
||||
# each artifact's revision to check the identity scheme. On a
|
||||
# depth-1 clone that derivation either fails or returns the tip sha
|
||||
@@ -366,6 +389,8 @@ jobs:
|
||||
working-directory: frontend
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||
# feedback-no-local-runs). Using `npm install` instead of `npm ci`.
|
||||
# If we want strict lockfile-based reproducibility later, commit a
|
||||
@@ -378,6 +403,78 @@ jobs:
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
# The extension's lane: web-ext lint, the vitest suite, and the check that
|
||||
# asks web-ext what it ACTUALLY packaged. Moved in from `extension.yml`
|
||||
# (deleted) at milestone 429 — the same move `ci.yml` made on 2026-09-23 and
|
||||
# for the same reason: a separate workflow cannot gate this one, so a red
|
||||
# extension suite still let `sign-extension` sign and `build-web` ship the
|
||||
# XPI. It is a lane in THE GATE now. Its old path filter is gone with it; a
|
||||
# filter here would make the lane skip, and a skipped lane blocks the publish.
|
||||
#
|
||||
# The vitest suite is also the JS half of the #3093 artist-pattern mirror —
|
||||
# the Python half runs in backend-lint-and-test — so until this move one half
|
||||
# of that guard could fail without stopping anything.
|
||||
extension-test:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:24-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
# Not --no-save: vitest and web-ext are both real devDependencies now,
|
||||
# and the suite needs vitest resolvable from node_modules.
|
||||
- name: Install dev dependencies
|
||||
run: cd extension && npm install --no-audit --no-fund
|
||||
- name: Lint
|
||||
run: cd extension && npm run lint
|
||||
# Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
|
||||
# package version-consistency checks. No browser, no network.
|
||||
- name: Unit tests
|
||||
run: cd extension && npm run test:unit
|
||||
|
||||
# Everything else about packaging is asserted against our own declaration
|
||||
# of what ships. This is the only check that asks web-ext what it ACTUALLY
|
||||
# put in the archive. Until now that was an unverified assumption about
|
||||
# glob semantics — and a fragile one: `test/**` reaches web-ext intact
|
||||
# only because callers `set -f` first, so losing that quoting would
|
||||
# silently start shipping dev files with no other signal.
|
||||
- name: Verify XPI contents
|
||||
run: |
|
||||
set -eu
|
||||
command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
|
||||
cd extension
|
||||
npm run build
|
||||
ZIP=$(ls web-ext-artifacts/*.zip | head -1)
|
||||
echo "=== packaged entries in $ZIP ==="
|
||||
unzip -Z1 "$ZIP" | sort
|
||||
echo "=== end ==="
|
||||
ENTRIES=$(unzip -Z1 "$ZIP")
|
||||
fail=0
|
||||
# Must NOT ship: repo infrastructure with no business in a user's browser.
|
||||
for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
|
||||
if echo "$ENTRIES" | grep -q "^$pat"; then
|
||||
echo "ERROR: '$pat' was packaged into the XPI but must not be"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
# Must ship: if an exclusion pattern ever over-matches, the extension
|
||||
# breaks at runtime rather than at build time, so assert presence too.
|
||||
for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js' 'lib/chip.js' 'lib/popup-format.js'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$req$"; then
|
||||
echo "ERROR: '$req' is missing from the XPI"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$dir"; then
|
||||
echo "ERROR: nothing from '$dir' was packaged"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
[ "$fail" -eq 0 ] || exit 1
|
||||
echo "XPI contents verified."
|
||||
|
||||
# Single integration job — collapsed from a 3-way shard split on 2026-06-04.
|
||||
# The shards existed to parallelize ~8.5min of integration tests; once the
|
||||
# throwaway Postgres runs with fsync OFF (the durability step below) the whole
|
||||
@@ -419,7 +516,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
@@ -427,6 +524,8 @@ jobs:
|
||||
--health-retries 10
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ env.LANE_REF }}
|
||||
- name: Integration suite (resolve service IPs, migrate, test)
|
||||
run: |
|
||||
set -eux
|
||||
@@ -434,7 +533,7 @@ jobs:
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:8-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
@@ -521,7 +620,8 @@ jobs:
|
||||
# itself through a job-level `if:` that could not read `env`, and a design
|
||||
# where only a FAILED gate blocks would have published unverified images
|
||||
# while reporting success.
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
|
||||
integration]
|
||||
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
||||
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
||||
# it is stated on each publishing job rather than inferred from a `needs`
|
||||
@@ -534,8 +634,8 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
# scheduled refresh this is `main`; on everything else it is the
|
||||
# commit that fired, the same one the lanes above tested.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history is load-bearing, not a convenience: the version this
|
||||
# job signs is derived from the commit TIME of the newest packaged
|
||||
@@ -850,13 +950,16 @@ jobs:
|
||||
# reuse hit — deliberately NOT folded into `digest`, which the :c-<sha>
|
||||
# repoint reads and which must keep meaning "what this run built" (#4290).
|
||||
published_digest: ${{ steps.reuse.outputs.published_digest }}
|
||||
# Every tag this commit should end up under (channel, plus :c-<sha> on
|
||||
# main). `promote` writes them from `digest` once the smoke has passed.
|
||||
tags: ${{ steps.tag.outputs.tags }}
|
||||
# A plain `needs` — no `always()`. That expression existed to let a
|
||||
# SKIPPED sign-extension through on a tag push while still blocking a
|
||||
# FAILED one. With no tag trigger, sign-extension always runs, so the
|
||||
# default behaviour is exactly what we want: a failed sign skips build-web
|
||||
# rather than shipping an image without its XPI.
|
||||
needs: [sign-extension, lint, extension-version, backend-lint-and-test,
|
||||
frontend-build, integration]
|
||||
frontend-build, extension-test, integration]
|
||||
# The lanes in that list are THE GATE (2026-09-23) — see sign-extension's
|
||||
# copy of this comment for why, including the property that a SKIPPED lane
|
||||
# blocks as firmly as a failing one. They are repeated here rather than
|
||||
@@ -871,8 +974,8 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
# scheduled refresh this is `main`; on everything else it is the
|
||||
# commit that fired, the same one the lanes above tested.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history: this job RE-DERIVES the extension version rather than
|
||||
# being handed it, and a depth-1 clone derives a wrong, too-low value
|
||||
@@ -1108,11 +1211,13 @@ jobs:
|
||||
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||
# whether the channel then has to be written separately.
|
||||
#
|
||||
# On a push the build writes the channel tag directly: the bytes came
|
||||
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
||||
# it behind.
|
||||
# Every build writes a CANDIDATE tag, never the channel (#4310):
|
||||
# :dev-candidate / :latest-candidate on a push, :refresh-candidate on
|
||||
# the refresh. `promote` moves the channel only after smoke-web has
|
||||
# booted the bytes. The lanes prove the SOURCE; only the smoke proves
|
||||
# the BYTES, and the two are not the same claim.
|
||||
#
|
||||
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
||||
# The refresh is the case that made this obvious. A
|
||||
# refresh rebuilds against freshly resolved base images, and the web
|
||||
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||
@@ -1142,7 +1247,10 @@ jobs:
|
||||
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||
echo "promote=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||
# A push builds to a per-channel candidate too (#4310). The channel
|
||||
# tag moves in `promote`, after smoke-web has booted these bytes —
|
||||
# so a broken image never reaches the tag deployments follow.
|
||||
echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
|
||||
echo "promote=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
@@ -1418,6 +1526,16 @@ jobs:
|
||||
TAGS: ${{ steps.tag.outputs.tags }}
|
||||
run: |
|
||||
set -euf
|
||||
# A BUILD publishes nothing from here (#4310). Its bytes sit on the
|
||||
# candidate tag until smoke-web has booted them; `promote` then
|
||||
# writes the channel tag AND :c-<sha> from this run's digest. Writing
|
||||
# :c-<sha> here would publish an immutable rollback tag for bytes
|
||||
# that might then fail the smoke.
|
||||
if [ -n "${BUILT_DIGEST:-}" ]; then
|
||||
echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
|
||||
echo "repoint: after the smoke; nothing to write here."
|
||||
exit 0
|
||||
fi
|
||||
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
||||
#
|
||||
# This step used to copy from the channel tag by NAME. Nothing
|
||||
@@ -1511,14 +1629,9 @@ jobs:
|
||||
# source tree, and not a static inspection: `ffmpeg -version` exiting 0 would
|
||||
# pass while a codec removal broke every thumbnail in the library.
|
||||
#
|
||||
# Refresh-only. On a push the bytes came from a commit, and a commit is what
|
||||
# the lanes above already test — and since 2026-09-23 they gate the build, so
|
||||
# those bytes could not exist without them having passed.
|
||||
#
|
||||
# Reports a verdict; it does not yet gate the promote (milestone 362 step 4).
|
||||
# Landing the gate and the thing it gates in one change would mean the first
|
||||
# time anyone saw this job run would also be the first time it could stop a
|
||||
# publish.
|
||||
# It gates `promote` on every trigger (#4310). The lanes above prove the
|
||||
# source; they cannot see the bytes, and a Dockerfile or base change breaks
|
||||
# the bytes without touching the source.
|
||||
smoke-web:
|
||||
needs: [build-web]
|
||||
# Every run that actually BUILT something, not just the weekly refresh.
|
||||
@@ -1530,13 +1643,10 @@ jobs:
|
||||
# and were smoked when they were built. Re-smoking them would burn two
|
||||
# minutes to re-learn a fact.
|
||||
#
|
||||
# HONEST LIMIT, and it is the reason #4299 exists: on a push this runs
|
||||
# AFTER build-web has written the channel tag, so it detects rather than
|
||||
# gates. Rule 164's verify_with asks for the check BETWEEN build and push.
|
||||
# Closing that needs the push path to adopt the candidate-then-promote
|
||||
# shape the refresh already has — per-channel candidate tags, promote
|
||||
# learning its channel, and the :c-<sha> repoint moving after the gate.
|
||||
# That is a redesign of the production publish path and is its own task.
|
||||
# This GATES the publish on every trigger (#4310): builds land on a
|
||||
# candidate tag, and `promote` needs this job, so :dev / :latest and a
|
||||
# built :c-<sha> move only after it passes. Rule 164's verify_with — the
|
||||
# check BETWEEN build and push.
|
||||
#
|
||||
# NO `if:` — this job always runs (#4323). It used to be gated on the
|
||||
# build having published something, which skipped it on a reuse hit. That
|
||||
@@ -1583,7 +1693,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
@@ -1609,7 +1719,7 @@ jobs:
|
||||
# in a container against a mounted docker socket, so the services are
|
||||
# SIBLINGS reachable by IP, not by hostname.
|
||||
PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:8-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
@@ -1954,88 +2064,113 @@ jobs:
|
||||
# smoked it, and it should not be read as if it were.
|
||||
promote:
|
||||
needs: [build-web, build-agent, smoke-web]
|
||||
# Only a refresh publishes through a candidate; a push writes its channel
|
||||
# tag directly from the build. Reads the same reuse-step decision the build
|
||||
# took, via a job output — a job's `if:` cannot see the `env` context.
|
||||
if: needs.build-web.outputs.candidate == 'true'
|
||||
# THE PUBLISH (#4310). Both image jobs build to a candidate tag — a push to
|
||||
# :dev-candidate / :latest-candidate, the weekly refresh to
|
||||
# :refresh-candidate — and nothing names those bytes under a tag anybody
|
||||
# deploys until this job runs. It runs only when every job in `needs`
|
||||
# SUCCEEDED (a job-level `if:` without a status function implies
|
||||
# success()), so a failed smoke — or a skipped one, which is not the same
|
||||
# as a passing one (run 5290) — leaves :dev / :latest on the last build
|
||||
# that worked. Rule 164's verify_with: the check sits BETWEEN build and
|
||||
# push.
|
||||
#
|
||||
# Per image, from the DIGEST that image's job built (#4290): a tag can
|
||||
# move between the build and this job, a digest cannot. An image whose job
|
||||
# hit reuse built nothing and has nothing to promote — its build job
|
||||
# already wrote its :c-<sha> from the channel tag, which "hit" proved
|
||||
# carries this commit.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- name: Point the channel tags at the smoked candidates
|
||||
- name: Point this commit's tags at the smoked builds
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
ACTOR: ${{ github.actor }}
|
||||
WEB_DIGEST: ${{ needs.build-web.outputs.digest }}
|
||||
WEB_TAGS: ${{ needs.build-web.outputs.tags }}
|
||||
AGENT_DIGEST: ${{ needs.build-agent.outputs.digest }}
|
||||
AGENT_TAGS: ${{ needs.build-agent.outputs.tags }}
|
||||
run: |
|
||||
set -eu
|
||||
# `latest` is not a guess: a refresh always builds `main` (BUILD_REF),
|
||||
# and the "must have checked out main" guard in every build job fails
|
||||
# the run if that did not hold. So the channel is main's.
|
||||
TAG=latest
|
||||
FAILED=""
|
||||
MOVED=0
|
||||
|
||||
for NAME in fabledcurator fabledcurator-agent; do
|
||||
promote() {
|
||||
NAME="$1"; DIGEST="$2"; TAGS="$3"
|
||||
REPO="bvandeusen/$NAME"
|
||||
echo "promote: $REPO"
|
||||
if [ -z "$DIGEST" ]; then
|
||||
echo "promote: $NAME — no build this run (reuse hit); nothing to publish"
|
||||
return 0
|
||||
fi
|
||||
echo "promote: $NAME $DIGEST"
|
||||
|
||||
# Registry auth is its own token exchange — `docker login`
|
||||
# authenticates the docker client, not curl. Deadline on every call
|
||||
# (rule 156): a registry that stops answering must fail this step,
|
||||
# not hang the weekly refresh until the job times out.
|
||||
# not hang the run until the job times out.
|
||||
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
|
||||
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
|
||||
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
|
||||
|
||||
# Ask for the IMAGE manifest media types only. Offering the index
|
||||
# types too would let the registry hand back an index if one ever
|
||||
# existed at this tag, and we would faithfully copy the thing this
|
||||
# whole approach exists to avoid creating.
|
||||
# types too would let the registry hand back an index, and we would
|
||||
# faithfully copy the thing this approach exists to avoid creating.
|
||||
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
|
||||
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
|
||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$DIGEST" \
|
||||
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
|
||||
test -n "$CT"
|
||||
SRC=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||
echo "promote: candidate $SRC ($CT)"
|
||||
|
||||
# NOT `imagetools create`. That wraps its source in an INDEX, and
|
||||
# `.Image.Config.Labels` does not resolve through one — the
|
||||
# fc.revision the reuse check reads off the channel tag would come
|
||||
# back empty, every later push would miss and rebuild, and nothing
|
||||
# would go red (#3183, run 4751). A manifest PUT is what "make this
|
||||
# tag name that image" means at the registry: same bytes, same media
|
||||
# type, same digest, no layer transfer.
|
||||
curl -fsS --max-time 120 -X PUT \
|
||||
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
|
||||
--data-binary @manifest.json \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
|
||||
# tag name that image" means at the registry: same bytes, same
|
||||
# media type, same digest, no layer transfer. (It also makes a
|
||||
# built :c-<sha> a plain image rather than the index the old
|
||||
# repoint left.)
|
||||
IFS=,
|
||||
for REF in $TAGS; do
|
||||
TAG="${REF##*:}"
|
||||
curl -fsS --max-time 120 -X PUT \
|
||||
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
|
||||
--data-binary @manifest.json \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
|
||||
|
||||
# Read it back. A PUT that returned 2xx but landed something else is
|
||||
# exactly the silent-and-plausible failure this pipeline keeps
|
||||
# producing, and the check costs one request.
|
||||
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
|
||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
|
||||
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||
if [ "$NOW" != "$SRC" ]; then
|
||||
echo "promote: FAILED — $NAME:$TAG is $NOW, expected $SRC" >&2
|
||||
FAILED="$FAILED $NAME"
|
||||
continue
|
||||
fi
|
||||
echo "promote: $NAME:$TAG now names $NOW"
|
||||
done
|
||||
# Read it back. A PUT that returned 2xx but landed something else
|
||||
# is exactly the silent-and-plausible failure this pipeline keeps
|
||||
# producing, and the check costs one request.
|
||||
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
|
||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
|
||||
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||
if [ "$NOW" != "$DIGEST" ]; then
|
||||
echo "promote: FAILED — $NAME:$TAG is $NOW, expected $DIGEST" >&2
|
||||
FAILED="$FAILED $NAME:$TAG"
|
||||
continue
|
||||
fi
|
||||
echo "promote: $NAME:$TAG now names $NOW"
|
||||
MOVED=$((MOVED + 1))
|
||||
done
|
||||
unset IFS
|
||||
}
|
||||
|
||||
promote fabledcurator "$WEB_DIGEST" "$WEB_TAGS"
|
||||
promote fabledcurator-agent "$AGENT_DIGEST" "$AGENT_TAGS"
|
||||
|
||||
if [ -n "$FAILED" ]; then
|
||||
echo "" >&2
|
||||
echo "promote: FAILED for:$FAILED" >&2
|
||||
echo "promote: the channel tags are now INCONSISTENT — some images" >&2
|
||||
echo "promote: moved and some did not. Re-run this refresh; the" >&2
|
||||
echo "promote: candidates are still published and the promote is" >&2
|
||||
echo "promote: idempotent." >&2
|
||||
echo "promote: the tags are now INCONSISTENT — some moved and some" >&2
|
||||
echo "promote: did not. Re-run this workflow; the builds are still" >&2
|
||||
echo "promote: published by digest and the promote is idempotent." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "promote: both channel tags moved"
|
||||
echo "promote: $MOVED tag(s) written"
|
||||
build-agent:
|
||||
# THE GATE (2026-09-23). Every lane above must have PASSED before this job
|
||||
# exists at all — so a red suite does not produce an image, let alone push
|
||||
@@ -2048,7 +2183,14 @@ jobs:
|
||||
# itself through a job-level `if:` that could not read `env`, and a design
|
||||
# where only a FAILED gate blocks would have published unverified images
|
||||
# while reporting success.
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
|
||||
integration]
|
||||
# What `promote` needs to publish this image once the smoke has passed:
|
||||
# the manifest this run built (empty on a reuse hit) and every tag it
|
||||
# belongs under. Same meaning as build-web's outputs of the same names.
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
tags: ${{ steps.tag.outputs.tags }}
|
||||
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
||||
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
||||
# it is stated on each publishing job rather than inferred from a `needs`
|
||||
@@ -2061,8 +2203,8 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
# Not the triggering ref — see the `env:` block at the top. On a
|
||||
# scheduled refresh this is `main`; on everything else it is the ref
|
||||
# that fired, so this is a no-op on every ordinary path.
|
||||
# scheduled refresh this is `main`; on everything else it is the
|
||||
# commit that fired, the same one the lanes above tested.
|
||||
ref: ${{ env.BUILD_REF }}
|
||||
# Full history: this job derives its artifact's version from the
|
||||
# commit its shipped files last changed in (milestone 313). A
|
||||
@@ -2240,11 +2382,13 @@ jobs:
|
||||
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||
# whether the channel then has to be written separately.
|
||||
#
|
||||
# On a push the build writes the channel tag directly: the bytes came
|
||||
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
||||
# it behind.
|
||||
# Every build writes a CANDIDATE tag, never the channel (#4310):
|
||||
# :dev-candidate / :latest-candidate on a push, :refresh-candidate on
|
||||
# the refresh. `promote` moves the channel only after smoke-web has
|
||||
# booted the bytes. The lanes prove the SOURCE; only the smoke proves
|
||||
# the BYTES, and the two are not the same claim.
|
||||
#
|
||||
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
||||
# The refresh is the case that made this obvious. A
|
||||
# refresh rebuilds against freshly resolved base images, and the web
|
||||
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||
@@ -2273,7 +2417,10 @@ jobs:
|
||||
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||
# A push builds to a per-channel candidate too (#4310). The channel
|
||||
# tag moves in `promote`, after smoke-web has booted these bytes —
|
||||
# so a broken image never reaches the tag deployments follow.
|
||||
echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
||||
@@ -2464,6 +2611,16 @@ jobs:
|
||||
TAGS: ${{ steps.tag.outputs.tags }}
|
||||
run: |
|
||||
set -euf
|
||||
# A BUILD publishes nothing from here (#4310). Its bytes sit on the
|
||||
# candidate tag until smoke-web has booted them; `promote` then
|
||||
# writes the channel tag AND :c-<sha> from this run's digest. Writing
|
||||
# :c-<sha> here would publish an immutable rollback tag for bytes
|
||||
# that might then fail the smoke.
|
||||
if [ -n "${BUILT_DIGEST:-}" ]; then
|
||||
echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
|
||||
echo "repoint: after the smoke; nothing to write here."
|
||||
exit 0
|
||||
fi
|
||||
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
||||
#
|
||||
# This step used to copy from the channel tag by NAME. Nothing
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
name: extension
|
||||
# Lint + unit tests. Deliberately NOT a publishing lane, which is why it is
|
||||
# not part of build.yml's gate. The sign-and-publish dance moved into build.yml's
|
||||
# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
|
||||
# because sign-extension runs as a build-web dependency in the SAME workflow,
|
||||
# eliminating the prior race between build.yml and a separate extension.yml.
|
||||
# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
paths:
|
||||
- 'extension/**'
|
||||
- '.forgejo/workflows/extension.yml'
|
||||
# test/version.spec.js asserts things ABOUT build.yml — that it does not
|
||||
# inline the packaged-file set, and that build.yml derives
|
||||
# the shipped version rather than reading it out of the repo. A
|
||||
# workflow-only edit can therefore break this suite, so it has to trigger
|
||||
# it. build.yml joined the list at milestone 271 step 5, when the spec
|
||||
# started asserting against it.
|
||||
- '.forgejo/workflows/build.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'extension/**'
|
||||
- '.forgejo/workflows/build.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:24-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Not --no-save: vitest and web-ext are both real devDependencies now,
|
||||
# and the suite needs vitest resolvable from node_modules.
|
||||
- name: Install dev dependencies
|
||||
run: cd extension && npm install --no-audit --no-fund
|
||||
- name: Lint
|
||||
run: cd extension && npm run lint
|
||||
# Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
|
||||
# package version-consistency checks. No browser, no network.
|
||||
- name: Unit tests
|
||||
run: cd extension && npm run test:unit
|
||||
|
||||
# Everything else about packaging is asserted against our own declaration
|
||||
# of what ships. This is the only check that asks web-ext what it ACTUALLY
|
||||
# put in the archive. Until now that was an unverified assumption about
|
||||
# glob semantics — and a fragile one: `test/**` reaches web-ext intact
|
||||
# only because callers `set -f` first, so losing that quoting would
|
||||
# silently start shipping dev files with no other signal.
|
||||
- name: Verify XPI contents
|
||||
run: |
|
||||
set -eu
|
||||
command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
|
||||
cd extension
|
||||
npm run build
|
||||
ZIP=$(ls web-ext-artifacts/*.zip | head -1)
|
||||
echo "=== packaged entries in $ZIP ==="
|
||||
unzip -Z1 "$ZIP" | sort
|
||||
echo "=== end ==="
|
||||
ENTRIES=$(unzip -Z1 "$ZIP")
|
||||
fail=0
|
||||
# Must NOT ship: repo infrastructure with no business in a user's browser.
|
||||
for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
|
||||
if echo "$ENTRIES" | grep -q "^$pat"; then
|
||||
echo "ERROR: '$pat' was packaged into the XPI but must not be"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
# Must ship: if an exclusion pattern ever over-matches, the extension
|
||||
# breaks at runtime rather than at build time, so assert presence too.
|
||||
for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$req$"; then
|
||||
echo "ERROR: '$req' is missing from the XPI"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$dir"; then
|
||||
echo "ERROR: nothing from '$dir' was packaged"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
[ "$fail" -eq 0 ] || exit 1
|
||||
echo "XPI contents verified."
|
||||
+3
-7
@@ -36,11 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libwebp7 \
|
||||
libpng16-16 \
|
||||
ca-certificates \
|
||||
# opencv-python-headless (via requirements-ml.txt) links these even in its
|
||||
# headless build. Came from Dockerfile.ml when the images merged
|
||||
# (milestone 422 step 6).
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
@@ -61,7 +56,8 @@ RUN pip install -r requirements.txt
|
||||
#
|
||||
# torch 2.12.1+cpu wheel 192.3 MB
|
||||
# torchvision 0.27.1+cpu 1.8 MB
|
||||
# transformers / onnxruntime / opencv / sklearn and friends
|
||||
# transformers / onnxruntime / opencv / sklearn and friends (opencv and
|
||||
# onnxruntime since dropped, #1451 — nothing here imported them)
|
||||
# 62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB
|
||||
# largest newly-pushed layer 222.07 MB
|
||||
#
|
||||
@@ -85,7 +81,7 @@ RUN pip install -r requirements.txt
|
||||
# CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the
|
||||
# GPU agent is a separate service with its own image.
|
||||
RUN pip install --index-url https://download.pytorch.org/whl/cpu \
|
||||
"torch>=2.12,<3.0" "torchvision>=0.27,<0.28"
|
||||
"torch>=2.14" "torchvision>=0.29"
|
||||
RUN pip install -r requirements-ml.txt
|
||||
|
||||
# Where the model lands. Deliberately NOT a VOLUME instruction: that mints an
|
||||
|
||||
@@ -22,7 +22,8 @@ through afterwards.
|
||||
- **ML tagging.** Runs image models in-container to suggest tags, group
|
||||
characters, find near-duplicates and power similarity search. Suggestions are
|
||||
reviewable — it proposes, you confirm, and it learns which proposals you keep
|
||||
rejecting.
|
||||
rejecting. It ships switched off: turn it on under Settings → System when
|
||||
you want it, and it fetches its model weights then.
|
||||
- **Deduplication and provenance.** Everything that arrives is hashed and
|
||||
deduplicated by content, metadata sidecars are read wherever the source
|
||||
writes them, and every file keeps a record of where it came from.
|
||||
@@ -119,9 +120,12 @@ needs every credential entered again by hand.
|
||||
|
||||
A few other things are worth knowing about the first few minutes:
|
||||
|
||||
- **The ML worker downloads its model weights on first boot**, several GB from
|
||||
HuggingFace into `./models`. Until that finishes, tagging is queued rather
|
||||
than broken. It is idempotent — a restart resumes rather than refetches.
|
||||
- **ML tagging starts switched off, and nothing is downloaded at boot.** Give
|
||||
the ML lane a slot under **Settings → System** and it fetches its model
|
||||
weights then — several GB from HuggingFace into `./models`, shown as a job
|
||||
under **Settings → Activity** that you can watch and retry. Until it
|
||||
finishes, tagging is queued rather than broken, and the fetch only takes
|
||||
what is missing, so turning the lane off and on again does not refetch.
|
||||
- **The gallery starts empty**, and that is the expected state. Add a creator
|
||||
under **Subscriptions** and it fills as posts come down.
|
||||
- **If you already have a library on disk**, there is no screen that imports
|
||||
@@ -271,11 +275,12 @@ Four deployable pieces, built by `.forgejo/workflows/build.yml`:
|
||||
|
||||
## CI / Forgejo setup
|
||||
|
||||
Three workflows: `build.yml` (the five verification lanes — lint,
|
||||
extension-version check, backend unit tests, frontend build, integration — and
|
||||
then sign + publish), `extension.yml` (extension lint, vitest, XPI content
|
||||
verification), and `release.yml`, which runs only on a `v*` tag and publishes a
|
||||
changelog without building anything.
|
||||
Two workflows that matter here: `build.yml` (the six verification lanes — lint,
|
||||
extension-version check, backend unit tests, frontend build, extension lint +
|
||||
vitest + XPI content check, integration — and then sign + publish), and
|
||||
`release.yml`, which runs only on a `v*` tag and publishes a changelog without
|
||||
building anything. The extension lane was its own `extension.yml` until
|
||||
milestone 429, which let a red extension suite sign and ship the XPI anyway.
|
||||
|
||||
**The lanes and the publish are one workflow on purpose.** They were two
|
||||
(`ci.yml` and `build.yml`) until 2026-09-23, on the same push trigger, which
|
||||
|
||||
+23
-10
@@ -1,10 +1,21 @@
|
||||
# FabledCurator GPU agent — runs on the desktop with the GPU.
|
||||
# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs
|
||||
# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open
|
||||
# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
|
||||
# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are
|
||||
# built against (CUDA 13 has only nascent ONNX Runtime support).
|
||||
FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04
|
||||
#
|
||||
# The `base` flavour, not `cudnn-runtime`: CUDA and cuDNN arrive as the
|
||||
# `nvidia-*` pip packages torch and onnxruntime-gpu depend on, so the base only
|
||||
# has to hand the container the driver (it sets NVIDIA_VISIBLE_DEVICES /
|
||||
# NVIDIA_DRIVER_CAPABILITIES for the Container Toolkit). Until #1451 this was
|
||||
# `12.9.2-cudnn-runtime` under a `torch==2.6.0+cu124` — and requirements.txt then
|
||||
# REPLACED that torch with PyPI's CUDA-13 build (ultralytics pulls torchvision,
|
||||
# which pulls its matching torch), beside a CUDA-13 onnxruntime-gpu. The image
|
||||
# ran CUDA 13 on a CUDA-12 base, carrying ~3 GB of base libraries and a ~3 GB
|
||||
# torch nothing loaded: 10 GB compressed.
|
||||
#
|
||||
# 13.0 because that is the line both wheels are built for (torch's cu130 index,
|
||||
# onnxruntime-gpu's `nvidia-cuda-runtime~=13.0`). Needs an NVIDIA driver that
|
||||
# supports CUDA 13 (580+); fc_agent/accel.py logs at startup whether torch and
|
||||
# onnxruntime actually got the GPU, since both fall back to the CPU silently.
|
||||
# ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
|
||||
FROM nvidia/cuda:13.0.3-base-ubuntu24.04
|
||||
|
||||
# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
|
||||
# managed (PEP 668), so a global `pip install` errors without this. It's a
|
||||
@@ -16,10 +27,12 @@ RUN apt-get update \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN
|
||||
# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first
|
||||
# + separately so the GPU build of torch is deterministic and layer-cached.
|
||||
RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
|
||||
# torch AND torchvision from the cu130 index, together and first. Installing
|
||||
# torch alone is what let the next step swap it out: ultralytics needs
|
||||
# torchvision, PyPI's torchvision pins its own torch, and pip replaced ours to
|
||||
# match. With both present, requirements.txt finds them satisfied.
|
||||
RUN pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu130 \
|
||||
torch torchvision
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install --no-cache-dir -r requirements.txt
|
||||
COPY fc_agent ./fc_agent
|
||||
|
||||
+16
-1
@@ -15,7 +15,22 @@ sudo pacman -S nvidia-container-toolkit
|
||||
sudo nvidia-ctk runtime configure --runtime=docker
|
||||
sudo systemctl restart docker
|
||||
# verify:
|
||||
docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi
|
||||
docker run --rm --gpus all nvidia/cuda:13.0.3-base-ubuntu24.04 nvidia-smi
|
||||
# the header's CUDA version must be 13.0 or later (driver 580+)
|
||||
```
|
||||
|
||||
### After a driver update: regenerate the CDI spec
|
||||
If the agent's first log lines say `accel: torch is NOT on the GPU` or report
|
||||
`cudaGetDeviceCount: unknown error (999)` while `nvidia-smi` still works, the
|
||||
toolkit's saved device list (`/etc/cdi/nvidia.yaml`) is out of date. The
|
||||
`nvidia-uvm` device number changes between driver versions, and a spec
|
||||
generated before the update hands the container a device node that no longer
|
||||
exists (2026-09-24: host `511,0`, container `235,0`). Compare
|
||||
`ls -l /dev/nvidia-uvm` on the host with the same inside the container, then:
|
||||
```sh
|
||||
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
|
||||
# if your toolkit ships it, this keeps it current on every driver update:
|
||||
sudo systemctl enable --now nvidia-cdi-refresh.path
|
||||
```
|
||||
|
||||
## 1. Get a token
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Which accelerator each runtime actually got — reported once, at startup.
|
||||
|
||||
The agent has two GPU runtimes and both fall back to the CPU without raising:
|
||||
torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils
|
||||
detector + CCIP models) when its CUDA provider cannot load its libraries. A
|
||||
fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the
|
||||
image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base
|
||||
(#1451), and whether the ONNX half was on the GPU could not be answered from
|
||||
anything the agent had ever logged.
|
||||
|
||||
Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA
|
||||
provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image
|
||||
they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT
|
||||
1.21+) loads them from there, so the provider resolves them by soname.
|
||||
|
||||
Stdlib-only at import, so the unit suite can import it — torch and onnxruntime
|
||||
are imported inside the functions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import importlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("fc_agent.accel")
|
||||
|
||||
# Filled by report(); /status carries it so the page can show it too.
|
||||
LAST: dict = {}
|
||||
|
||||
|
||||
def torch_status(imp=importlib.import_module) -> dict:
|
||||
try:
|
||||
torch = imp("torch")
|
||||
except Exception as e:
|
||||
return {"device": "unavailable", "error": str(e)}
|
||||
out = {"version": torch.__version__, "cuda_build": torch.version.cuda}
|
||||
if torch.cuda.is_available():
|
||||
out["device"] = "cuda"
|
||||
out["gpu"] = torch.cuda.get_device_name(0)
|
||||
else:
|
||||
out["device"] = "cpu"
|
||||
return out
|
||||
|
||||
|
||||
def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict:
|
||||
try:
|
||||
ort = imp("onnxruntime")
|
||||
except Exception as e:
|
||||
return {"device": "unavailable", "error": str(e)}
|
||||
out = {"version": ort.__version__, "providers": list(ort.get_available_providers())}
|
||||
if "CUDAExecutionProvider" not in out["providers"]:
|
||||
out["device"] = "cpu"
|
||||
return out
|
||||
preload = getattr(ort, "preload_dlls", None)
|
||||
if preload is not None:
|
||||
try:
|
||||
preload()
|
||||
except Exception as e:
|
||||
out["preload_error"] = str(e)
|
||||
# "Available" only means the build HAS the provider. Loading its library is
|
||||
# what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are
|
||||
# missing, and the one a session would otherwise fail silently on.
|
||||
capi = Path(ort.__file__).parent / "capi"
|
||||
try:
|
||||
load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL)
|
||||
load(str(capi / "libonnxruntime_providers_cuda.so"))
|
||||
except OSError as e:
|
||||
out["device"] = "cpu"
|
||||
out["error"] = str(e)
|
||||
return out
|
||||
# Loading proves the libraries resolve, NOT that a GPU can be used: on
|
||||
# 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with
|
||||
# "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA
|
||||
# runtime for a device initialises the driver the provider would use.
|
||||
error = _cuda_device_error(load)
|
||||
out["device"] = "cpu" if error else "cuda"
|
||||
if error:
|
||||
out["error"] = error
|
||||
return out
|
||||
|
||||
|
||||
def _cuda_device_error(load=ctypes.CDLL) -> str | None:
|
||||
"""None when the CUDA runtime can reach a device, else why it cannot."""
|
||||
try:
|
||||
cudart = load("libcudart.so.13")
|
||||
except OSError as e:
|
||||
return str(e)
|
||||
count = ctypes.c_int(0)
|
||||
rc = cudart.cudaGetDeviceCount(ctypes.byref(count))
|
||||
if rc != 0:
|
||||
cudart.cudaGetErrorString.restype = ctypes.c_char_p
|
||||
return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})"
|
||||
return None if count.value > 0 else "no CUDA device visible"
|
||||
|
||||
|
||||
def summary() -> dict | None:
|
||||
"""The report as FabledCurator stores it: each runtime's device, and why
|
||||
when it is not the GPU. Sent on every lease and heartbeat, so the System
|
||||
view can call a running agent that fell back to the CPU "degraded" rather
|
||||
than "running" — the 2026-09-24 fallback went unseen for weeks because
|
||||
only this agent's own log said so. None before report() has run."""
|
||||
if not LAST:
|
||||
return None
|
||||
out = {}
|
||||
for name, s in LAST.items():
|
||||
entry = {"device": s.get("device")}
|
||||
if s.get("error"):
|
||||
entry["error"] = str(s["error"])[:200]
|
||||
out[name] = entry
|
||||
return out
|
||||
|
||||
|
||||
def report() -> dict:
|
||||
"""Check both runtimes, log the result, and keep it for /status."""
|
||||
LAST.clear()
|
||||
LAST.update(torch=torch_status(), onnx=onnx_status())
|
||||
for name, s in LAST.items():
|
||||
if s.get("device") == "cuda":
|
||||
log.info("accel: %s on GPU (%s)", name, s)
|
||||
else:
|
||||
log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s)
|
||||
return dict(LAST)
|
||||
+28
-3
@@ -11,7 +11,7 @@ import logging
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from . import logbuf
|
||||
from . import accel, logbuf
|
||||
from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version
|
||||
from .config import Config
|
||||
from .gpu import read_gpu
|
||||
@@ -47,6 +47,9 @@ async def _no_store(request, call_next):
|
||||
|
||||
@app.on_event("startup")
|
||||
def _maybe_autostart() -> None:
|
||||
# Before the worker: the report also preloads the CUDA libraries the ONNX
|
||||
# models need, and it says in the log which runtimes landed on the GPU.
|
||||
accel.report()
|
||||
# With AUTO_START set, a container restart (host reboot, or `restart:
|
||||
# unless-stopped` after a crash) resumes the worker on its own — the slots
|
||||
# then ride out a still-down curator via lease backoff. Lets the agent
|
||||
@@ -137,6 +140,7 @@ def status():
|
||||
s["version"] = FC_VERSION or None
|
||||
s["channel"] = FC_CHANNEL or None
|
||||
s["revision"] = FC_REVISION or None
|
||||
s["accel"] = accel.LAST or None
|
||||
return JSONResponse(s)
|
||||
|
||||
|
||||
@@ -188,7 +192,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
|
||||
.step:hover{border-color:var(--acc)}
|
||||
#conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
|
||||
color:var(--fg);border:1px solid var(--bd);border-radius:8px}
|
||||
color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
|
||||
/* The browser's own spin arrows, hidden: the − / + beside each field are the
|
||||
control, styled like the rest of the page (operator, 2026-09-24). */
|
||||
#conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
|
||||
#bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
|
||||
.unit{color:var(--mut);font-size:12px;font-weight:600}
|
||||
.hint{color:var(--mut);font-size:12px;margin-top:12px}
|
||||
.tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
|
||||
@@ -227,6 +235,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
<div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
|
||||
a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
|
||||
</div>
|
||||
<div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
|
||||
<div id=banner class=banner style=display:none>
|
||||
curator unreachable — holding work + retrying, resumes on its own (no restart needed)
|
||||
</div>
|
||||
@@ -244,7 +253,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
<button class=step onclick=setc(1)>+</button>
|
||||
</div>
|
||||
<div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
|
||||
<button class=step onclick=stepbw(-1)>−</button>
|
||||
<input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
|
||||
<button class=step onclick=stepbw(1)>+</button>
|
||||
<span class=unit>MB/s</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -312,6 +323,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({value:on})});refresh()
|
||||
}
|
||||
function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
|
||||
// Runtimes that did NOT get the GPU, from the startup report. Both fall back
|
||||
// to the CPU without raising, so this banner and the pill are the only place
|
||||
// on this page a slow, CPU-bound agent announces itself.
|
||||
function cpuOnly(s){
|
||||
const a=s.accel||{}
|
||||
return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
|
||||
}
|
||||
async function setbw(v){
|
||||
v=Math.max(0,parseFloat(v)||0); bw.value=v
|
||||
await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
@@ -382,11 +401,17 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
// unreachable curator; grey when stopped; red with no token.
|
||||
let dc='dot', lbl='stopped'
|
||||
if(!ok){ dc='dot red'; lbl='no token' }
|
||||
else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
|
||||
else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
|
||||
if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
|
||||
else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
|
||||
else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
|
||||
dot.className=dc; connlbl.textContent=lbl
|
||||
banner.style.display=(st==='running' && !s.queue)?'block':'none'
|
||||
const slow=cpuOnly(s)
|
||||
accelbanner.style.display=slow.length?'block':'none'
|
||||
accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
|
||||
+slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
|
||||
+'. After a driver update, regenerate the CDI spec (agent README).'):''
|
||||
queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from . import accel
|
||||
|
||||
|
||||
class FcClient:
|
||||
def __init__(self, base_url: str, token: str, agent_id: str):
|
||||
@@ -72,7 +74,10 @@ class FcClient:
|
||||
def lease(self, batch_size: int) -> list[dict]:
|
||||
r = self.s.post(
|
||||
f"{self.base}/api/gpu/jobs/lease",
|
||||
json={"agent_id": self.agent_id, "batch_size": batch_size},
|
||||
json={
|
||||
"agent_id": self.agent_id, "batch_size": batch_size,
|
||||
"accel": accel.summary(),
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
@@ -90,7 +95,9 @@ class FcClient:
|
||||
})
|
||||
|
||||
def heartbeat(self, job_ids: list[int]) -> None:
|
||||
self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
|
||||
self._post_quiet(
|
||||
"/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
|
||||
)
|
||||
|
||||
def fail(self, job_id: int, error: str) -> None:
|
||||
self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace).
|
||||
dghs-imgutils>=0.4
|
||||
# GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
|
||||
# server-side fallback run.
|
||||
onnxruntime-gpu
|
||||
# The crop EMBEDDER (concept bag). torch is installed separately in the
|
||||
# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic;
|
||||
# server-side fallback run. The extras declare the CUDA/cuDNN pip packages its
|
||||
# CUDA provider loads (fc_agent/accel.py preloads them) rather than relying on
|
||||
# torch happening to install the same ones.
|
||||
onnxruntime-gpu[cuda,cudnn]
|
||||
# The crop EMBEDDER (concept bag). torch + torchvision are installed separately
|
||||
# in the Dockerfile from the cu130 wheel index, so pip never swaps them out;
|
||||
# transformers loads whatever SigLIP-family model the server announces.
|
||||
transformers>=4.45
|
||||
# Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Discord native ingester ledgers — seen and dead-letter, per source.
|
||||
|
||||
Milestone 428, #4415. Discord moves off gallery-dl onto the native core, which
|
||||
keeps its memory of what a source has already fetched in these two tables
|
||||
instead of gallery-dl's archive. Same shape as the SubscribeStar pair.
|
||||
|
||||
Revision ID: 0111
|
||||
Revises: 0110
|
||||
Create Date: 2026-09-24
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0111"
|
||||
down_revision = "0110"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"discord_seen_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("post_id", sa.String(length=64), nullable=True),
|
||||
sa.Column(
|
||||
"seen_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_discord_seen_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_seen_media")),
|
||||
sa.UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_discord_seen_media_source_id"), "discord_seen_media", ["source_id"],
|
||||
)
|
||||
op.create_table(
|
||||
"discord_failed_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"first_failed_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"last_failed_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_discord_failed_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_failed_media")),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_discord_failed_media_source_id",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_discord_failed_media_source_id"), "discord_failed_media", ["source_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index(op.f("ix_discord_failed_media_source_id"), table_name="discord_failed_media")
|
||||
op.drop_table("discord_failed_media")
|
||||
op.drop_index(op.f("ix_discord_seen_media_source_id"), table_name="discord_seen_media")
|
||||
op.drop_table("discord_seen_media")
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Retire the sketch/doodle WIP title tier — its tags, its review flags, its toggle.
|
||||
|
||||
Milestone 430, #4428. The soft tier (#1474) tagged `wip` on any post titled
|
||||
sketch / doodle / scribble. Measured on the operator's library it was 6,096 of
|
||||
8,876 wip tags, and its conflict audit filled the Gallery's review strip with
|
||||
2,086 cards, because most finished art scores >= 0.5 on some content head. A
|
||||
"sketch" is usually finished work, so the operator retired it (2026-09-25).
|
||||
|
||||
Data:
|
||||
|
||||
* A soft tag the operator stood behind is kept and relabelled `manual`: one they
|
||||
confirmed (tag_positive_confirmation), or one whose review flag they resolved
|
||||
while leaving the tag on (the strip's "Keep tag").
|
||||
* Every other `wip_title_soft` row is deleted.
|
||||
* Unresolved review flags whose tag is no longer on the image are deleted: the
|
||||
question they ask no longer applies. That is the audit's cards, and any older
|
||||
orphan the same way.
|
||||
|
||||
Then `import_settings.wip_soft_title_tagging_enabled` is dropped. The downgrade
|
||||
restores the column only; deleted tags are not recreated.
|
||||
|
||||
Revision ID: 0112
|
||||
Revises: 0111
|
||||
Create Date: 2026-09-25
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0112"
|
||||
down_revision = "0111"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def retire_soft_wip_tags(conn) -> None:
|
||||
"""The data half, on a plain connection, so a test can run it directly."""
|
||||
conn.execute(sa.text("""
|
||||
UPDATE image_tag it SET source = 'manual'
|
||||
WHERE it.source = 'wip_title_soft'
|
||||
AND (
|
||||
EXISTS (
|
||||
SELECT 1 FROM tag_positive_confirmation c
|
||||
WHERE c.image_record_id = it.image_record_id AND c.tag_id = it.tag_id
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM presentation_review pr
|
||||
WHERE pr.image_record_id = it.image_record_id AND pr.tag_id = it.tag_id
|
||||
AND pr.resolved_at IS NOT NULL
|
||||
)
|
||||
)
|
||||
"""))
|
||||
conn.execute(sa.text("DELETE FROM image_tag WHERE source = 'wip_title_soft'"))
|
||||
conn.execute(sa.text("""
|
||||
DELETE FROM presentation_review pr
|
||||
WHERE pr.resolved_at IS NULL
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM image_tag it
|
||||
WHERE it.image_record_id = pr.image_record_id AND it.tag_id = pr.tag_id
|
||||
)
|
||||
"""))
|
||||
|
||||
|
||||
def upgrade():
|
||||
retire_soft_wip_tags(op.get_bind())
|
||||
op.drop_column("import_settings", "wip_soft_title_tagging_enabled")
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.add_column(
|
||||
"import_settings",
|
||||
sa.Column(
|
||||
"wip_soft_title_tagging_enabled",
|
||||
sa.Boolean(),
|
||||
nullable=False,
|
||||
server_default=sa.text("false"),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Re-date images whose post's date arrived after they were linked.
|
||||
|
||||
#4431. The native ingesters import a post's media before its record, and the
|
||||
date travels in the record (`_post.json`). Every natively downloaded image was
|
||||
therefore linked to an undated post and kept its download time in both gallery
|
||||
date columns, while the post itself was dated correctly. The importer now
|
||||
re-dates a post's images when its record lands; this repairs the images that
|
||||
landed before that.
|
||||
|
||||
Both columns get back the rules the importer keeps:
|
||||
|
||||
* `effective_date` is the primary post's date (left alone when that post has
|
||||
none, as the importer does);
|
||||
* `earliest_post_date` is the earliest dated post the image is linked to.
|
||||
|
||||
Only rows that differ are written. The downgrade does nothing: the old values
|
||||
were download times that no one chose.
|
||||
|
||||
Revision ID: 0113
|
||||
Revises: 0112
|
||||
Create Date: 2026-09-25
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0113"
|
||||
down_revision = "0112"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def redate_images(conn) -> None:
|
||||
"""The data step, on a plain connection, so a test can run it directly."""
|
||||
conn.execute(sa.text("""
|
||||
UPDATE image_record ir SET effective_date = p.post_date
|
||||
FROM post p
|
||||
WHERE p.id = ir.primary_post_id
|
||||
AND p.post_date IS NOT NULL
|
||||
AND ir.effective_date IS DISTINCT FROM p.post_date
|
||||
"""))
|
||||
conn.execute(sa.text("""
|
||||
UPDATE image_record ir SET earliest_post_date = m.earliest
|
||||
FROM (
|
||||
SELECT ip.image_record_id, MIN(p.post_date) AS earliest
|
||||
FROM image_provenance ip JOIN post p ON p.id = ip.post_id
|
||||
WHERE p.post_date IS NOT NULL
|
||||
GROUP BY ip.image_record_id
|
||||
) m
|
||||
WHERE m.image_record_id = ir.id
|
||||
AND ir.earliest_post_date IS DISTINCT FROM m.earliest
|
||||
"""))
|
||||
|
||||
|
||||
def upgrade():
|
||||
redate_images(op.get_bind())
|
||||
|
||||
|
||||
def downgrade():
|
||||
pass
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Fold the undated shell posts a misfiled attachment created into the real post.
|
||||
|
||||
#4435. A non-image file (an archive, a pdf) downloaded for one of an artist's
|
||||
Discord channels was filed under the artist's FIRST Discord source: the
|
||||
attachment path looked the source up by (artist, platform), which takes the
|
||||
lowest id. That created an undated, url-less post there holding only the
|
||||
attachment, and the message's real post record then created the dated post
|
||||
under the right source. The importer now uses the source it was downloading
|
||||
for; this repairs the pairs it left.
|
||||
|
||||
A shell is folded only when all of this holds:
|
||||
|
||||
* it has no date and no url, and nothing synthesized it;
|
||||
* another post of the same artist, platform and external id HAS a date;
|
||||
* no image is linked to the shell (it held an attachment and nothing else).
|
||||
|
||||
Its attachments move to the dated post, dropping any the dated post already
|
||||
has (same sha256, which the per-post unique forbids twice), and the shell is
|
||||
deleted. A shell with no dated twin is left alone: which channel it belongs to
|
||||
is not recorded anywhere but the file name. The downgrade does nothing.
|
||||
|
||||
Revision ID: 0114
|
||||
Revises: 0113
|
||||
Create Date: 2026-09-25
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0114"
|
||||
down_revision = "0113"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_PAIRS = """
|
||||
SELECT DISTINCT ON (shell.id) shell.id AS shell_id, real.id AS real_id
|
||||
FROM post shell
|
||||
JOIN source ss ON ss.id = shell.source_id
|
||||
JOIN post real
|
||||
ON real.artist_id = shell.artist_id
|
||||
AND real.external_post_id = shell.external_post_id
|
||||
AND real.id <> shell.id
|
||||
AND real.post_date IS NOT NULL
|
||||
JOIN source rs ON rs.id = real.source_id AND rs.platform = ss.platform
|
||||
WHERE shell.post_date IS NULL
|
||||
AND shell.post_url IS NULL
|
||||
AND shell.synthesized_by IS NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM image_provenance ip WHERE ip.post_id = shell.id)
|
||||
ORDER BY shell.id, real.id
|
||||
"""
|
||||
|
||||
|
||||
def fold_misfiled_attachment_posts(conn) -> int:
|
||||
"""The data step, on a plain connection, so a test can run it directly.
|
||||
Returns how many shells were folded."""
|
||||
pairs = conn.execute(sa.text(_PAIRS)).all()
|
||||
for shell_id, real_id in pairs:
|
||||
conn.execute(sa.text("""
|
||||
DELETE FROM post_attachment pa
|
||||
WHERE pa.post_id = :shell
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM post_attachment keep
|
||||
WHERE keep.post_id = :real AND keep.sha256 = pa.sha256
|
||||
)
|
||||
"""), {"shell": shell_id, "real": real_id})
|
||||
conn.execute(
|
||||
sa.text("UPDATE post_attachment SET post_id = :real WHERE post_id = :shell"),
|
||||
{"shell": shell_id, "real": real_id},
|
||||
)
|
||||
conn.execute(sa.text("DELETE FROM post WHERE id = :shell"), {"shell": shell_id})
|
||||
return len(pairs)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
fold_misfiled_attachment_posts(op.get_bind())
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -0,0 +1,27 @@
|
||||
"""source.display_name — what a source is called on its platform.
|
||||
|
||||
#4481. A Discord source's URL is a server id and a channel id, so the
|
||||
Subscriptions page could only show two numbers. The Discord ingester already
|
||||
loads the server and channel names to walk them; it now keeps them here,
|
||||
refreshed on every walk. NULL until a walk has read it.
|
||||
|
||||
Revision ID: 0115
|
||||
Revises: 0114
|
||||
Create Date: 2026-09-28
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0115"
|
||||
down_revision = "0114"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True))
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_column("source", "display_name")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""The single-color filter's default becomes near-total: 0.95 → 0.995.
|
||||
|
||||
#4483. At 0.95 the filter rejected line art: a pencil doodle on white is
|
||||
mostly white, and five of Todding's Discord doodles were skipped on import as
|
||||
"single color", leaving their posts with text and no image. The predicate now
|
||||
samples real pixels instead of a blurred thumbnail, and the default only
|
||||
calls an image blank when it essentially is one.
|
||||
|
||||
A stored 0.95 is the old default, so it moves with it. Any other value was
|
||||
chosen by hand and is left alone.
|
||||
|
||||
Revision ID: 0116
|
||||
Revises: 0115
|
||||
Create Date: 2026-09-28
|
||||
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
revision = "0116"
|
||||
down_revision = "0115"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.alter_column("import_settings", "single_color_threshold", server_default="0.995")
|
||||
op.execute(
|
||||
"UPDATE import_settings SET single_color_threshold = 0.995 "
|
||||
"WHERE single_color_threshold = 0.95"
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.alter_column("import_settings", "single_color_threshold", server_default="0.95")
|
||||
@@ -19,7 +19,10 @@ from ..models import AppSetting
|
||||
from ..services.extension_service import (
|
||||
ExtensionService,
|
||||
InvalidUrlError,
|
||||
PosterLookupError,
|
||||
UnknownArtistError,
|
||||
UnknownPlatformError,
|
||||
UnknownSourceError,
|
||||
)
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
from ._responses import error_response as _bad
|
||||
@@ -87,10 +90,16 @@ async def probe_source():
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return _bad("invalid_body", detail="url query parameter is required")
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
result = await ExtensionService(session).probe(url)
|
||||
# crypto lets a Discord probe name the server and channel with the
|
||||
# stored token; every other platform ignores it.
|
||||
result = await ExtensionService(session, _get_crypto()).probe(
|
||||
url, names=request.args.get("names") in ("1", "true"),
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@@ -102,6 +111,22 @@ async def quick_add_source():
|
||||
url = body.get("url")
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
# Optional: connect the new source to an existing artist (artist_id) or to
|
||||
# the artist of that name (artist_name). A Discord channel names no
|
||||
# creator, so the extension's Add panel always sends one of them.
|
||||
artist_id = body.get("artist_id")
|
||||
if artist_id is not None and (isinstance(artist_id, bool) or not isinstance(artist_id, int)):
|
||||
return _bad("invalid_body", detail="artist_id must be an integer")
|
||||
artist_name = body.get("artist_name")
|
||||
if artist_name is not None and not isinstance(artist_name, str):
|
||||
return _bad("invalid_body", detail="artist_name must be a string")
|
||||
# Patreon is canon: adding a Patreon source to an existing artist can take
|
||||
# the creator's Patreon display name (name only; the slug never moves).
|
||||
use_platform_name = body.get("use_platform_name") is True
|
||||
# #4488: a Discord add can carry the posters picked in the panel.
|
||||
discord_authors = body.get("discord_authors")
|
||||
if discord_authors is not None and not _valid_authors(discord_authors):
|
||||
return _bad("invalid_body", detail="discord_authors must be a list of {id, name}")
|
||||
|
||||
from .credentials import _get_crypto
|
||||
|
||||
@@ -111,7 +136,13 @@ async def quick_add_source():
|
||||
try:
|
||||
# crypto lets an add resolve the artist's display name via the
|
||||
# stored credential (else it falls back to the URL handle). #130.
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(
|
||||
url, artist_id=artist_id, artist_name=artist_name,
|
||||
use_platform_name=use_platform_name,
|
||||
discord_authors=discord_authors,
|
||||
)
|
||||
except UnknownArtistError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad(
|
||||
"unknown_platform",
|
||||
@@ -123,6 +154,59 @@ async def quick_add_source():
|
||||
return jsonify(result), (201 if result["created_source"] else 200)
|
||||
|
||||
|
||||
def _valid_authors(value) -> bool:
|
||||
"""[{id, name}]: the id is what the list keys on, the name only shows it."""
|
||||
return isinstance(value, list) and all(
|
||||
isinstance(a, dict) and isinstance(a.get("id"), str | int)
|
||||
and not isinstance(a.get("id"), bool)
|
||||
and (a.get("name") is None or isinstance(a.get("name"), str))
|
||||
for a in value
|
||||
)
|
||||
|
||||
|
||||
@extension_bp.route("/discord/posters", methods=["GET"])
|
||||
async def discord_posters():
|
||||
"""Who posts in the Discord channel `url` shows, lately, with the likely
|
||||
creator marked (#4488). Reads the channel with the stored token; writes
|
||||
nothing."""
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return _bad("invalid_body", detail="url query parameter is required")
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session, _get_crypto()).discord_posters(url)
|
||||
except (InvalidUrlError, UnknownPlatformError) as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
except PosterLookupError as exc:
|
||||
return _bad("posters_unavailable", detail=str(exc), status=409)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@extension_bp.route("/discord/posters", methods=["POST"])
|
||||
async def set_discord_posters():
|
||||
"""Set the poster list of the source following `url`'s channel."""
|
||||
body = await request.get_json(silent=True)
|
||||
if not isinstance(body, dict) or not isinstance(body.get("url"), str):
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
authors = body.get("authors")
|
||||
if not _valid_authors(authors):
|
||||
return _bad("invalid_body", detail="authors must be a list of {id, name}")
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
try:
|
||||
result = await ExtensionService(session).set_discord_posters(body["url"], authors)
|
||||
except (InvalidUrlError, UnknownPlatformError, PosterLookupError) as exc:
|
||||
return _bad("invalid_url", detail=str(exc))
|
||||
except UnknownSourceError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
def _read_manifest_sync() -> dict | None:
|
||||
"""All the filesystem-touching work for /api/extension/manifest,
|
||||
in a sync helper so the async route can dispatch it via
|
||||
|
||||
+31
-2
@@ -245,6 +245,29 @@ async def errors_recover(image_id: int):
|
||||
|
||||
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
||||
|
||||
|
||||
def _accel_detail(body: dict) -> dict:
|
||||
"""The agent's own report of which runtime got the GPU, kept on its roster
|
||||
row so the System view can call a CPU-bound agent degraded (#4410).
|
||||
|
||||
Only a dict of {runtime: {device, error?}} is kept, and each value is
|
||||
reduced to those two short strings: this is written on every lease, by a
|
||||
client the server does not control. An agent that sends nothing (an
|
||||
older build) simply has no `accel`, which reads as not-yet-reported.
|
||||
"""
|
||||
raw = body.get("accel")
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
accel = {}
|
||||
for name, entry in list(raw.items())[:4]:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
clean = {"device": str(entry.get("device") or "")[:16]}
|
||||
if entry.get("error"):
|
||||
clean["error"] = str(entry["error"])[:200]
|
||||
accel[str(name)[:16]] = clean
|
||||
return {"accel": accel} if accel else {}
|
||||
|
||||
@gpu_bp.route("/jobs/lease", methods=["POST"])
|
||||
async def lease():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
@@ -267,7 +290,10 @@ async def lease():
|
||||
key=f"agent:{agent_id}",
|
||||
kind="agent",
|
||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||
details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)},
|
||||
details={
|
||||
"agent_id": agent_id, "last_call": "lease", "leased": len(jobs),
|
||||
**_accel_detail(body),
|
||||
},
|
||||
)
|
||||
ml = await MLSettings.load(session)
|
||||
# image rows for url/mime in one shot
|
||||
@@ -347,7 +373,10 @@ async def heartbeat():
|
||||
key=f"agent:{agent_id}",
|
||||
kind="agent",
|
||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||
details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n},
|
||||
details={
|
||||
"agent_id": agent_id, "last_call": "heartbeat", "extended": n,
|
||||
**_accel_detail(body),
|
||||
},
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"extended": n})
|
||||
|
||||
@@ -57,7 +57,6 @@ _EDITABLE_FIELDS = (
|
||||
"translation_target_lang",
|
||||
"translation_min_confidence",
|
||||
"wip_title_tagging_enabled",
|
||||
"wip_soft_title_tagging_enabled",
|
||||
)
|
||||
|
||||
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
||||
@@ -196,12 +195,6 @@ async def update_import_settings():
|
||||
return jsonify(
|
||||
{"error": "wip_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
if "wip_soft_title_tagging_enabled" in body and not isinstance(
|
||||
body["wip_soft_title_tagging_enabled"], bool
|
||||
):
|
||||
return jsonify(
|
||||
{"error": "wip_soft_title_tagging_enabled must be a boolean"}
|
||||
), 400
|
||||
|
||||
async with get_session() as session:
|
||||
row = await ImportSettings.load(session)
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from quart import Blueprint, jsonify, request
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..extensions import get_session
|
||||
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
||||
from ..services import discord_poster_cleanup as poster_cleanup
|
||||
from ..services.artist_membership_service import ArtistMembershipService
|
||||
from ..services.artist_membership_service import rescan as membership_rescan
|
||||
from ..services.artist_service import ArtistService
|
||||
@@ -463,3 +466,49 @@ async def adopt_membership():
|
||||
return jsonify({"already_tracked": exc.existing_id})
|
||||
artist_id = artist.id
|
||||
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
||||
|
||||
|
||||
# -- Discord: remove posts by people outside the poster list (#4486) --------------
|
||||
|
||||
_IMAGES_ROOT = Path("/images")
|
||||
|
||||
|
||||
async def _poster_cleanup_preview(source_id: int):
|
||||
async with get_session() as session:
|
||||
return await session.run_sync(
|
||||
lambda s: poster_cleanup.preview(s, source_id=source_id)
|
||||
)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
|
||||
async def other_posters_preview(source_id: int):
|
||||
"""What removing other posters' posts would delete. Nothing is touched."""
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
projection["confirm_token"] = poster_cleanup.confirm_token(projection)
|
||||
return jsonify(projection)
|
||||
|
||||
|
||||
@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
|
||||
async def other_posters_remove(source_id: int):
|
||||
"""Delete them. `confirm` must be the token of the preview the operator saw:
|
||||
a poster list edited since then changes the set, and the token with it."""
|
||||
body = await request.get_json(silent=True) or {}
|
||||
try:
|
||||
projection = await _poster_cleanup_preview(source_id)
|
||||
except LookupError:
|
||||
return _bad("not_found", status=404)
|
||||
except poster_cleanup.PosterCleanupError as exc:
|
||||
return _bad("not_applicable", detail=str(exc), status=409)
|
||||
expected = poster_cleanup.confirm_token(projection)
|
||||
if body.get("confirm") != expected:
|
||||
return _bad("confirm_mismatch", expected=expected)
|
||||
async with get_session() as session:
|
||||
result = await session.run_sync(
|
||||
lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
@@ -152,6 +152,8 @@ async def list_runs():
|
||||
queue=<name> filter to one queue
|
||||
status=<status> filter to one status (running/ok/error/timeout/retry)
|
||||
task=<substr> case-insensitive substring match on task_name
|
||||
celery_task_id=<id> exactly one run — how a page follows a job it
|
||||
started without having to know its lane
|
||||
limit=<int> default 50, max 200
|
||||
before_id=<int> cursor for keyset pagination
|
||||
|
||||
@@ -167,6 +169,7 @@ async def list_runs():
|
||||
queue = request.args.get("queue")
|
||||
status = request.args.get("status")
|
||||
task = request.args.get("task")
|
||||
celery_task_id = request.args.get("celery_task_id")
|
||||
before_id_raw = request.args.get("before_id")
|
||||
before_id = int(before_id_raw) if before_id_raw else None
|
||||
|
||||
@@ -176,6 +179,8 @@ async def list_runs():
|
||||
stmt = stmt.where(TaskRun.queue == queue)
|
||||
if status:
|
||||
stmt = stmt.where(TaskRun.status == status)
|
||||
if celery_task_id:
|
||||
stmt = stmt.where(TaskRun.celery_task_id == celery_task_id)
|
||||
if task:
|
||||
# Task names contain literal underscores (download_source,
|
||||
# vacuum_analyze) — escape LIKE wildcards so a search for
|
||||
|
||||
@@ -72,9 +72,14 @@ assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, (
|
||||
PROBE_TIMEOUT_SECONDS = 2.0
|
||||
|
||||
_OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
|
||||
# Checking in, but working at a fraction of its speed: a GPU agent whose
|
||||
# runtimes fell back to the CPU (#4410). Below stale — a part that may have
|
||||
# stopped is the more urgent question — and above unknown, because this one
|
||||
# IS known to be wrong.
|
||||
_DEGRADED = "degraded"
|
||||
|
||||
# Worst-first, so an overall verdict is just the max.
|
||||
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3}
|
||||
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4}
|
||||
|
||||
|
||||
def _age_state(age_seconds: float) -> str:
|
||||
@@ -100,6 +105,39 @@ def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
|
||||
return f"{name} has not checked in for {ago} — treat it as stopped"
|
||||
|
||||
|
||||
def _cpu_runtimes(details: dict) -> list[str]:
|
||||
"""The runtimes an agent reported as NOT on the GPU, with why.
|
||||
|
||||
Both torch and onnxruntime fall back to the CPU without raising, so an
|
||||
agent in that state leases, works and checks in exactly like a healthy
|
||||
one. On 2026-09-24 one had been doing so since a driver update left a
|
||||
stale CDI spec; the only sign was a line in the agent's own log.
|
||||
"""
|
||||
accel = details.get("accel")
|
||||
if not isinstance(accel, dict):
|
||||
return []
|
||||
out = []
|
||||
for name, entry in sorted(accel.items()):
|
||||
if not isinstance(entry, dict) or entry.get("device") == "cuda":
|
||||
continue
|
||||
why = entry.get("error") or entry.get("device") or "unknown"
|
||||
out.append(f"{name} ({why})")
|
||||
return out
|
||||
|
||||
|
||||
def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]:
|
||||
"""A roster row's state and its sentence, degraded included."""
|
||||
if state == _OK:
|
||||
cpu = _cpu_runtimes(details)
|
||||
if cpu:
|
||||
return _DEGRADED, (
|
||||
f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. "
|
||||
"After a driver update, regenerate the agent host's CDI spec "
|
||||
"(agent README)."
|
||||
)
|
||||
return state, _describe_learned(name, state, age, details)
|
||||
|
||||
|
||||
async def _probe_postgres(session) -> dict:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
@@ -175,13 +213,15 @@ async def system_health():
|
||||
).scalars().all()
|
||||
for row in rows:
|
||||
age = (now - row.last_seen_at).total_seconds()
|
||||
state = _age_state(age)
|
||||
state, detail = _learned_state(
|
||||
row.display_name, _age_state(age), age, row.details or {},
|
||||
)
|
||||
parts.append({
|
||||
"key": row.key,
|
||||
"kind": row.kind,
|
||||
"name": row.display_name,
|
||||
"state": state,
|
||||
"detail": _describe_learned(row.display_name, state, age, row.details or {}),
|
||||
"detail": detail,
|
||||
"last_seen_at": row.last_seen_at.isoformat(),
|
||||
"first_seen_at": row.first_seen_at.isoformat(),
|
||||
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Celery beat that remembers when each job last ran — from task_run, not a file.
|
||||
|
||||
Celery's default PersistentScheduler keeps its memory in a shelve file in the
|
||||
working directory. Nothing mounts that directory, so every container recreate
|
||||
forgets it, and a scheduler that remembers nothing seeds every entry with
|
||||
`last_run_at = now`: each job waits a FULL interval after startup. A daily job
|
||||
therefore needs 24 hours without a redeploy to fire. Since the one-container
|
||||
image (172e33d) every redeploy restarts beat, and on 2026-09-24 no daily or
|
||||
weekly job had run since the 21st (#4408).
|
||||
|
||||
task_run already records every task that starts (celery_signals), indexed on
|
||||
(task_name, started_at DESC), and prune_task_runs keeps the newest row of each
|
||||
task however old it is. So on startup each entry takes its last_run_at from
|
||||
there:
|
||||
- a job that is overdue runs at once;
|
||||
- a job that is not due waits only the remainder of its interval;
|
||||
- a job that has never run is due now.
|
||||
|
||||
Beat keeps last_run_at in memory from then on, as the default scheduler does;
|
||||
only the startup seed changes. If the database cannot be read, the entries keep
|
||||
Celery's own default rather than beat failing to start.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.beat import Scheduler
|
||||
from sqlalchemy import func, select
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Seed for a job with no recorded run: far enough back that any interval or
|
||||
# crontab reads as due.
|
||||
NEVER = datetime(2000, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def last_runs(session, task_names: list[str]) -> dict[str, datetime]:
|
||||
"""The latest recorded start of each task, by task name."""
|
||||
from .models import TaskRun
|
||||
|
||||
if not task_names:
|
||||
return {}
|
||||
rows = session.execute(
|
||||
select(TaskRun.task_name, func.max(TaskRun.started_at))
|
||||
.where(TaskRun.task_name.in_(sorted(set(task_names))))
|
||||
.group_by(TaskRun.task_name)
|
||||
).all()
|
||||
return dict(rows)
|
||||
|
||||
|
||||
def seed(entries, last: dict[str, datetime]) -> None:
|
||||
"""Set each entry's last_run_at from `last`; a task with none is due now."""
|
||||
for entry in entries:
|
||||
entry.last_run_at = last.get(entry.task, NEVER)
|
||||
|
||||
|
||||
class TaskRunScheduler(Scheduler):
|
||||
"""An in-memory beat seeded from task_run history at startup."""
|
||||
|
||||
def setup_schedule(self):
|
||||
super().setup_schedule()
|
||||
try:
|
||||
from .tasks._sync_engine import sync_session_factory
|
||||
|
||||
with sync_session_factory()() as session:
|
||||
last = last_runs(session, [e.task for e in self.schedule.values()])
|
||||
except Exception:
|
||||
log.exception("beat: could not read task_run; every job waits a full interval")
|
||||
return
|
||||
seed(self.schedule.values(), last)
|
||||
due = sum(1 for e in self.schedule.values() if e.is_due()[0])
|
||||
log.info(
|
||||
"beat: seeded %d job(s) from task_run, %d due now",
|
||||
len(self.schedule), due,
|
||||
)
|
||||
@@ -62,6 +62,15 @@ def make_celery() -> Celery:
|
||||
# can never starve the quick self-healing sweeps (operator-flagged
|
||||
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||
# The one long job in maintenance.py: a whole-library phash
|
||||
# recompute (35 min hard limit; the library was cleared for
|
||||
# re-hashing by migration 0098). On the quick lane it held a
|
||||
# scheduler process for its whole run, and the minute ticks queued
|
||||
# up behind it (2026-09-24: 7 waiting, "all workers busy for 18
|
||||
# minutes"). An exact name wins over the glob above.
|
||||
"backend.app.tasks.maintenance.backfill_phash": {"queue": "maintenance_long"},
|
||||
# Walks a folder tree per artist with undated posts (#4436).
|
||||
"backend.app.tasks.maintenance.date_posts_from_records": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||
@@ -149,6 +158,11 @@ def make_celery() -> Celery:
|
||||
"schedule": 86400.0, # daily — sweep .part/.partial left by a
|
||||
# download/import killed mid-write (graceful-shutdown fallout)
|
||||
},
|
||||
"date-posts-from-records-hourly": {
|
||||
"task": "backend.app.tasks.maintenance.date_posts_from_records",
|
||||
"schedule": 3600.0, # an empty query once every native post is
|
||||
# dated; otherwise upserts the _post.json a killed walk left (#4436)
|
||||
},
|
||||
"backfill-phash-daily": {
|
||||
"task": "backend.app.tasks.maintenance.backfill_phash",
|
||||
"schedule": 86400.0, # daily — NULL-only, so a no-op once the
|
||||
@@ -216,11 +230,6 @@ def make_celery() -> Celery:
|
||||
"schedule": 86400.0, # auto-tag wip/editor process art (#1464);
|
||||
# no-op unless process_auto_apply_enabled (opt-in)
|
||||
},
|
||||
"soft-wip-conflict-audit-daily": {
|
||||
"task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
||||
"schedule": 86400.0, # flag ring-loud soft-WIP (sketch/doodle) tags
|
||||
# for review (#1474); no-op with no content heads
|
||||
},
|
||||
"prune-presentation-reviews-daily": {
|
||||
"task": "backend.app.tasks.ml.prune_presentation_reviews",
|
||||
"schedule": 86400.0, # retention: drop resolved review flags >30d
|
||||
@@ -353,6 +362,9 @@ def make_celery() -> Celery:
|
||||
},
|
||||
},
|
||||
timezone="UTC",
|
||||
# Beat's memory of when each job last ran comes from task_run, not a
|
||||
# shelve file nothing persists — see beat_scheduler (#4408).
|
||||
beat_scheduler="backend.app.beat_scheduler:TaskRunScheduler",
|
||||
)
|
||||
# FC-3i: register task_run signal handlers (side-effect import).
|
||||
from . import celery_signals # noqa: F401
|
||||
|
||||
@@ -18,11 +18,18 @@ dark for that interval. Monitoring NEVER breaks the thing it's
|
||||
monitoring.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.exceptions import SoftTimeLimitExceeded
|
||||
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
|
||||
from celery.signals import (
|
||||
task_failure,
|
||||
task_postrun,
|
||||
task_prerun,
|
||||
task_retry,
|
||||
worker_ready,
|
||||
)
|
||||
|
||||
from .models import TaskRun
|
||||
from .tasks._sync_engine import sync_session_factory
|
||||
@@ -53,42 +60,29 @@ _INT32_MIN = -2_147_483_648
|
||||
|
||||
|
||||
def _queue_for(task) -> str:
|
||||
"""Reverse the task→queue routing from celery_app.task_routes.
|
||||
Keep in sync if task_routes is reordered.
|
||||
"""The queue Celery routes this task to — asked of the router itself.
|
||||
|
||||
Audit 2026-06-02: backup/admin/library_audit prefixes were
|
||||
missing here even though task_routes sent all three to
|
||||
'maintenance'. The TaskRun.queue column then lied for those
|
||||
rows (claimed 'default') so per-queue dashboard filters and
|
||||
per-queue threshold overrides silently missed them.
|
||||
This was a hand-kept copy of `celery_app.task_routes`, and it drifted
|
||||
twice (the 2026-06-02 audit, then #4432). Long-lane jobs were recorded as
|
||||
`maintenance`, and translation and gpu_queue runs as `default`, where the
|
||||
5-minute stall sweep failed healthy 35-minute translation runs. The router
|
||||
answers from the same table the broker uses, so the two cannot disagree.
|
||||
"""
|
||||
name = getattr(task, "name", "") or ""
|
||||
if name.startswith("backend.app.tasks.import_file."):
|
||||
return "import"
|
||||
if name.startswith("backend.app.tasks.ml."):
|
||||
return "ml"
|
||||
if name.startswith("backend.app.tasks.thumbnail."):
|
||||
return "thumbnail"
|
||||
if name.startswith((
|
||||
"backend.app.tasks.download.",
|
||||
# External file-host fetches share the download lane (celery_app
|
||||
# routes external.* → download). Mirror it here or TaskRun.queue
|
||||
# lies 'default' for them, so per-queue dashboard filters and the
|
||||
# per-queue threshold override miss them — the same gap the
|
||||
# 2026-06-02 audit fixed for backup/admin/library_audit.
|
||||
"backend.app.tasks.external.",
|
||||
)):
|
||||
return "download"
|
||||
if name.startswith("backend.app.tasks.scan."):
|
||||
return "scan"
|
||||
if name.startswith((
|
||||
"backend.app.tasks.maintenance.",
|
||||
"backend.app.tasks.backup.",
|
||||
"backend.app.tasks.admin.",
|
||||
"backend.app.tasks.library_audit.",
|
||||
)):
|
||||
return "maintenance"
|
||||
return "default"
|
||||
app = getattr(task, "app", None)
|
||||
if app is None:
|
||||
from .celery_app import celery as app
|
||||
return _routed_queue(app, name)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1024)
|
||||
def _routed_queue(app, name: str) -> str:
|
||||
try:
|
||||
queue = app.amqp.router.route({}, name).get("queue")
|
||||
except Exception: # noqa: BLE001 — monitoring never breaks the task
|
||||
log.warning("task_run: could not resolve the queue for %s", name)
|
||||
return "default"
|
||||
return getattr(queue, "name", None) or (queue if isinstance(queue, str) else "default")
|
||||
|
||||
|
||||
def _target_id_from_args(args) -> int | None:
|
||||
@@ -225,3 +219,43 @@ def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
|
||||
error_message=str(reason) if reason else None,
|
||||
retry_count=getattr(request, "retries", 0),
|
||||
)
|
||||
|
||||
|
||||
def _consumed_queues(consumer) -> set[str]:
|
||||
"""The queue names this worker process consumes (its `-Q`), or empty when
|
||||
they can't be read — which makes the boot hook below a no-op, not a guess."""
|
||||
try:
|
||||
return {q.name for q in consumer.task_consumer.queues}
|
||||
except Exception: # noqa: BLE001 — shape varies across celery versions
|
||||
return set()
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def _on_worker_ready(sender=None, **_):
|
||||
"""The download lane clears what its previous process left behind (#4433).
|
||||
|
||||
A restart SIGKILLs any walk that outlives the stop grace, so its event is
|
||||
never finalized and its platform lock is never released. Both used to wait
|
||||
out timers — a 30-min stall sweep that then blamed the source, and a 27-min
|
||||
lock TTL that stalled every other source on the platform. Only the process
|
||||
consuming `download` does this; the other lanes booting beside it must not.
|
||||
Best-effort: a failure here is logged and the worker still starts.
|
||||
"""
|
||||
if "download" not in _consumed_queues(sender):
|
||||
return
|
||||
booted_at = datetime.now(UTC)
|
||||
try:
|
||||
from .services.download_recovery import interrupt_orphaned_download_events
|
||||
from .services.platform_lock import release_all_platform_locks
|
||||
|
||||
with sync_session_factory()() as session:
|
||||
closed = interrupt_orphaned_download_events(session, booted_at=booted_at)
|
||||
session.commit()
|
||||
released = release_all_platform_locks()
|
||||
if closed or released:
|
||||
log.info(
|
||||
"download lane boot: closed %d orphaned download event(s) as "
|
||||
"interrupted, released %d platform lock(s)", closed, released,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — never block the worker from starting
|
||||
log.exception("download lane boot recovery failed")
|
||||
|
||||
+20
-2
@@ -46,6 +46,14 @@ async def serve_extension(filename: str):
|
||||
|
||||
The application/x-xpinstall MIME tells Firefox to show its native
|
||||
install prompt instead of downloading the file as a blob.
|
||||
|
||||
Caching differs by name, and has to. A versioned name is one build's bytes
|
||||
forever, so it can be cached for good. `fabledcurator-latest.xpi` is ONE
|
||||
URL whose bytes change on every release, and Quart's default for a file
|
||||
is `public, max-age=43200`: a browser that fetched it once reused those
|
||||
bytes for 12 hours, so "install the latest" quietly reinstalled the
|
||||
previous build (operator-flagged 2026-09-25). It is `no-cache` — the ETag
|
||||
still makes an unchanged file a cheap 304.
|
||||
"""
|
||||
if not _XPI_NAME_RE.fullmatch(filename):
|
||||
abort(404)
|
||||
@@ -56,10 +64,11 @@ async def serve_extension(filename: str):
|
||||
if not xpis:
|
||||
abort(404)
|
||||
latest = xpis[-1]
|
||||
return await send_file(
|
||||
resp = await send_file(
|
||||
latest, mimetype="application/x-xpinstall",
|
||||
attachment_filename=latest.name,
|
||||
)
|
||||
return _cache(resp, "no-cache")
|
||||
target = (XPI_DIR / filename).resolve()
|
||||
try:
|
||||
target.relative_to(XPI_DIR)
|
||||
@@ -67,10 +76,19 @@ async def serve_extension(filename: str):
|
||||
abort(404)
|
||||
if not target.is_file():
|
||||
abort(404)
|
||||
return await send_file(
|
||||
resp = await send_file(
|
||||
target, mimetype="application/x-xpinstall",
|
||||
attachment_filename=filename,
|
||||
)
|
||||
return _cache(resp, "public, max-age=31536000, immutable")
|
||||
|
||||
|
||||
def _cache(resp, policy: str):
|
||||
"""Set the XPI's Cache-Control, dropping the Expires send_file adds so the
|
||||
two can never disagree."""
|
||||
resp.headers["Cache-Control"] = policy
|
||||
resp.headers.pop("Expires", None)
|
||||
return resp
|
||||
|
||||
|
||||
@frontend_bp.route("/")
|
||||
|
||||
@@ -8,6 +8,8 @@ from .backup_run import BackupRun
|
||||
from .base import Base
|
||||
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
||||
from .credential import Credential
|
||||
from .discord_failed_media import DiscordFailedMedia
|
||||
from .discord_seen_media import DiscordSeenMedia
|
||||
from .download_event import DownloadEvent
|
||||
from .external_link import ExternalLink
|
||||
from .gpu_job import GpuJob
|
||||
@@ -56,6 +58,8 @@ __all__ = [
|
||||
"BackupRun",
|
||||
"Source",
|
||||
"Credential",
|
||||
"DiscordFailedMedia",
|
||||
"DiscordSeenMedia",
|
||||
"PatreonFailedMedia",
|
||||
"PatreonSeenMedia",
|
||||
"SubscribeStarFailedMedia",
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""DiscordFailedMedia — per-source dead-letter ledger of Discord files that
|
||||
keep failing to download or validate.
|
||||
|
||||
Mirror of SubscribeStarFailedMedia. After `attempts` reaches the dead-letter
|
||||
threshold a routine walk skips the file (recovery still retries it); a later
|
||||
clean download clears the row. `filehash` is the seen-ledger's key.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class DiscordFailedMedia(Base):
|
||||
__tablename__ = "discord_failed_media"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "filehash", name="uq_discord_failed_media_source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
last_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""DiscordSeenMedia — per-source ledger of Discord files already downloaded.
|
||||
|
||||
Mirror of SubscribeStarSeenMedia for the native Discord ingester (milestone
|
||||
428). `filehash` holds the ingester's per-file key, `<message_id>:<media_id>`:
|
||||
the attachment id, or for an embed a hash of its URL path. Not the file's
|
||||
position in the message — an edit that removes a file renumbers the rest
|
||||
(see `discord_client.MediaItem`). The message record's own gate is the
|
||||
synthetic `message:<id>` key in the same column.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class DiscordSeenMedia(Base):
|
||||
__tablename__ = "discord_seen_media"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -39,7 +39,7 @@ class ImportSettings(Base):
|
||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
|
||||
|
||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
|
||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995")
|
||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
|
||||
|
||||
# Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The
|
||||
@@ -238,13 +238,6 @@ class ImportSettings(Base):
|
||||
wip_title_tagging_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=True, server_default="true",
|
||||
)
|
||||
# Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
|
||||
# a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
|
||||
# are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
|
||||
# precision tier is opt-in (the ring-loud audit surfaces false positives).
|
||||
wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
|
||||
Boolean, nullable=False, default=False, server_default="false",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def load(cls, session) -> ImportSettings:
|
||||
|
||||
@@ -46,6 +46,11 @@ class Source(Base):
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||
|
||||
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
# alembic 0115: what the source is called on its platform, where the URL
|
||||
# alone doesn't say — a Discord link is two numbers. The ingester that
|
||||
# walks it refreshes it every walk, so a rename reads as the new name.
|
||||
# NULL until a walk has read it, and on platforms that don't write it.
|
||||
display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
|
||||
@@ -316,17 +316,32 @@ class ArtistService:
|
||||
cleaned = (prefix or "").strip()
|
||||
if not cleaned:
|
||||
return []
|
||||
like = f"%{cleaned.lower()}%"
|
||||
prefix_like = f"{cleaned.lower()}%"
|
||||
# Rank: exact (0) < prefix (1) < substring (2).
|
||||
low = cleaned.lower()
|
||||
like = f"%{low}%"
|
||||
prefix_like = f"{low}%"
|
||||
# Spacing- and punctuation-insensitive too, so "Tamada Heijun" finds
|
||||
# "TamadaHeijun" and "sabu_art" finds "Sabu Art" — the same creator is
|
||||
# spelled differently on every platform, and the browser extension's
|
||||
# Add panel matches a Discord server name against these (milestone 429).
|
||||
# [[:alnum:]] keeps non-Latin letters; a query with none (all
|
||||
# punctuation) skips this arm rather than matching every artist.
|
||||
squashed = "".join(ch for ch in low if ch.isalnum())
|
||||
name_squashed = func.regexp_replace(func.lower(Artist.name), "[^[:alnum:]]", "", "g")
|
||||
matches = [func.lower(Artist.name).like(like)]
|
||||
if squashed:
|
||||
matches.append(name_squashed.like(f"%{squashed}%"))
|
||||
# Rank: exact (0) < exact ignoring spacing (1) < prefix (2) <
|
||||
# substring (3) < substring ignoring spacing (4).
|
||||
rank = case(
|
||||
(func.lower(Artist.name) == cleaned.lower(), 0),
|
||||
(func.lower(Artist.name).like(prefix_like), 1),
|
||||
else_=2,
|
||||
(func.lower(Artist.name) == low, 0),
|
||||
(name_squashed == squashed, 1),
|
||||
(func.lower(Artist.name).like(prefix_like), 2),
|
||||
(func.lower(Artist.name).like(like), 3),
|
||||
else_=4,
|
||||
).label("rank")
|
||||
rows = (await self.session.execute(
|
||||
select(Artist, rank)
|
||||
.where(func.lower(Artist.name).like(like))
|
||||
.where(or_(*matches))
|
||||
.order_by(rank, Artist.name.asc())
|
||||
.limit(limit)
|
||||
)).all()
|
||||
|
||||
@@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
|
||||
2026-07-02 — the import-side filter (Importer._single_color_hit /
|
||||
SkipReason.single_color), so what the audit flags and what the import
|
||||
skips can never disagree.
|
||||
|
||||
It is meant to catch the blank: a placeholder, an error tile, a solid fill.
|
||||
Line art is the case it must not catch (#4483): a pencil doodle on white can be
|
||||
well over 95% white even at full size, and a smoothing downsample blends its strokes
|
||||
into the paper until it measures as blank. So the sample is taken by NEAREST
|
||||
(each sampled pixel is a real pixel, strokes keep their contrast), at 256px
|
||||
so thin strokes are still hit, and the default threshold is near-total
|
||||
(0.995) — a few percent of ink is a drawing, not an empty image.
|
||||
"""
|
||||
|
||||
from PIL import Image
|
||||
|
||||
_THUMB_SIZE = (64, 64)
|
||||
_THUMB_SIZE = (256, 256)
|
||||
|
||||
|
||||
def evaluate(
|
||||
@@ -20,7 +28,8 @@ def evaluate(
|
||||
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||
distance) of the dominant color exceeds `threshold`.
|
||||
|
||||
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
||||
Samples 256x256 by NEAREST (see the module docstring for why not a
|
||||
smoothing resample).
|
||||
Alpha channels are stripped; only RGB is considered. Animated images
|
||||
use frame 0 (PIL's default after Image.open without seek).
|
||||
"""
|
||||
@@ -30,7 +39,7 @@ def evaluate(
|
||||
elif im.mode not in ("RGB", "L"):
|
||||
im = im.convert("RGB")
|
||||
if im.size != _THUMB_SIZE:
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
||||
im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST)
|
||||
pixels = list(im.getdata())
|
||||
if not pixels:
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,752 @@
|
||||
"""Native Discord read client — the Discord counterpart to subscribestar_client.
|
||||
|
||||
Mirrors gallery-dl 1.32.13's `extractor/discord.py` (rule 130: gallery-dl is the
|
||||
known-working base), adapted to the native core's client contract
|
||||
(`ingest_core` module docstring): `iter_posts` / `extract_media`, plus the
|
||||
post-first `post_record_key` and the `post_meta` date the revisit window reads.
|
||||
|
||||
What is mirrored exactly, because drift in any of it changes what we fetch or
|
||||
where it lands on disk:
|
||||
|
||||
- API v10, `Authorization: <user token>` (a USER token, not a bot token).
|
||||
- gallery-dl's request profile: its date-derived Firefox User-Agent,
|
||||
`Accept: */*`, `Accept-Language`, `Referer: https://discord.com/`.
|
||||
- `GET /channels/{id}/messages?limit=100&before=<last id>`, newest first,
|
||||
stopping on a short page. Message types {0, 19, 21} only.
|
||||
- The walk: a text/news channel's own messages then its threads, a forum's
|
||||
threads, a category's children, a server's text/news/forum channels.
|
||||
- Files: attachments, then embeds of type image/gifv/video (FC configures
|
||||
`embeds: all`, which for files is the same three plus rich/link embeds that
|
||||
carry an image), then forwarded `message_snapshots`, numbered from 1 across
|
||||
the lot — the `num` in `{date}_{message_id}_{num}_{filename}`.
|
||||
- Text: `content`, rich-embed author/title/description/fields/footer, poll.
|
||||
|
||||
Two deliberate departures, both about the walk order, neither about content:
|
||||
|
||||
- Threads are walked newest-CREATED first (by id), not by last-message time.
|
||||
A backfill resumes from a checkpointed channel; last-message order shifts
|
||||
between chunks whenever someone posts, which can move an unwalked thread
|
||||
above the resume point and skip it. Creation order only ever grows at the
|
||||
front, where the next tick finds it.
|
||||
- A 403 on a thread or a nested channel skips that feed instead of failing
|
||||
the walk. gallery-dl skips only nested channels; one private thread the
|
||||
token cannot read would otherwise stop every channel after it.
|
||||
|
||||
And one about content: a message is taken only if it has an image or video
|
||||
ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
|
||||
as post, and the operator wants the art, not the conversation (2026-09-28): a
|
||||
text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
|
||||
chat. A message that passes keeps every file gallery-dl would take, numbered
|
||||
as gallery-dl numbers them, so the on-disk names still match.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from urllib.parse import unquote
|
||||
|
||||
import requests
|
||||
|
||||
from .native_ingest_common import (
|
||||
NativeAuthError,
|
||||
NativeDriftError,
|
||||
NativeIngestError,
|
||||
retry_after_seconds,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
API_ROOT = "https://discord.com/api/v10"
|
||||
_ROOT = "https://discord.com"
|
||||
|
||||
_TIMEOUT_SECONDS = 60.0
|
||||
_MESSAGES_BATCH = 100
|
||||
_THREADS_BATCH = 25
|
||||
# gallery-dl retries a 429 up to its default 4 retries, waiting
|
||||
# `request_interval_429` (60s) between them. Discord's Retry-After is exact, so
|
||||
# it is honoured when present; 60s is the fallback and the cap.
|
||||
_MAX_429_RETRIES = 4
|
||||
_429_WAIT_SECONDS = 60.0
|
||||
# How far back the poster picker looks: two pages. Enough to see who posts in
|
||||
# a channel, few enough to answer while the operator waits (#4488).
|
||||
_POSTER_SCAN = 200
|
||||
|
||||
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||||
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||||
MESSAGE_TYPES = frozenset({0, 19, 21})
|
||||
# https://discord.com/developers/docs/resources/channel#channel-object-channel-types
|
||||
_TEXT = frozenset({0, 5}) # text, announcement: messages + threads
|
||||
_DIRECT = frozenset({1, 3, 10, 11, 12}) # DMs and threads: messages only
|
||||
_FORUM = frozenset({15, 16}) # forum, media: threads only
|
||||
_CATEGORY = 4
|
||||
_SERVER_WALK = _TEXT | _FORUM
|
||||
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
|
||||
# What makes a message art rather than chat: an attached file FC imports as an
|
||||
# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
|
||||
# import pipeline, and this one is a network client).
|
||||
_VISUAL_EXTS = frozenset({
|
||||
"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
|
||||
"mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
|
||||
})
|
||||
|
||||
_URL_RE = re.compile(
|
||||
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
|
||||
r"(?P<server>@me|\d+)(?:/(?:\d+/threads/)?(?P<channel>\d+))?(?P<rest>/.*)?/?$"
|
||||
)
|
||||
|
||||
|
||||
class DiscordAPIError(NativeIngestError):
|
||||
"""Base for native Discord client failures."""
|
||||
|
||||
|
||||
class DiscordAuthError(DiscordAPIError, NativeAuthError):
|
||||
"""401 (the token is invalid or expired) or a 403 on the channel the
|
||||
source names. The fix is a new token, not a new client."""
|
||||
|
||||
|
||||
class DiscordDriftError(DiscordAPIError, NativeDriftError):
|
||||
"""A response did not have the shape the walk depends on."""
|
||||
|
||||
|
||||
def firefox_user_agent(today: date | None = None) -> str:
|
||||
"""gallery-dl's default User-Agent: a Firefox whose version advances every
|
||||
four weeks (`util._ff_ver`, "147 on 2026-01-13"). Computed the same way so
|
||||
the profile keeps matching the gallery-dl this replaced."""
|
||||
ver = ((today or date.today()).toordinal() - 735_513) // 28
|
||||
return (
|
||||
f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:{ver}.0) "
|
||||
f"Gecko/20100101 Firefox/{ver}.0"
|
||||
)
|
||||
|
||||
|
||||
def nameext_from_url(url: str) -> tuple[str, str]:
|
||||
"""gallery-dl's `text.nameext_from_url`: the URL's last path segment,
|
||||
unquoted, split at the last dot when the extension is at most 16 chars
|
||||
(lowercased); otherwise the whole name and no extension."""
|
||||
filename = unquote(url.partition("?")[0].rpartition("/")[2])
|
||||
name, _, ext = filename.rpartition(".")
|
||||
if name and len(ext) <= 16:
|
||||
return name, ext.lower()
|
||||
return filename, ""
|
||||
|
||||
|
||||
def parse_source_url(url: str) -> tuple[str | None, str | None]:
|
||||
"""`(server_id, channel_id)` from a Discord channel/server URL. `server_id`
|
||||
is None for a DM (`@me`); `channel_id` is None for a whole server. Raises
|
||||
DiscordAPIError for anything else, including a link to a single message —
|
||||
a message is not something a source can subscribe to."""
|
||||
m = _URL_RE.match((url or "").strip())
|
||||
if not m or (m.group("rest") or "").strip("/"):
|
||||
raise DiscordAPIError(
|
||||
f"Not a Discord channel or server link: {url!r} "
|
||||
"(expected https://discord.com/channels/<server>[/<channel>])"
|
||||
)
|
||||
server = m.group("server")
|
||||
channel = m.group("channel")
|
||||
if server == "@me":
|
||||
if not channel:
|
||||
raise DiscordAPIError(f"A DM link needs a channel id: {url!r}")
|
||||
return None, channel
|
||||
return server, channel
|
||||
|
||||
|
||||
def message_text(message: dict) -> str:
|
||||
"""gallery-dl's `extract_message_text`: the body plus the text of rich
|
||||
embeds and polls, newline-joined, empties dropped."""
|
||||
parts = [message.get("content") or ""]
|
||||
for embed in message.get("embeds") or []:
|
||||
if embed.get("type") != "rich":
|
||||
continue
|
||||
parts.append((embed.get("author") or {}).get("name") or "")
|
||||
parts.append(embed.get("title") or "")
|
||||
parts.append(embed.get("description") or "")
|
||||
for fld in embed.get("fields") or []:
|
||||
parts.append(fld.get("name") or "")
|
||||
parts.append(fld.get("value") or "")
|
||||
parts.append((embed.get("footer") or {}).get("text") or "")
|
||||
poll = message.get("poll")
|
||||
if poll:
|
||||
parts.append(((poll.get("question") or {}).get("text")) or "")
|
||||
for answer in poll.get("answers") or []:
|
||||
parts.append(((answer.get("poll_media") or {}).get("text")) or "")
|
||||
return "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _message_and_snapshots(message: dict) -> list[dict]:
|
||||
"""The message itself, then each forwarded message it carries."""
|
||||
return [message] + [
|
||||
(s or {}).get("message") or {}
|
||||
for s in message.get("message_snapshots") or []
|
||||
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||
]
|
||||
|
||||
|
||||
def _is_visual(attachment: dict) -> bool:
|
||||
ctype = (attachment.get("content_type") or "").lower()
|
||||
if ctype.startswith(("image/", "video/")):
|
||||
return True
|
||||
name = attachment.get("filename") or attachment.get("url") or ""
|
||||
return nameext_from_url(name)[1] in _VISUAL_EXTS
|
||||
|
||||
|
||||
def has_visual_attachment(message: dict) -> bool:
|
||||
"""Does this message, or a message it forwards, have an image or video
|
||||
attached? The one test for "art, not chat" — see the module docstring."""
|
||||
return any(
|
||||
att.get("url") and _is_visual(att)
|
||||
for snap in _message_and_snapshots(message)
|
||||
for att in snap.get("attachments") or []
|
||||
)
|
||||
|
||||
|
||||
_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
|
||||
# Words a server name wraps around its creator's: "Todding's Server",
|
||||
# "The Official Todding Discord".
|
||||
_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
|
||||
|
||||
|
||||
def _squash(text: str | None) -> str:
|
||||
return _SQUASH.sub("", (text or "").lower())
|
||||
|
||||
|
||||
def _server_tokens(server_name: str | None) -> list[str]:
|
||||
name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
|
||||
words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
|
||||
tokens = [w for w in words if len(w) >= 3]
|
||||
joined = "".join(words)
|
||||
if len(joined) >= 3 and joined not in tokens:
|
||||
tokens.append(joined)
|
||||
return tokens
|
||||
|
||||
|
||||
def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
|
||||
"""Mark who is probably the creator a server is named for, and order the
|
||||
list for picking: suggested first, then by images posted, then messages.
|
||||
|
||||
Two signals, each a reason the picker shows beside the name:
|
||||
- they own the server;
|
||||
- their username or display name matches the server's name, once
|
||||
"'s Server", "Official", "Discord" and punctuation are taken off
|
||||
("Todding's Server" and "todding").
|
||||
Neither signal alone is proof, which is why this suggests and never
|
||||
decides: the operator ticks the list. Bots are never suggested."""
|
||||
tokens = _server_tokens(server_name)
|
||||
owner = str(owner_id) if owner_id else None
|
||||
ranked = []
|
||||
for p in posters:
|
||||
reasons = []
|
||||
if owner and p.get("id") == owner:
|
||||
reasons.append("owns the server")
|
||||
names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
|
||||
if any(t in n or n in t for t in tokens for n in names):
|
||||
reasons.append("name matches the server")
|
||||
ranked.append({**p, "reasons": reasons, "suggested": False})
|
||||
best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
|
||||
for r in ranked:
|
||||
r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
|
||||
ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
|
||||
return ranked
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||||
split of the URL; `num` is its 1-based position across the message's files,
|
||||
which is what names it on disk.
|
||||
|
||||
`media_id` is what the seen-ledger keys on, and it is deliberately NOT
|
||||
`num`: an edit that removes a file renumbers the ones after it, and a
|
||||
positional key would then call a different file seen. It is the
|
||||
attachment's id, or for an embed (which has none) a hash of its URL path —
|
||||
the query string is a signature that changes on every fetch. `filehash` is
|
||||
always None; nothing in a signed CDN URL is a content hash."""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
extension: str
|
||||
kind: str
|
||||
post_id: str
|
||||
num: int
|
||||
media_id: str
|
||||
filehash: str | None = None
|
||||
|
||||
|
||||
class DiscordClient:
|
||||
"""Synchronous Discord API v10 read client for one user token."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token: str | None,
|
||||
*,
|
||||
request_sleep: float = 0.0,
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
self._session = session or requests.Session()
|
||||
self._session.headers.update({
|
||||
"User-Agent": firefox_user_agent(),
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Referer": _ROOT + "/",
|
||||
})
|
||||
if token:
|
||||
self._session.headers["Authorization"] = token
|
||||
self._token = token
|
||||
self._request_sleep = request_sleep or 0.0
|
||||
self._max_retries = max_retries
|
||||
self._server: dict = {}
|
||||
self._channels: dict[str, dict] = {}
|
||||
self._skip_feed = False
|
||||
# Whose messages the walk takes (`only_from`); empty takes everyone's.
|
||||
self._authors: frozenset[str] = frozenset()
|
||||
|
||||
# -- request -----------------------------------------------------------
|
||||
|
||||
def _get(self, endpoint: str, params: dict | None = None):
|
||||
if not self._token:
|
||||
raise DiscordAuthError("No Discord token is configured for this source")
|
||||
if self._request_sleep > 0:
|
||||
time.sleep(self._request_sleep)
|
||||
url = API_ROOT + endpoint
|
||||
attempt = 0
|
||||
while True:
|
||||
try:
|
||||
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
||||
except requests.RequestException as exc:
|
||||
raise DiscordAPIError(f"Discord request failed ({endpoint}): {exc}") from exc
|
||||
if resp.status_code == 429 and attempt < self._max_retries:
|
||||
attempt += 1
|
||||
delay = retry_after_seconds(
|
||||
resp, attempt, base=_429_WAIT_SECONDS, cap=_429_WAIT_SECONDS,
|
||||
)
|
||||
log.warning(
|
||||
"Discord 429 (%s) — waiting %.1fs (retry %d/%d)",
|
||||
endpoint, delay, attempt, self._max_retries,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
break
|
||||
if resp.status_code == 401:
|
||||
raise DiscordAuthError(
|
||||
"Discord rejected the token (HTTP 401) — it is invalid or has "
|
||||
"expired; copy a fresh one from the browser",
|
||||
status_code=401,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise DiscordAPIError(
|
||||
f"Discord returned HTTP {resp.status_code} ({endpoint})",
|
||||
status_code=resp.status_code,
|
||||
retry_after=_retry_after(resp),
|
||||
)
|
||||
try:
|
||||
return resp.json()
|
||||
except ValueError as exc:
|
||||
raise DiscordDriftError(
|
||||
f"Discord returned non-JSON for {endpoint} ({len(resp.content)} bytes)"
|
||||
) from exc
|
||||
|
||||
# -- metadata (gallery-dl parse_server / parse_channel) -----------------
|
||||
|
||||
def _load_server(self, server_id: str) -> None:
|
||||
server = self._get(f"/guilds/{server_id}")
|
||||
if not isinstance(server, dict) or "id" not in server:
|
||||
raise DiscordDriftError(f"Discord server {server_id} came back without an id")
|
||||
self._server = {
|
||||
"server": server.get("name") or "",
|
||||
"server_id": str(server["id"]),
|
||||
"owner_id": server.get("owner_id"),
|
||||
}
|
||||
channels = self._get(f"/guilds/{server_id}/channels")
|
||||
if not isinstance(channels, list):
|
||||
raise DiscordDriftError(f"Discord server {server_id} channel list is not a list")
|
||||
# Categories first, so every child can name its parent.
|
||||
for channel in sorted(channels, key=lambda ch: ch.get("type") != _CATEGORY):
|
||||
self._parse_channel(channel)
|
||||
|
||||
def _parse_channel(self, channel: dict) -> dict:
|
||||
parent_id = channel.get("parent_id")
|
||||
meta = {
|
||||
"channel": channel.get("name") or "",
|
||||
"channel_id": str(channel.get("id")),
|
||||
"channel_type": channel.get("type"),
|
||||
"channel_topic": channel.get("topic") or "",
|
||||
"parent_id": parent_id,
|
||||
"is_thread": "thread_metadata" in channel,
|
||||
}
|
||||
parent = self._channels.get(parent_id) if parent_id else None
|
||||
if parent:
|
||||
meta["parent"] = parent["channel"]
|
||||
meta["parent_type"] = parent["channel_type"]
|
||||
if meta["channel_type"] in {1, 3}:
|
||||
recipients = channel.get("recipients") or []
|
||||
meta["channel"] = "DMs"
|
||||
meta["recipients"] = [u.get("username") for u in recipients]
|
||||
meta["recipients_id"] = [u.get("id") for u in recipients]
|
||||
self._channels[meta["channel_id"]] = meta
|
||||
return meta
|
||||
|
||||
def _channel_meta(self, channel_id: str) -> dict:
|
||||
if channel_id not in self._channels:
|
||||
self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||||
return self._channels[channel_id]
|
||||
|
||||
def _threads(self, channel_id: str) -> list[dict]:
|
||||
"""Every thread of a channel or forum, newest-created first (see the
|
||||
module docstring for why not last-message order)."""
|
||||
threads: list[dict] = []
|
||||
offset = 0
|
||||
while True:
|
||||
data = self._get(f"/channels/{channel_id}/threads/search", {
|
||||
"sort_by": "last_message_time",
|
||||
"sort_order": "desc",
|
||||
"limit": _THREADS_BATCH,
|
||||
"offset": offset,
|
||||
})
|
||||
batch = (data.get("threads") or []) if isinstance(data, dict) else []
|
||||
threads.extend(batch)
|
||||
if len(batch) < _THREADS_BATCH:
|
||||
break
|
||||
offset += len(batch)
|
||||
threads.sort(key=lambda t: int(t.get("id") or 0), reverse=True)
|
||||
return threads
|
||||
|
||||
# -- the walk ------------------------------------------------------------
|
||||
|
||||
def _feeds(self, channel_id: str, *, safe: bool) -> Iterator[tuple[str, bool]]:
|
||||
"""`(channel_id, safe)` for every message feed under `channel_id`, in
|
||||
gallery-dl's order. `safe` feeds are skipped on a 403."""
|
||||
try:
|
||||
ctype = self._channel_meta(channel_id)["channel_type"]
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code != 403:
|
||||
raise
|
||||
if not safe:
|
||||
raise DiscordAuthError(
|
||||
f"The Discord token cannot see channel {channel_id} (HTTP 403)",
|
||||
status_code=403,
|
||||
) from exc
|
||||
log.info("Discord: no access to channel %s — skipped", channel_id)
|
||||
return
|
||||
if ctype in _TEXT or ctype in _DIRECT:
|
||||
yield channel_id, safe
|
||||
if ctype in _TEXT or ctype in _FORUM:
|
||||
try:
|
||||
threads = self._threads(channel_id)
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code != 403:
|
||||
raise
|
||||
log.info("Discord: cannot list threads of %s — skipped", channel_id)
|
||||
threads = []
|
||||
for thread in threads:
|
||||
yield self._parse_channel(thread)["channel_id"], True
|
||||
elif ctype == _CATEGORY:
|
||||
for child in list(self._channels.values()):
|
||||
if child.get("parent_id") == channel_id:
|
||||
yield from self._feeds(child["channel_id"], safe=True)
|
||||
elif ctype not in _DIRECT and not safe:
|
||||
raise DiscordAPIError(
|
||||
f"Discord channel {channel_id} is of type {ctype}, which has no messages"
|
||||
)
|
||||
|
||||
def _source_feeds(self, url: str) -> Iterator[tuple[str, bool]]:
|
||||
server_id, channel_id = parse_source_url(url)
|
||||
self._server, self._channels = {}, {}
|
||||
if server_id is not None:
|
||||
self._load_server(server_id)
|
||||
if channel_id is not None:
|
||||
yield from self._feeds(channel_id, safe=False)
|
||||
return
|
||||
for meta in list(self._channels.values()):
|
||||
if meta["channel_type"] in _SERVER_WALK:
|
||||
yield from self._feeds(meta["channel_id"], safe=True)
|
||||
|
||||
def only_from(self, authors) -> None:
|
||||
"""Take only messages posted by these people: user ids, usernames or
|
||||
display names, any case. A creator's server is full of other members
|
||||
posting their own pictures; the source is subscribed to the creator.
|
||||
Empty or None takes everyone's, as before."""
|
||||
self._authors = frozenset(
|
||||
str(a).strip().lower() for a in authors or () if str(a).strip()
|
||||
)
|
||||
|
||||
def _author_wanted(self, message: dict) -> bool:
|
||||
if not self._authors:
|
||||
return True
|
||||
author = message.get("author") or {}
|
||||
names = (author.get("id"), author.get("username"), author.get("global_name"))
|
||||
return any(str(n).lower() in self._authors for n in names if n)
|
||||
|
||||
def source_label(self, url: str) -> str | None:
|
||||
"""What the source is called in Discord — `Server · #channel`, a
|
||||
thread as `#parent › thread`, a whole server by its name — from the
|
||||
metadata the walk loaded. None when the walk never got that far."""
|
||||
try:
|
||||
_, channel_id = parse_source_url(url)
|
||||
except DiscordAPIError:
|
||||
return None
|
||||
server = self._server.get("server") or None
|
||||
if channel_id is None:
|
||||
return server
|
||||
meta = self._channels.get(channel_id) or {}
|
||||
name = meta.get("channel")
|
||||
if not name:
|
||||
return None
|
||||
if name != "DMs":
|
||||
name = f"#{name}"
|
||||
if meta.get("is_thread") and meta.get("parent"):
|
||||
name = f"#{meta['parent']} › {meta['channel']}"
|
||||
return f"{server} · {name}" if server else name
|
||||
|
||||
def skip_feed(self) -> None:
|
||||
"""Optional core seam (#4413): end the current channel and go on to the
|
||||
next one. A tick's early-out means THIS channel has nothing new, not
|
||||
that the server has nothing new."""
|
||||
self._skip_feed = True
|
||||
|
||||
def iter_posts(
|
||||
self, campaign_id: str, cursor: str | None = None
|
||||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||
"""Yield `(message, channel_meta, page_cursor)` for every content
|
||||
message the source reaches, channel by channel, each newest first.
|
||||
|
||||
`campaign_id` is the source URL. The cursor is `<channel_id>:<before>`
|
||||
— the channel and the `before` id that fetched the page (empty for a
|
||||
channel's first page) — so a backfill resumes inside the right channel
|
||||
and re-fetches the page it was cut in. A cursor naming a channel the
|
||||
walk no longer reaches (a deleted thread) restarts from the top rather
|
||||
than walking nothing.
|
||||
"""
|
||||
resume_channel, _, resume_before = (cursor or "").partition(":")
|
||||
resuming = bool(resume_channel)
|
||||
feeds = list(self._source_feeds(campaign_id)) if resuming else None
|
||||
if feeds is not None and resume_channel not in {cid for cid, _ in feeds}:
|
||||
log.warning(
|
||||
"Discord: resume channel %s is no longer in %s — restarting",
|
||||
resume_channel, campaign_id,
|
||||
)
|
||||
resuming = False
|
||||
for channel_id, safe in feeds if feeds is not None else self._source_feeds(campaign_id):
|
||||
before = None
|
||||
if resuming:
|
||||
if channel_id != resume_channel:
|
||||
continue
|
||||
resuming = False
|
||||
before = resume_before or None
|
||||
yield from self._iter_channel(channel_id, before, safe=safe)
|
||||
|
||||
def _iter_channel(
|
||||
self, channel_id: str, before: str | None, *, safe: bool
|
||||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||
self._skip_feed = False
|
||||
meta = {**self._server, **self._channels.get(channel_id, {})}
|
||||
while True:
|
||||
page_cursor = f"{channel_id}:{before or ''}"
|
||||
try:
|
||||
messages = self._get(
|
||||
f"/channels/{channel_id}/messages",
|
||||
{"limit": _MESSAGES_BATCH, "before": before},
|
||||
)
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code != 403:
|
||||
raise
|
||||
if not safe:
|
||||
raise DiscordAuthError(
|
||||
f"The Discord token cannot read channel {channel_id} (HTTP 403)",
|
||||
status_code=403,
|
||||
) from exc
|
||||
log.info("Discord: no access to messages of %s — skipped", channel_id)
|
||||
return
|
||||
if not isinstance(messages, list):
|
||||
raise DiscordDriftError(
|
||||
f"Discord messages of {channel_id} came back as "
|
||||
f"{type(messages).__name__}, not a list"
|
||||
)
|
||||
for message in messages:
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
if not self._author_wanted(message):
|
||||
continue
|
||||
message["_meta"] = meta
|
||||
yield message, meta, page_cursor
|
||||
if self._skip_feed:
|
||||
return
|
||||
if len(messages) < _MESSAGES_BATCH:
|
||||
return
|
||||
before = str(messages[-1]["id"])
|
||||
|
||||
# -- per-message -------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
|
||||
"""gallery-dl's file list for one message: attachments, then the first
|
||||
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
|
||||
the same for every forwarded snapshot; numbered from 1 across them.
|
||||
|
||||
Empty for a message with no image or video attached: that is chat,
|
||||
whatever else it carries (`has_visual_attachment`)."""
|
||||
if not has_visual_attachment(post):
|
||||
return []
|
||||
mid = str(post.get("id") or "")
|
||||
snapshots = _message_and_snapshots(post)
|
||||
found: list[tuple[str, str, str | None]] = []
|
||||
for snap in snapshots:
|
||||
for att in snap.get("attachments") or []:
|
||||
if att.get("url"):
|
||||
aid = att.get("id")
|
||||
found.append((att["url"], "attachment", str(aid) if aid else None))
|
||||
for embed in snap.get("embeds") or []:
|
||||
if embed.get("type") not in _EMBED_TYPES:
|
||||
continue
|
||||
for fld in ("video", "image", "thumbnail"):
|
||||
url = (embed.get(fld) or {}).get("proxy_url")
|
||||
if url:
|
||||
found.append((url, "embed", None))
|
||||
break
|
||||
items = []
|
||||
for num, (url, kind, fid) in enumerate(found, start=1):
|
||||
name, ext = nameext_from_url(url)
|
||||
if fid is None:
|
||||
path = url.partition("?")[0].encode()
|
||||
fid = "u" + hashlib.sha1(path, usedforsecurity=False).hexdigest()[:32]
|
||||
items.append(MediaItem(
|
||||
url=url, filename=name, extension=ext, kind=kind, post_id=mid,
|
||||
num=num, media_id=fid,
|
||||
))
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def post_meta(post: dict) -> dict:
|
||||
"""No title (Discord has none); `date` is the message timestamp, ISO
|
||||
with an offset — what the core's revisit window reads."""
|
||||
return {"title": None, "date": post.get("timestamp")}
|
||||
|
||||
@classmethod
|
||||
def post_record_key(cls, post: dict) -> tuple[str, str] | None:
|
||||
"""`(message:<id>, <id>)` — gates the message record through the seen
|
||||
ledger, like `post:<id>` on the other platforms.
|
||||
|
||||
None for a message `extract_media` takes nothing from, i.e. one with
|
||||
no image or video attached. The drop grouping (discord_grouping) is
|
||||
built on that: a channel's chatter recorded as posts would bury the
|
||||
drops it exists to surface."""
|
||||
mid = post.get("id")
|
||||
mid = str(mid) if mid is not None else ""
|
||||
if not mid or not cls.extract_media(post):
|
||||
return None
|
||||
return (f"message:{mid}", mid)
|
||||
|
||||
# -- verify ------------------------------------------------------------
|
||||
|
||||
def describe(self, server_id: str | None, channel_id: str | None) -> dict:
|
||||
"""The display names behind a server/channel pair, for the browser
|
||||
extension's Add panel. Best-effort per name: one that can't be read
|
||||
comes back None, and the other is still returned."""
|
||||
out: dict = {"server": None, "channel": None, "parent": None}
|
||||
if server_id:
|
||||
try:
|
||||
out["server"] = (self._get(f"/guilds/{server_id}") or {}).get("name") or None
|
||||
except DiscordAPIError:
|
||||
pass
|
||||
if channel_id:
|
||||
try:
|
||||
meta = self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||||
out["channel"] = meta.get("channel") or None
|
||||
except (DiscordAPIError, AttributeError):
|
||||
pass
|
||||
return out
|
||||
|
||||
def recent_posters(
|
||||
self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
|
||||
) -> dict:
|
||||
"""Who has posted lately in a channel, for picking a source's poster
|
||||
list (#4488): the newest `max_messages` messages, tallied per author.
|
||||
Deliberately shallow — it answers "who posts here", not "who ever did".
|
||||
Every poster comes back with their stable id, which is what the list
|
||||
stores: a username can change on a whim, an id can't."""
|
||||
server: dict = {}
|
||||
if server_id:
|
||||
try:
|
||||
server = self._get(f"/guilds/{server_id}") or {}
|
||||
except DiscordAPIError:
|
||||
server = {}
|
||||
tally: dict[str, dict] = {}
|
||||
before = None
|
||||
seen = 0
|
||||
while seen < max_messages:
|
||||
page = self._get(
|
||||
f"/channels/{channel_id}/messages",
|
||||
{"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
|
||||
)
|
||||
if not isinstance(page, list) or not page:
|
||||
break
|
||||
for message in page:
|
||||
seen += 1
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
author = message.get("author") or {}
|
||||
aid = str(author.get("id") or "")
|
||||
if not aid:
|
||||
continue
|
||||
row = tally.setdefault(aid, {
|
||||
"id": aid,
|
||||
"username": author.get("username"),
|
||||
"global_name": author.get("global_name"),
|
||||
"bot": bool(author.get("bot")),
|
||||
"messages": 0,
|
||||
"images": 0,
|
||||
})
|
||||
row["messages"] += 1
|
||||
if has_visual_attachment(message):
|
||||
row["images"] += 1
|
||||
if len(page) < _MESSAGES_BATCH:
|
||||
break
|
||||
before = str(page[-1]["id"])
|
||||
return {
|
||||
"server": server.get("name") or None,
|
||||
"owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
|
||||
"scanned": seen,
|
||||
"posters": rank_posters(
|
||||
server.get("name"), server.get("owner_id"), list(tally.values()),
|
||||
),
|
||||
}
|
||||
|
||||
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||||
"""Is the token valid, and can it see what the source names?"""
|
||||
try:
|
||||
server_id, channel_id = parse_source_url(url)
|
||||
except DiscordAPIError as exc:
|
||||
return None, str(exc)
|
||||
try:
|
||||
me = self._get("/users/@me")
|
||||
if channel_id is not None:
|
||||
self._get(f"/channels/{channel_id}")
|
||||
elif server_id is not None:
|
||||
self._get(f"/guilds/{server_id}")
|
||||
except DiscordAuthError as exc:
|
||||
return False, f"Discord rejected the token — {exc}"
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code in (403, 404):
|
||||
return False, (
|
||||
"The token is valid, but its account cannot see "
|
||||
f"{'this channel' if channel_id else 'this server'} "
|
||||
f"(HTTP {exc.status_code})"
|
||||
)
|
||||
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||||
who = (me or {}).get("username") if isinstance(me, dict) else None
|
||||
return True, f"Token valid{f' ({who})' if who else ''} — the source is readable."
|
||||
|
||||
|
||||
def _retry_after(resp: requests.Response) -> float | None:
|
||||
hdr = resp.headers.get("Retry-After")
|
||||
try:
|
||||
return float(hdr) if hdr else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Native Discord media downloader — the Discord counterpart to
|
||||
subscribestar_downloader.
|
||||
|
||||
Writes files exactly where gallery-dl wrote them, so a cutover finds every
|
||||
existing file on disk (`skipped_disk`) instead of fetching it again:
|
||||
|
||||
<images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>
|
||||
|
||||
That is what FC's gallery-dl config produced (directory `{channel}`, filename
|
||||
`{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the
|
||||
per-source base directory `<images>/<artist>/<platform>`), retired from that
|
||||
config once Discord moved here; tests/test_discord_naming.py pins the match
|
||||
against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it
|
||||
on Linux — `/` becomes `_` and control characters are removed, nothing else
|
||||
(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
|
||||
whose Windows set would turn a `:` in a channel or file name into `_` and miss
|
||||
the file gallery-dl wrote.
|
||||
|
||||
Post-first (rule 120): each file gets a minimal sidecar named like it minus the
|
||||
extension (what `find_sidecar` pairs first), and the message itself gets one
|
||||
record, `<YYYYMMDD>_<message_id>_post.json`, carrying gallery-dl's metadata keys
|
||||
— `message_id` for the post id, `server_id`/`channel_id` for the permalink,
|
||||
`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read
|
||||
the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both
|
||||
outrank `message_id` in the post-id chain (`platforms.base`).
|
||||
|
||||
PURE: no DB; the seen-skip is an injected predicate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .discord_client import firefox_user_agent, message_text
|
||||
from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
make_session,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM = "discord"
|
||||
_CONTROL = re.compile("[\x00-\x1f\x7f]")
|
||||
# gallery-dl falls back to the response's type for a URL with no extension;
|
||||
# we never see the response before naming, and such URLs do not occur for
|
||||
# Discord attachments or embed proxies in practice.
|
||||
_NO_EXTENSION = "bin"
|
||||
|
||||
|
||||
def gdl_clean(segment: str) -> str:
|
||||
"""One path segment as gallery-dl writes it on Linux."""
|
||||
return _CONTROL.sub("", segment.replace("/", "_"))
|
||||
|
||||
|
||||
def message_date(post: dict) -> datetime | None:
|
||||
raw = post.get("timestamp")
|
||||
if not isinstance(raw, str) or not raw:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC)
|
||||
|
||||
|
||||
def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path:
|
||||
"""gallery-dl's `{channel}` directory; an empty name adds no segment."""
|
||||
base = Path(images_root) / artist_slug / PLATFORM
|
||||
channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip())
|
||||
return base / channel if channel else base
|
||||
|
||||
|
||||
def media_stem(post: dict, media) -> str:
|
||||
"""`<YYYYMMDD>_<message_id>_<NN>_<name>` — the file's name minus `.<ext>`."""
|
||||
when = message_date(post)
|
||||
day = f"{when:%Y%m%d}" if when else "None"
|
||||
return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}")
|
||||
|
||||
|
||||
class DiscordDownloader(BaseNativeDownloader):
|
||||
"""Download a message's files to gallery-dl's layout. The CDN gets
|
||||
gallery-dl's browser profile and no token — gallery-dl sends the token only
|
||||
to the API, and the CDN URLs are pre-signed."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
images_root, None, platform=PLATFORM,
|
||||
validate=validate, rate_limit=rate_limit,
|
||||
session=session if session is not None else make_session(None, extra_headers={
|
||||
"User-Agent": firefox_user_agent(),
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Referer": "https://discord.com/",
|
||||
}),
|
||||
)
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Every file of one message; per-file outcomes, one failure isolated."""
|
||||
folder = channel_dir(self.images_root, artist_slug, post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for media in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(self._download_one(
|
||||
post, media, folder, artist_slug, is_seen, recapture=recapture,
|
||||
))
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"Discord media failed (message %s, file %d): %s",
|
||||
post.get("id"), media.num, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
folder: Path,
|
||||
artist_slug: str,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
stem = media_stem(post, media)
|
||||
path = folder / f"{stem}.{media.extension or _NO_EXTENSION}"
|
||||
if path.exists(): # tier-2: gallery-dl (or an earlier walk) wrote it
|
||||
return MediaOutcome(media=media, status="skipped_disk", path=path, error=None)
|
||||
if seen: # recapture never re-fetches a seen file that is gone
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
out = self._fetch_get(media.url, path)
|
||||
reason, quarantined = self._validate_path(out, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
|
||||
sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")}
|
||||
sidecar["source_url"] = media.url
|
||||
(folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2))
|
||||
return MediaOutcome(media=media, status="downloaded", path=out, error=None)
|
||||
|
||||
def write_post_record(
|
||||
self, post: dict, artist_slug: str, *, revisit: bool = False,
|
||||
) -> PostRecordOutcome:
|
||||
"""The message record — the one writer of a Discord post's body, date
|
||||
and permalink ids. `revisit` re-reads a message already captured (an
|
||||
edit); an empty re-read writes nothing, so it never blanks a body."""
|
||||
mid = str(post.get("id") or "")
|
||||
body = message_text(post)
|
||||
if not mid or (revisit and not body.strip()):
|
||||
return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0)
|
||||
meta = post.get("_meta") or {}
|
||||
author = post.get("author") or {}
|
||||
record = {
|
||||
"category": PLATFORM,
|
||||
"message_id": mid,
|
||||
"server": meta.get("server"),
|
||||
"server_id": meta.get("server_id"),
|
||||
"channel": meta.get("channel"),
|
||||
"channel_id": meta.get("channel_id") or post.get("channel_id"),
|
||||
"parent": meta.get("parent"),
|
||||
"is_thread": meta.get("is_thread"),
|
||||
"author": author.get("username"),
|
||||
"author_name": author.get("global_name"),
|
||||
"author_id": author.get("id"),
|
||||
"message": body,
|
||||
"date": post.get("timestamp"),
|
||||
}
|
||||
folder = channel_dir(self.images_root, artist_slug, post)
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
when = message_date(post)
|
||||
day = f"{when:%Y%m%d}" if when else "None"
|
||||
path = folder / f"{day}_{mid}_post.json"
|
||||
path.write_text(json.dumps(
|
||||
{k: v for k, v in record.items() if v is not None}, indent=2,
|
||||
))
|
||||
return PostRecordOutcome(
|
||||
path=path, post_type=None, title=None, body_chars=len(body),
|
||||
)
|
||||
@@ -63,7 +63,7 @@ from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import Select, delete, func, select, update
|
||||
from sqlalchemy import Select, delete, func, select, union, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
@@ -126,6 +126,31 @@ def cosine_distance(a, b) -> float:
|
||||
return 1.0 - (dot / (na * nb))
|
||||
|
||||
|
||||
def _message_images(posts):
|
||||
"""(post_id, image_id) for every image a message carries — owned AND re-posted.
|
||||
|
||||
A message owns an image through `primary_post_id`, but only the FIRST
|
||||
message imported with a given file does. The same file posted again is a
|
||||
provenance link, and the backfill runs newest-first, so it is usually the
|
||||
ORIGINAL message that ends up owning nothing. Reading ownership alone left
|
||||
101 of Yellowroom's messages (2018–2020 mostly) ungroupable: every image
|
||||
they carried also sat in another message.
|
||||
|
||||
`posts` is a list of ids or a select of them; filtering both branches by it
|
||||
keeps the union to the messages in hand rather than the whole library.
|
||||
Callers pass MESSAGE posts only — a drop's own provenance rows would read
|
||||
as images it carries.
|
||||
"""
|
||||
owned = select(
|
||||
ImageRecord.primary_post_id.label("post_id"), ImageRecord.id.label("image_id"),
|
||||
).where(ImageRecord.primary_post_id.in_(posts))
|
||||
reposted = select(
|
||||
ImageProvenance.post_id.label("post_id"),
|
||||
ImageProvenance.image_record_id.label("image_id"),
|
||||
).where(ImageProvenance.post_id.in_(posts))
|
||||
return union(owned, reposted).subquery()
|
||||
|
||||
|
||||
def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
||||
"""Ungrouped Discord message-posts, one representative image each, OLDEST
|
||||
FIRST — which is the order `build_groups` requires.
|
||||
@@ -143,13 +168,15 @@ def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
||||
take the OLDEST candidates instead of the lowest-numbered ones.
|
||||
"""
|
||||
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||
carried = _message_images(select(Post.id).where(Post.source_id == source_id))
|
||||
inner = (
|
||||
select(
|
||||
Post.id.label("post_id"),
|
||||
sort_key.label("occurred_at"),
|
||||
ImageRecord.siglip_embedding.label("embedding"),
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.primary_post_id == Post.id)
|
||||
.join(carried, carried.c.post_id == Post.id)
|
||||
.join(ImageRecord, ImageRecord.id == carried.c.image_id)
|
||||
.where(
|
||||
Post.source_id == source_id,
|
||||
# Never absorb a post FC wrote, and never re-absorb one already
|
||||
@@ -295,9 +322,10 @@ async def _link_member_images(
|
||||
"""
|
||||
if not member_ids:
|
||||
return 0
|
||||
image_rows = (await session.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids))
|
||||
)).scalars().all()
|
||||
carried = _message_images(member_ids)
|
||||
image_rows = sorted(set((await session.execute(
|
||||
select(carried.c.image_id)
|
||||
)).scalars().all()))
|
||||
if not image_rows:
|
||||
return 0
|
||||
await session.execute(
|
||||
@@ -421,9 +449,11 @@ async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None
|
||||
free to disagree; this way there is one.
|
||||
"""
|
||||
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||
carried = _message_images(select(Post.id).where(Post.absorbed_by_post_id == post_id))
|
||||
return (await session.execute(
|
||||
select(ImageRecord.siglip_embedding)
|
||||
.join(Post, ImageRecord.primary_post_id == Post.id)
|
||||
.join(carried, carried.c.image_id == ImageRecord.id)
|
||||
.join(Post, Post.id == carried.c.post_id)
|
||||
.where(
|
||||
Post.absorbed_by_post_id == post_id,
|
||||
ImageRecord.siglip_embedding.is_not(None),
|
||||
@@ -725,12 +755,15 @@ async def _load_drops(session: AsyncSession, source: Source) -> list[_Drop]:
|
||||
names: dict[int, set[str]] = {pid: set() for pid in by_id}
|
||||
images: dict[int, list[int]] = {pid: [] for pid in by_id}
|
||||
if owner_of:
|
||||
for iid, primary, path in (await session.execute(
|
||||
select(ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path)
|
||||
.where(ImageRecord.primary_post_id.in_(list(owner_of)))
|
||||
carried = _message_images(list(owner_of))
|
||||
for iid, message, path in (await session.execute(
|
||||
select(ImageRecord.id, carried.c.post_id, ImageRecord.path)
|
||||
.join(carried, carried.c.image_id == ImageRecord.id)
|
||||
.order_by(ImageRecord.id)
|
||||
)).all():
|
||||
drop = owner_of[primary]
|
||||
drop = owner_of[message]
|
||||
if iid in images[drop]:
|
||||
continue # one file carried by two of the drop's messages
|
||||
images[drop].append(iid)
|
||||
if (name := leading_name(path)) is not None:
|
||||
names[drop].add(name)
|
||||
@@ -844,7 +877,12 @@ async def merge_trickles(
|
||||
if drop.first_at - earlier.last_at > gap:
|
||||
continue
|
||||
shared = mine & gated(earlier.names)
|
||||
if shared:
|
||||
if set(drop.images) & set(earlier.images):
|
||||
# The creator posted the very same file again — the strongest
|
||||
# evidence there is, and one nearest-neighbour cannot see: it
|
||||
# skips the image itself, which is the one they share.
|
||||
targets[earlier.post.id] = (earlier, "same_image")
|
||||
elif shared:
|
||||
targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
|
||||
elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
|
||||
targets[earlier.post.id] = (earlier, "nearest")
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Native Discord ingester — the Discord ADAPTER over `ingest_core.Ingester`.
|
||||
|
||||
Thin counterpart to subscribestar_ingester (milestone 428). The walk's modes,
|
||||
both ledgers, cursor checkpointing and the post-first capture live in the core;
|
||||
this wires in the Discord client, downloader, ledger models and key.
|
||||
|
||||
Two things differ from the cookie platforms:
|
||||
|
||||
- Discord authenticates with a user TOKEN, so `auth_token` is the credential
|
||||
here rather than an argument accepted and ignored.
|
||||
- The body canary is off. It fails a walk whose first 30+ captured posts all
|
||||
came back without text, on the theory that a creator nearly always writes
|
||||
something; a Discord drop is routinely files and nothing else, so on
|
||||
Discord that is an ordinary backfill, not a broken parser.
|
||||
|
||||
And two things only a Discord source has: `discord_authors` in its
|
||||
config_overrides limits it to the creator's own messages (a server's other
|
||||
members post pictures too), and it learns its own name — the server and
|
||||
channel it walks — into `source.display_name`.
|
||||
|
||||
`campaign_id` is the source URL (a server, channel, thread or category link).
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from ..models import DiscordFailedMedia, DiscordSeenMedia, Source
|
||||
from .discord_client import DiscordAPIError, DiscordClient, MediaItem
|
||||
from .discord_downloader import DiscordDownloader
|
||||
from .ingest_core import Ingester
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_LEDGER_KEY_MAX = 128
|
||||
# `config_overrides` key: the people whose messages this source takes (user
|
||||
# ids, usernames or display names). Absent or empty takes everyone's.
|
||||
AUTHORS_KEY = "discord_authors"
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
"""`<message_id>:<media_id>` — stable across edits (see MediaItem)."""
|
||||
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||
|
||||
|
||||
class DiscordIngester(Ingester):
|
||||
"""Walk a Discord source's channels, download unseen files, return a
|
||||
`DownloadResult`. `client` / `downloader` are injectable for tests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None,
|
||||
session_factory: Callable[[], object],
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
request_sleep: float = 0.0,
|
||||
auth_token: str | None = None,
|
||||
client: DiscordClient | None = None,
|
||||
downloader: DiscordDownloader | None = None,
|
||||
):
|
||||
del cookies_path # Discord authenticates by token (uniform signature)
|
||||
self.images_root = Path(images_root)
|
||||
super().__init__(
|
||||
client=client if client is not None else DiscordClient(
|
||||
auth_token, request_sleep=request_sleep,
|
||||
),
|
||||
downloader=downloader if downloader is not None else DiscordDownloader(
|
||||
self.images_root, validate=validate, rate_limit=rate_limit,
|
||||
),
|
||||
session_factory=session_factory,
|
||||
seen_model=DiscordSeenMedia,
|
||||
failed_model=DiscordFailedMedia,
|
||||
seen_constraint="uq_discord_seen_media_source_id",
|
||||
failed_constraint="uq_discord_failed_media_source_id",
|
||||
ledger_key=_ledger_key,
|
||||
platform="discord",
|
||||
error_base=DiscordAPIError,
|
||||
drift_label="Discord API",
|
||||
body_canary=False,
|
||||
)
|
||||
|
||||
def run(self, **kwargs):
|
||||
"""The core walk, bracketed by the two things only a Discord source
|
||||
has: whose messages it takes, read before the walk, and the name of
|
||||
what it walks, written after it (the walk is what loads that name)."""
|
||||
source_id = kwargs["source_id"]
|
||||
self.client.only_from(self._source_authors(source_id))
|
||||
try:
|
||||
return super().run(**kwargs)
|
||||
finally:
|
||||
self._record_display_name(source_id, kwargs["campaign_id"])
|
||||
|
||||
def _source_authors(self, source_id: int) -> list:
|
||||
if self.session_factory is None:
|
||||
return []
|
||||
with self.session_factory() as session:
|
||||
overrides = session.execute(
|
||||
select(Source.config_overrides).where(Source.id == source_id)
|
||||
).scalar_one_or_none() or {}
|
||||
authors = overrides.get(AUTHORS_KEY) or []
|
||||
return authors if isinstance(authors, list) else []
|
||||
|
||||
def _record_display_name(self, source_id: int, url: str) -> None:
|
||||
"""Refreshed on every walk, never written once: a renamed channel
|
||||
should read as its new name. A name the walk couldn't read leaves the
|
||||
stored one alone, and a failure here never fails the walk."""
|
||||
label = self.client.source_label(url)
|
||||
if not label or self.session_factory is None:
|
||||
return
|
||||
try:
|
||||
with self.session_factory() as session:
|
||||
session.execute(
|
||||
update(Source)
|
||||
.where(Source.id == source_id)
|
||||
.where(Source.display_name.is_distinct_from(label))
|
||||
.values(display_name=label)
|
||||
)
|
||||
session.commit()
|
||||
except Exception as exc: # a name is decoration — never fail the walk
|
||||
log.warning("Discord: couldn't record source %s's name: %s", source_id, exc)
|
||||
|
||||
|
||||
async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
|
||||
"""The uniform `(ok, message)` probe: is the token valid, and can its
|
||||
account see the channel or server the source names?"""
|
||||
if not auth_token:
|
||||
return False, "No Discord token is saved — add one under Credentials."
|
||||
client = DiscordClient(auth_token)
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(None, client.verify_auth, url)
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Remove a Discord source's posts by people outside its poster list (#4486).
|
||||
|
||||
A creator's server is full of other members posting their own pictures. The
|
||||
poster list (`discord_authors`, #4481) stops a walk taking them; this removes
|
||||
the ones taken before the list existed. The operator sets the list first, then
|
||||
previews what would go, then applies.
|
||||
|
||||
## What goes
|
||||
|
||||
A post of this source whose record names its poster (`author_id` / `author`,
|
||||
written by `DiscordDownloader.write_post_record`) and names someone NOT on the
|
||||
list. Two kinds of post are never touched:
|
||||
|
||||
* a post whose record names no poster — a message recorded before the native
|
||||
ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
|
||||
* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
|
||||
below as a consequence, not matched as a poster's post.
|
||||
|
||||
An image goes when every REAL post it belongs to is going. A link to a
|
||||
synthetic drop does not keep an image: the drop only references its members'
|
||||
images. An image also on a kept post — the creator re-posting a piece someone
|
||||
else shared — stays.
|
||||
|
||||
A drop that absorbed a removed message is deleted outright. Deleting a drop is
|
||||
its undo (discord_grouping's honesty rule): its remaining members return to
|
||||
the feed and the grouping sweep regroups them on its next run, so no half-
|
||||
rebuilt drop keeps text or thumbnails from someone it no longer contains.
|
||||
|
||||
Preview and apply spread the same predicates (rule 93, snippet #3087).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import and_, delete, exists, func, or_, select
|
||||
from sqlalchemy.orm import Session, aliased
|
||||
|
||||
from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
|
||||
from .cleanup_service import delete_images
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM = "discord"
|
||||
# The record keys that name a message's poster. `author_name` (the display
|
||||
# name) is only on records written after 2026-09-28.
|
||||
_POSTER_KEYS = ("author_id", "author", "author_name")
|
||||
|
||||
|
||||
class PosterCleanupError(ValueError):
|
||||
"""The source can't be cleaned this way (not Discord, or no list)."""
|
||||
|
||||
|
||||
def source_authors(source: Source) -> list[str]:
|
||||
"""The source's poster list, lowercased; empty means everyone's wanted."""
|
||||
authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
|
||||
if not isinstance(authors, list):
|
||||
return []
|
||||
return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
|
||||
|
||||
|
||||
def _poster(key: str):
|
||||
return func.lower(Post.raw_metadata[key].as_string())
|
||||
|
||||
|
||||
def _has_poster():
|
||||
return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
|
||||
|
||||
|
||||
def _real_post_conditions(source_id: int) -> list:
|
||||
return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
|
||||
|
||||
|
||||
def _named_poster_conditions(source_id: int) -> list:
|
||||
"""A real post of this source whose record says who posted it."""
|
||||
return [*_real_post_conditions(source_id), _has_poster()]
|
||||
|
||||
|
||||
def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The posts that go: a named poster none of whose names is on the list."""
|
||||
return [
|
||||
*_named_poster_conditions(source_id),
|
||||
*(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
|
||||
]
|
||||
|
||||
|
||||
def _doomed_post_ids(source_id: int, authors: list[str]):
|
||||
# correlate(None): used inside queries that are themselves over `post` (the
|
||||
# drops, the delete), where auto-correlation would bind this to the outer
|
||||
# row instead of scanning the table.
|
||||
return (
|
||||
select(Post.id)
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.correlate(None)
|
||||
)
|
||||
|
||||
|
||||
def _image_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""Images that belong to a removed post and to no kept real post."""
|
||||
doomed = _doomed_post_ids(source_id, authors)
|
||||
keeper = aliased(Post)
|
||||
on_doomed = or_(
|
||||
ImageRecord.primary_post_id.in_(doomed),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id.in_(doomed),
|
||||
),
|
||||
)
|
||||
on_kept = or_(
|
||||
and_(
|
||||
ImageRecord.primary_post_id.isnot(None),
|
||||
ImageRecord.primary_post_id.notin_(doomed),
|
||||
),
|
||||
exists().where(
|
||||
ImageProvenance.image_record_id == ImageRecord.id,
|
||||
ImageProvenance.post_id == keeper.id,
|
||||
keeper.synthesized_by.is_(None),
|
||||
keeper.id.notin_(doomed),
|
||||
),
|
||||
)
|
||||
return [on_doomed, ~on_kept]
|
||||
|
||||
|
||||
def _drop_conditions(source_id: int, authors: list[str]) -> list:
|
||||
"""The synthetic drops that absorbed a removed message."""
|
||||
member = aliased(Post)
|
||||
return [
|
||||
Post.source_id == source_id,
|
||||
Post.synthesized_by.isnot(None),
|
||||
exists().where(
|
||||
member.absorbed_by_post_id == Post.id,
|
||||
member.id.in_(_doomed_post_ids(source_id, authors)),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _attachment_conditions(source_id: int, authors: list[str]) -> list:
|
||||
return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
|
||||
|
||||
|
||||
def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
|
||||
source = session.get(Source, source_id)
|
||||
if source is None:
|
||||
raise LookupError(source_id)
|
||||
if source.platform != PLATFORM:
|
||||
raise PosterCleanupError("Only a Discord source has posters to filter by.")
|
||||
authors = source_authors(source)
|
||||
if not authors:
|
||||
raise PosterCleanupError(
|
||||
"Set this source's 'Only posts by' list first — with no list, "
|
||||
"every poster is wanted and nothing would be removed."
|
||||
)
|
||||
return source, authors
|
||||
|
||||
|
||||
def _count(session: Session, stmt) -> int:
|
||||
return session.execute(stmt).scalar_one()
|
||||
|
||||
|
||||
def preview(session: Session, *, source_id: int) -> dict:
|
||||
"""What `apply` would remove, per poster, without touching anything."""
|
||||
_, authors = _load(session, source_id)
|
||||
who = func.coalesce(
|
||||
Post.raw_metadata["author_name"].as_string(),
|
||||
Post.raw_metadata["author"].as_string(),
|
||||
Post.raw_metadata["author_id"].as_string(),
|
||||
)
|
||||
rows = session.execute(
|
||||
select(who, func.count(Post.id))
|
||||
.where(*_other_poster_post_conditions(source_id, authors))
|
||||
.group_by(who)
|
||||
.order_by(func.count(Post.id).desc())
|
||||
).all()
|
||||
posters = [{"poster": name, "posts": n} for name, n in rows]
|
||||
unknown = _count(session, select(func.count(Post.id)).where(
|
||||
*_real_post_conditions(source_id), ~_has_poster(),
|
||||
))
|
||||
return {
|
||||
"authors": authors,
|
||||
"posters": posters,
|
||||
"posts": sum(p["posts"] for p in posters),
|
||||
"images": _count(session, select(func.count(ImageRecord.id)).where(
|
||||
*_image_conditions(source_id, authors))),
|
||||
"attachments": _count(session, select(func.count(PostAttachment.id)).where(
|
||||
*_attachment_conditions(source_id, authors))),
|
||||
"drops": _count(session, select(func.count(Post.id)).where(
|
||||
*_drop_conditions(source_id, authors))),
|
||||
"unknown_posts": unknown,
|
||||
}
|
||||
|
||||
|
||||
def confirm_token(projection: dict) -> str:
|
||||
"""What the operator's apply must echo: the preview it was shown, so a
|
||||
list edited between preview and apply can't delete an unseen set."""
|
||||
return f"remove-{projection['posts']}-posts-{projection['images']}-images"
|
||||
|
||||
|
||||
def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
|
||||
"""Remove them. Counts are taken before the deletes they describe — once
|
||||
the rows are gone there is nothing left to count (snippet #3087 note 2)."""
|
||||
projection = preview(session, source_id=source_id)
|
||||
_, authors = _load(session, source_id)
|
||||
|
||||
image_ids = session.execute(
|
||||
select(ImageRecord.id).where(*_image_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
# Drops first-class: found BEFORE their members go, since the link that
|
||||
# finds them is the member's absorbed_by_post_id.
|
||||
drop_ids = session.execute(
|
||||
select(Post.id).where(*_drop_conditions(source_id, authors))
|
||||
).scalars().all()
|
||||
|
||||
deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
|
||||
# Attachments before posts: post_attachment.post_id is SET NULL, and a
|
||||
# NULL-post attachment collides on its partial unique index (see
|
||||
# cleanup_service.delete_artist_cascade).
|
||||
attachments = session.execute(
|
||||
delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
|
||||
).rowcount or 0
|
||||
posts = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
|
||||
).rowcount or 0
|
||||
drops = 0
|
||||
if drop_ids:
|
||||
drops = session.execute(
|
||||
Post.__table__.delete().where(Post.id.in_(drop_ids))
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
log.info(
|
||||
"discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
|
||||
"%d attachment(s), %d drop(s) removed",
|
||||
source_id, authors, posts, deleted["images_deleted"], attachments, drops,
|
||||
)
|
||||
return {
|
||||
**projection,
|
||||
"posts_deleted": posts,
|
||||
"images_deleted": deleted["images_deleted"],
|
||||
"files_deleted": deleted["files_deleted"],
|
||||
"attachments_deleted": attachments,
|
||||
"drops_deleted": drops,
|
||||
}
|
||||
@@ -23,6 +23,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from .discord_ingester import DiscordIngester
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .ingest_core import DEFAULT_REVISIT_DAYS
|
||||
from .patreon_ingester import PatreonIngester
|
||||
@@ -31,9 +32,13 @@ from .platforms import known_platform_keys
|
||||
from .subscribestar_ingester import SubscribeStarIngester
|
||||
|
||||
# Platforms whose download + verify go through the native ingester rather than
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
|
||||
# they migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry) until it
|
||||
# migrates too. Discord joined in milestone 428.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||
|
||||
# Native platforms whose feed id IS the source URL, so there is nothing to
|
||||
# resolve: SubscribeStar's creator page, Discord's server/channel link.
|
||||
_URL_IS_FEED = frozenset({"subscribestar", "discord"})
|
||||
|
||||
|
||||
def _unsupported_platform_message(platform: str) -> str | None:
|
||||
@@ -67,6 +72,7 @@ def _native_ingester_cls(platform: str):
|
||||
return {
|
||||
"patreon": PatreonIngester,
|
||||
"subscribestar": SubscribeStarIngester,
|
||||
"discord": DiscordIngester,
|
||||
}[platform]
|
||||
|
||||
|
||||
@@ -126,10 +132,10 @@ async def _resolve_native_campaign_id(
|
||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
||||
so phase 3 caches it)."""
|
||||
if platform == "subscribestar":
|
||||
and Discord's feed id IS the source URL (no lookup → resolved None). Patreon
|
||||
resolves the campaign id from the vanity URL (resolved non-None when a lookup
|
||||
actually ran, so phase 3 caches it)."""
|
||||
if platform in _URL_IS_FEED:
|
||||
return url, None
|
||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||
|
||||
@@ -252,6 +258,10 @@ async def verify_source_credential(
|
||||
from .subscribestar_ingester import verify_subscribestar_credential
|
||||
|
||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||
if platform == "discord":
|
||||
from .discord_ingester import verify_discord_credential
|
||||
|
||||
return await verify_discord_credential(url, auth_token)
|
||||
from .patreon_ingester import verify_patreon_credential
|
||||
|
||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Closing the download runs a worker restart orphaned (#4433).
|
||||
|
||||
Its own module, importing nothing but the model, because its caller is the
|
||||
worker boot hook in `celery_signals` — which every download task imports via
|
||||
`celery_app`. Living in `tasks.maintenance` put the whole maintenance import
|
||||
graph, the membership roster included, on the fetch path, which
|
||||
`test_gated_reason` forbids.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import literal, update
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
|
||||
from ..models import DownloadEvent
|
||||
|
||||
DOWNLOAD_INTERRUPTED_MESSAGE = (
|
||||
"interrupted by a worker restart — the next check picks it up where it left off"
|
||||
)
|
||||
|
||||
|
||||
def interrupt_orphaned_download_events(session, *, booted_at: datetime) -> int:
|
||||
"""Close the download events a restart orphaned, without blaming the source.
|
||||
|
||||
Called when the download lane comes up (#4433). Anything still
|
||||
pending/running from before this boot belongs to the previous process:
|
||||
a walk that outlived the 90s stop grace was SIGKILLed, and a queued or
|
||||
serialize-deferred task is held unacked until Redis redelivers it about an
|
||||
hour later. Left alone, the 30-min stall sweep would error each one and
|
||||
bump `consecutive_failures`, backing the source off as if the platform had
|
||||
failed.
|
||||
|
||||
Instead they end as `skipped` (terminal, not a failure) and the source is
|
||||
not touched: `last_checked_at` keeps its old value, so the next tick finds
|
||||
it due and the walk resumes from its checkpoint. A redelivered message
|
||||
that arrives later finds no pending event and opens a fresh one.
|
||||
|
||||
An event promoted to running after the boot has `started_at` reset to its
|
||||
real start (download_service), so it is never caught here. Does NOT commit.
|
||||
"""
|
||||
now = datetime.now(UTC)
|
||||
result = session.execute(
|
||||
update(DownloadEvent)
|
||||
.where(DownloadEvent.status.in_(["pending", "running"]))
|
||||
.where(DownloadEvent.started_at < booted_at)
|
||||
.values(
|
||||
status="skipped",
|
||||
finished_at=now,
|
||||
error=DOWNLOAD_INTERRUPTED_MESSAGE,
|
||||
metadata_=DownloadEvent.metadata_.op("||")(
|
||||
literal({"error_type": "interrupted"}, JSONB)
|
||||
),
|
||||
)
|
||||
.returning(DownloadEvent.id)
|
||||
)
|
||||
return len(result.all())
|
||||
@@ -21,6 +21,15 @@ from .source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# The probe runs while the chip is drawing; names that take longer are skipped.
|
||||
_NAME_LOOKUP_SECONDS = 6.0
|
||||
# The poster picker waits on two pages of a channel's messages (#4488).
|
||||
_POSTER_LOOKUP_SECONDS = 20.0
|
||||
|
||||
# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this
|
||||
# keeps the name each id was picked under, for showing the list to a person.
|
||||
AUTHOR_LABELS_KEY = "discord_author_labels"
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
"""URL didn't match any platform pattern."""
|
||||
@@ -30,6 +39,19 @@ class InvalidUrlError(Exception):
|
||||
"""URL was empty or missing a scheme."""
|
||||
|
||||
|
||||
class UnknownArtistError(Exception):
|
||||
"""quick-add named an `artist_id` that does not exist."""
|
||||
|
||||
|
||||
class PosterLookupError(Exception):
|
||||
"""The Discord poster picker couldn't read the channel (no token, no
|
||||
access, not a channel page, or Discord was slow)."""
|
||||
|
||||
|
||||
class UnknownSourceError(Exception):
|
||||
"""No source follows this Discord channel or its server."""
|
||||
|
||||
|
||||
# Mirrored byte-for-byte from extension/lib/platforms.js
|
||||
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
||||
# reviewers catch drift.
|
||||
@@ -55,8 +77,39 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
# A Discord URL names a server or a channel, never a creator, so the slug is
|
||||
# `<server>` or `<server>/<channel>` and the artist is chosen, not derived.
|
||||
# A trailing message id (a jump link) still names its channel. DMs (`@me`)
|
||||
# are not sources; thread links (`/threads/`) are left to the manual form.
|
||||
("discord", re.compile(
|
||||
r"^https?://(?:www\.|ptb\.|canary\.)?discord\.com/channels/"
|
||||
r"(?P<slug>\d+(?:/\d+)?)(?:/\d+)?/?(?:[?#].*)?$",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
]
|
||||
|
||||
DISCORD = "discord"
|
||||
|
||||
|
||||
def canonical_source_url(platform: str, url: str, slug: str) -> str:
|
||||
"""The URL a new source is stored under. Discord's is rebuilt from the ids
|
||||
— the form the manual Add form and the ingester use — so a jump link, a
|
||||
ptb/canary host or a trailing slash never makes a second source for the
|
||||
same channel. Every other platform keeps the URL as given."""
|
||||
if platform == DISCORD:
|
||||
return f"https://discord.com/channels/{slug}"
|
||||
return url
|
||||
|
||||
|
||||
def _discord_ids(url: str) -> tuple[str | None, str | None] | None:
|
||||
"""`(server_id, channel_id)` of a stored Discord source URL, None if it
|
||||
does not parse (a DM or thread link, or an old malformed row)."""
|
||||
from .discord_client import DiscordAPIError, parse_source_url
|
||||
try:
|
||||
return parse_source_url(url)
|
||||
except DiscordAPIError:
|
||||
return None
|
||||
|
||||
|
||||
class ExtensionService:
|
||||
def __init__(self, session: AsyncSession, crypto=None) -> None:
|
||||
@@ -65,29 +118,179 @@ class ExtensionService:
|
||||
# add-time. None → skip resolution, fall back to the handle.
|
||||
self._crypto = crypto
|
||||
|
||||
async def quick_add_source(self, url: str) -> dict:
|
||||
async def quick_add_source(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
artist_id: int | None = None,
|
||||
artist_name: str | None = None,
|
||||
use_platform_name: bool = False,
|
||||
discord_authors: list[dict] | None = None,
|
||||
) -> dict:
|
||||
"""Add `url` as a source. `artist_id` connects it to an existing
|
||||
artist, `artist_name` to that artist (created if new); with neither,
|
||||
the artist is resolved from the platform as before.
|
||||
|
||||
`use_platform_name` applies the operator's convention that the Patreon
|
||||
name is canon: a Patreon source added to an existing artist renames
|
||||
that artist to the creator's Patreon display name. Name only — the
|
||||
slug, and every path keyed off it, never moves (#130). Ignored on every
|
||||
other platform, and when the name can't be read.
|
||||
|
||||
`discord_authors` ([{id, name}], #4488) sets a Discord source's poster
|
||||
list in the same step, on a new source or one already there."""
|
||||
platform, raw_slug = self._derive(url)
|
||||
url = canonical_source_url(platform, url, raw_slug)
|
||||
renamed_from = None
|
||||
# Identity by SOURCE handle (#130): an existing (platform, url) source
|
||||
# keeps its artist on re-add — even if that artist was since renamed (its
|
||||
# frozen slug no longer matches the current name). Only a genuinely new
|
||||
# source resolves/creates an artist.
|
||||
existing = (await self.session.execute(
|
||||
select(Source).where(Source.platform == platform, Source.url == url)
|
||||
)).scalar_one_or_none()
|
||||
# frozen slug no longer matches the current name), and even when the
|
||||
# add named a different artist. Only a genuinely new source
|
||||
# resolves/creates an artist.
|
||||
existing = await self._existing_source(platform, url)
|
||||
if existing is not None:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == existing.artist_id)
|
||||
)).scalar_one()
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(existing, discord_authors)
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
# New source → name the artist properly by resolving the real display
|
||||
# name from the platform (falls back to the URL handle).
|
||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||
artist, created_artist = await self._find_or_create_artist(name)
|
||||
if artist_id is not None:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == artist_id)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
raise UnknownArtistError(f"no artist with id {artist_id}")
|
||||
created_artist = False
|
||||
if use_platform_name and platform == "patreon":
|
||||
renamed_from = await self._adopt_patreon_name(artist, raw_slug, url)
|
||||
else:
|
||||
name = (artist_name or "").strip()
|
||||
if not name:
|
||||
# Name the artist properly by resolving the real display name
|
||||
# from the platform (falls back to the URL handle).
|
||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||
artist, created_artist = await self._find_or_create_artist(name)
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
return self._shape(source, artist, created_source, created_artist)
|
||||
if platform == DISCORD and discord_authors is not None:
|
||||
await self._apply_posters(source, discord_authors)
|
||||
shaped = self._shape(source, artist, created_source, created_artist)
|
||||
if renamed_from is not None:
|
||||
shaped["renamed_from"] = renamed_from
|
||||
return shaped
|
||||
|
||||
# -- Discord poster picker (#4488) ------------------------------------------
|
||||
|
||||
def _discord_page(self, url: str) -> tuple[str, str]:
|
||||
platform, raw_slug = self._derive(url)
|
||||
if platform != DISCORD:
|
||||
raise InvalidUrlError("not a Discord page")
|
||||
server_id, _, channel_id = raw_slug.partition("/")
|
||||
if not channel_id:
|
||||
raise PosterLookupError("Open a channel to see who posts in it.")
|
||||
return server_id, channel_id
|
||||
|
||||
async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None:
|
||||
"""The source this channel's posts land on: the channel's own, else a
|
||||
whole-server source that walks it."""
|
||||
base = f"https://discord.com/channels/{server_id}"
|
||||
return (
|
||||
await self._existing_source(DISCORD, f"{base}/{channel_id}")
|
||||
or await self._existing_source(DISCORD, base)
|
||||
)
|
||||
|
||||
async def discord_posters(self, url: str) -> dict:
|
||||
"""Who posts in the channel `url` shows, newest `_POSTER_SCAN`
|
||||
messages deep, with the likely creator marked — and, when a source
|
||||
already follows it, the poster list that source has now."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .discord_client import DiscordAPIError, DiscordClient
|
||||
token = None
|
||||
if self._crypto is not None:
|
||||
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||
if not token:
|
||||
raise PosterLookupError("No Discord token is saved in FabledCurator.")
|
||||
client = DiscordClient(token, max_retries=0)
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
found = await asyncio.wait_for(
|
||||
loop.run_in_executor(None, client.recent_posters, server_id, channel_id),
|
||||
timeout=_POSTER_LOOKUP_SECONDS,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise PosterLookupError("Discord took too long to answer; try again.") from exc
|
||||
except DiscordAPIError as exc:
|
||||
raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
co = (source.config_overrides or {}) if source is not None else {}
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
return {
|
||||
**found,
|
||||
"source_id": source.id if source is not None else None,
|
||||
"selected": list(co.get(AUTHORS_KEY) or []),
|
||||
# So someone on the list who hasn't posted lately still shows by
|
||||
# name, and can be unticked.
|
||||
"selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}),
|
||||
}
|
||||
|
||||
async def set_discord_posters(self, url: str, authors: list[dict]) -> dict:
|
||||
"""Store the picked posters on the source that follows `url`'s channel."""
|
||||
server_id, channel_id = self._discord_page(url)
|
||||
source = await self._source_for_page(server_id, channel_id)
|
||||
if source is None:
|
||||
raise UnknownSourceError("No source follows this channel yet.")
|
||||
return await self._apply_posters(source, authors)
|
||||
|
||||
async def _apply_posters(self, source: Source, authors: list[dict]) -> dict:
|
||||
"""Ids are what the list keys on — a username can change on a whim,
|
||||
an id can't — and the name each was picked under rides beside it for
|
||||
showing the list to a person. An empty pick clears the list, which
|
||||
takes everyone's posts again."""
|
||||
from .discord_ingester import AUTHORS_KEY
|
||||
picked = [a for a in authors if str(a.get("id") or "").strip()]
|
||||
co = dict(source.config_overrides or {})
|
||||
if picked:
|
||||
co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked]
|
||||
co[AUTHOR_LABELS_KEY] = {
|
||||
str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked
|
||||
}
|
||||
else:
|
||||
co.pop(AUTHORS_KEY, None)
|
||||
co.pop(AUTHOR_LABELS_KEY, None)
|
||||
source.config_overrides = co or None
|
||||
await self.session.commit()
|
||||
return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])}
|
||||
|
||||
async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
|
||||
"""Rename `artist` to the Patreon display name; the old name when it
|
||||
changed, else None. Unreadable name → no rename, never the handle."""
|
||||
name = await self._platform_display_name("patreon", raw_slug, url)
|
||||
if not name or name == artist.name:
|
||||
return None
|
||||
old = artist.name
|
||||
artist.name = name
|
||||
await self.session.commit()
|
||||
return old
|
||||
|
||||
async def _existing_source(self, platform: str, url: str) -> Source | None:
|
||||
"""The source this URL already is, whichever artist owns it. Discord
|
||||
compares ids, not strings, so a row stored before canonicalisation (a
|
||||
ptb host, a trailing slash) is still found."""
|
||||
if platform != DISCORD:
|
||||
return (await self.session.execute(
|
||||
select(Source).where(Source.platform == platform, Source.url == url)
|
||||
)).scalars().first()
|
||||
want = _discord_ids(url)
|
||||
rows = (await self.session.execute(
|
||||
select(Source).where(Source.platform == DISCORD).order_by(Source.id)
|
||||
)).scalars().all()
|
||||
return next((s for s in rows if _discord_ids(s.url) == want), None)
|
||||
|
||||
@staticmethod
|
||||
def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
|
||||
@@ -117,8 +320,23 @@ class ExtensionService:
|
||||
platforms (and any failure — no credential, network error) fall back to
|
||||
the URL handle, which is already readable.
|
||||
The resolvers are sync, so they run in an executor."""
|
||||
if platform == DISCORD:
|
||||
# The server's name: what the operator knows the community as.
|
||||
server_id = raw_slug.split("/", 1)[0]
|
||||
names = await self._discord_names(server_id, None)
|
||||
return names.get("server") or f"Discord {server_id}"
|
||||
return await self._platform_display_name(platform, raw_slug, url) or raw_slug
|
||||
|
||||
async def _platform_display_name(
|
||||
self, platform: str, raw_slug: str, url: str
|
||||
) -> str | None:
|
||||
"""The creator's display name as Patreon or SubscribeStar shows it, read
|
||||
with the stored cookies; None when it can't be read (no credential, a
|
||||
network error, a slow answer, any other platform). None, not the handle,
|
||||
so a caller can tell a real name from a fallback — a rename to the
|
||||
Patreon name must never rename to a URL handle instead."""
|
||||
if self._crypto is None or platform not in ("patreon", "subscribestar"):
|
||||
return raw_slug
|
||||
return None
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
@@ -128,7 +346,7 @@ class ExtensionService:
|
||||
if platform == "patreon":
|
||||
cookies = await cred.get_cookies_path("patreon")
|
||||
from .patreon_resolver import resolve_display_name
|
||||
name = await loop.run_in_executor(
|
||||
call = loop.run_in_executor(
|
||||
None, resolve_display_name, raw_slug,
|
||||
str(cookies) if cookies else None,
|
||||
)
|
||||
@@ -136,15 +354,14 @@ class ExtensionService:
|
||||
cookies = await cred.get_cookies_path("subscribestar")
|
||||
from .subscribestar_client import SubscribeStarClient
|
||||
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||
name = await loop.run_in_executor(
|
||||
None, client.resolve_display_name, url
|
||||
)
|
||||
call = loop.run_in_executor(None, client.resolve_display_name, url)
|
||||
name = await asyncio.wait_for(call, timeout=_NAME_LOOKUP_SECONDS)
|
||||
except Exception as exc: # resolution is best-effort — never block the add
|
||||
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||
return raw_slug
|
||||
return name or raw_slug
|
||||
return None
|
||||
return (name or "").strip() or None
|
||||
|
||||
async def probe(self, url: str) -> dict:
|
||||
async def probe(self, url: str, *, names: bool = False) -> dict:
|
||||
"""Read-only resolution of a creator-page URL against the FC DB.
|
||||
Returns one of:
|
||||
- {state: 'unknown_platform'} — URL didn't match any
|
||||
@@ -160,21 +377,30 @@ class ExtensionService:
|
||||
— exact (artist, platform,
|
||||
url) Source already exists
|
||||
|
||||
Side-effect-free: two SELECTs at most.
|
||||
`names` (the Add panel asks, the chip does not) adds `display_name`:
|
||||
the creator's name as Patreon/SubscribeStar shows it, or None. It costs
|
||||
a request to the platform, so a plain page view never pays it.
|
||||
|
||||
Side-effect-free: two SELECTs at most, plus that one lookup.
|
||||
"""
|
||||
try:
|
||||
platform, raw_slug = self._derive(url)
|
||||
except (UnknownPlatformError, InvalidUrlError):
|
||||
return {"state": "unknown_platform"}
|
||||
if platform == DISCORD:
|
||||
return await self._probe_discord(raw_slug)
|
||||
|
||||
slug = slugify(raw_slug)
|
||||
result: dict = {"platform": platform, "slug": slug}
|
||||
if names:
|
||||
result["display_name"] = await self._platform_display_name(
|
||||
platform, raw_slug, url,
|
||||
)
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return {"state": "new", "platform": platform, "slug": slug}
|
||||
|
||||
artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
return {"state": "new", **result}
|
||||
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(
|
||||
@@ -184,26 +410,114 @@ class ExtensionService:
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
return {
|
||||
"state": "artist_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
}
|
||||
return {"state": "artist_match", **result, "artist": self._artist_payload(artist)}
|
||||
|
||||
return {
|
||||
"state": "source_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
"source": {
|
||||
"id": source.id,
|
||||
"artist_id": source.artist_id,
|
||||
"platform": source.platform,
|
||||
"url": source.url,
|
||||
"enabled": source.enabled,
|
||||
**result,
|
||||
"artist": self._artist_payload(artist),
|
||||
"source": self._source_payload(source),
|
||||
}
|
||||
|
||||
async def _probe_discord(self, raw_slug: str) -> dict:
|
||||
"""probe for a Discord server or channel. The states mean what they
|
||||
mean elsewhere, but the artist is never read off the URL:
|
||||
|
||||
- source_match: this channel is a source — or the whole server is
|
||||
(`covered_by_server`), which already walks every channel;
|
||||
- artist_match: another source on this server belongs to an artist,
|
||||
the one this channel most likely belongs to too (a suggestion the
|
||||
Add panel preselects, not a decision);
|
||||
- new: nothing on this server yet.
|
||||
|
||||
`discord` carries the ids, both canonical URLs and the display names,
|
||||
read with the stored token; a name that can't be read is None."""
|
||||
server_id, _, channel_id = raw_slug.partition("/")
|
||||
channel_id = channel_id or None
|
||||
rows = (await self.session.execute(
|
||||
select(Source, Artist)
|
||||
.join(Artist, Artist.id == Source.artist_id)
|
||||
.where(Source.platform == DISCORD)
|
||||
.order_by(Source.id)
|
||||
)).all()
|
||||
exact = server_whole = on_server = None
|
||||
for source, artist in rows:
|
||||
ids = _discord_ids(source.url)
|
||||
if ids is None or ids[0] != server_id:
|
||||
continue
|
||||
if ids[1] == channel_id and exact is None:
|
||||
exact = (source, artist)
|
||||
elif ids[1] is None and server_whole is None:
|
||||
server_whole = (source, artist)
|
||||
if on_server is None:
|
||||
on_server = (source, artist)
|
||||
|
||||
names = await self._discord_names(server_id, channel_id)
|
||||
base = f"https://discord.com/channels/{server_id}"
|
||||
result: dict = {
|
||||
"platform": DISCORD,
|
||||
"slug": raw_slug,
|
||||
"discord": {
|
||||
"server_id": server_id,
|
||||
"channel_id": channel_id,
|
||||
"server_name": names.get("server"),
|
||||
"channel_name": names.get("channel"),
|
||||
"server_url": base,
|
||||
"channel_url": f"{base}/{channel_id}" if channel_id else None,
|
||||
},
|
||||
}
|
||||
hit = exact or server_whole
|
||||
if hit is not None:
|
||||
source, artist = hit
|
||||
result.update(
|
||||
state="source_match",
|
||||
artist=self._artist_payload(artist),
|
||||
source=self._source_payload(source),
|
||||
covered_by_server=exact is None,
|
||||
)
|
||||
elif on_server is not None:
|
||||
result.update(state="artist_match", artist=self._artist_payload(on_server[1]))
|
||||
else:
|
||||
result["state"] = "new"
|
||||
return result
|
||||
|
||||
async def _discord_names(self, server_id: str | None, channel_id: str | None) -> dict:
|
||||
"""Server/channel display names via the stored Discord token. Never
|
||||
raises and never waits out a rate limit: it runs while the operator
|
||||
looks at a page, so a slow or missing answer just means no names."""
|
||||
if self._crypto is None:
|
||||
return {}
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .discord_client import DiscordClient
|
||||
try:
|
||||
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||
if not token:
|
||||
return {}
|
||||
client = DiscordClient(token, max_retries=0)
|
||||
loop = asyncio.get_running_loop()
|
||||
return await asyncio.wait_for(
|
||||
loop.run_in_executor(None, client.describe, server_id, channel_id),
|
||||
timeout=_NAME_LOOKUP_SECONDS,
|
||||
)
|
||||
except Exception as exc: # names are decoration — never fail the call
|
||||
log.info("Discord name lookup failed: %s", exc)
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _artist_payload(artist) -> dict:
|
||||
return {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
|
||||
@staticmethod
|
||||
def _source_payload(source) -> dict:
|
||||
return {
|
||||
"id": source.id,
|
||||
"artist_id": source.artist_id,
|
||||
"platform": source.platform,
|
||||
"url": source.url,
|
||||
"enabled": source.enabled,
|
||||
}
|
||||
|
||||
def _derive(self, url: str) -> tuple[str, str]:
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
|
||||
@@ -95,48 +95,6 @@ BACKFILL_CHUNK_SECONDS = 600
|
||||
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
||||
|
||||
|
||||
# --- Discord naming ---------------------------------------------------------
|
||||
#
|
||||
# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
|
||||
# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
|
||||
# for an attachment: `channel` is a plain STRING (the channel's name), the
|
||||
# message is `message_id`, the attachment's position in it is `num`, and there
|
||||
# is NO `id` key at all.
|
||||
#
|
||||
# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
|
||||
# "None", so every Discord download since the platform was added landed in a
|
||||
# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
|
||||
# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
|
||||
# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
|
||||
# got a Post or a post date, and (b) collides: every `image.png` in a channel
|
||||
# overwrote the same `image.json`, so the one sidecar that survived described
|
||||
# whichever message happened to be written last.
|
||||
#
|
||||
# The fix names the sidecar EXACTLY like the media minus its extension, so
|
||||
# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
|
||||
# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
|
||||
# these patterns against a sanitized copy of the real sidecar, so a key that
|
||||
# does not exist fails CI instead of silently becoming "None".
|
||||
DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
|
||||
DISCORD_DIRECTORY = ["{channel}"]
|
||||
|
||||
|
||||
def sidecar_name_for(media_pattern: str) -> str | None:
|
||||
"""The metadata filename pattern that names a sidecar exactly like its media.
|
||||
|
||||
Returns None for a pattern that does not end in `.{extension}`, since then
|
||||
there is no media stem to mirror and the caller must fall back.
|
||||
"""
|
||||
suffix = ".{extension}"
|
||||
if not media_pattern.endswith(suffix):
|
||||
return None
|
||||
return media_pattern[: -len(suffix)] + ".json"
|
||||
|
||||
|
||||
def metadata_postprocessor(filename: str) -> dict:
|
||||
return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
|
||||
|
||||
|
||||
def archive_path(images_root: Path) -> Path:
|
||||
"""gallery-dl's download archive: the record of what it has already fetched.
|
||||
|
||||
@@ -425,27 +383,16 @@ class GalleryDLService:
|
||||
# (services/patreon_ingester.py), not gallery-dl.
|
||||
PLATFORM_DEFAULTS = {
|
||||
# subscribestar removed — native-ingester platform now (#71); pixiv
|
||||
# removed likewise (#129); deviantart removed at #3069 as a dropped
|
||||
# platform, not a migrated one. The remaining entries are the
|
||||
# gallery-dl platforms not yet migrated.
|
||||
# removed likewise (#129); discord likewise (milestone 428, whose
|
||||
# downloader keeps this config's on-disk naming); deviantart removed at
|
||||
# #3069 as a dropped platform, not a migrated one. HentaiFoundry is the
|
||||
# one platform left here, by the operator's choice not to migrate it.
|
||||
"hentaifoundry": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
||||
"include": "all",
|
||||
},
|
||||
"discord": {
|
||||
"content_types": ["all"],
|
||||
"directory": DISCORD_DIRECTORY,
|
||||
"filename": DISCORD_FILENAME,
|
||||
# Overrides the global `{filename}.json` sidecar for this extractor
|
||||
# only — see the Discord naming note above.
|
||||
"postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
|
||||
"embeds": "all",
|
||||
"stickers": True,
|
||||
"reactions": False,
|
||||
"threads": True,
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(
|
||||
@@ -560,17 +507,6 @@ class GalleryDLService:
|
||||
if source_config.filename_pattern:
|
||||
platform_section["filename"] = source_config.filename_pattern
|
||||
|
||||
# A platform that names its sidecar after its media must keep doing so
|
||||
# under a per-source filename override, or the pairing breaks exactly the
|
||||
# way Discord's did. No metadata wanted means no platform postprocessor
|
||||
# either — the global list was already dropped above.
|
||||
if "postprocessors" in platform_section:
|
||||
mirrored = sidecar_name_for(platform_section.get("filename") or "")
|
||||
if not source_config.save_metadata or mirrored is None:
|
||||
platform_section.pop("postprocessors")
|
||||
else:
|
||||
platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
|
||||
|
||||
platform_section["metadata"] = source_config.save_metadata
|
||||
|
||||
return config
|
||||
@@ -818,9 +754,6 @@ class GalleryDLService:
|
||||
|
||||
if cookies_path:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})
|
||||
config["extractor"]["discord"]["token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
@@ -1004,8 +937,6 @@ class GalleryDLService:
|
||||
config = self._build_config_for_source(platform, source_config, artist_slug)
|
||||
if cookies_path:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})["token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
|
||||
@@ -49,10 +49,8 @@ from .audits import single_color
|
||||
from .link_extract import extract_external_links
|
||||
from .thumbnailer import Thumbnailer
|
||||
from .wip_title import (
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
WIP_TITLE_SOURCE,
|
||||
apply_wip_image_tags,
|
||||
matches_soft_wip_title,
|
||||
matches_wip_title,
|
||||
resolve_wip_tag_id,
|
||||
)
|
||||
@@ -428,11 +426,19 @@ class Importer:
|
||||
return self._upsert_artist(name) if name else None
|
||||
|
||||
def _post_for_sidecar(
|
||||
self, source: Path, artist: Artist | None
|
||||
self, source: Path, artist: Artist | None,
|
||||
*, source_row: Source | None = None,
|
||||
) -> Post | None:
|
||||
"""If a sidecar sits next to `source`, ensure its Source+Post
|
||||
exist (idempotent) and return the Post — so attachments can link
|
||||
to the same Post the per-member _apply_sidecar will reuse."""
|
||||
to the same Post the per-member _apply_sidecar will reuse.
|
||||
|
||||
`source_row` is the subscription being downloaded, when there is one,
|
||||
and wins over the (artist, platform) lookup — as it does in
|
||||
`upsert_post_record`. The lookup takes the artist's FIRST source on the
|
||||
platform, which is right only while an artist has one: a Discord artist
|
||||
has one per channel, and every non-image file from a later channel was
|
||||
filed under the first as an undated second post (#4435)."""
|
||||
sc = find_sidecar(source)
|
||||
if sc is None or artist is None:
|
||||
return None
|
||||
@@ -444,10 +450,13 @@ class Importer:
|
||||
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
||||
return None
|
||||
sd = parse_sidecar(data)
|
||||
platform = sd.platform or "unknown"
|
||||
src = self._lookup_source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
)
|
||||
if source_row is not None:
|
||||
src = source_row
|
||||
else:
|
||||
platform = sd.platform or "unknown"
|
||||
src = self._lookup_source_for_sidecar(
|
||||
artist_id=artist.id, platform=platform,
|
||||
)
|
||||
epid = sd.external_post_id or sc.stem
|
||||
return self._find_or_create_post(
|
||||
source_id=src.id if src else None,
|
||||
@@ -545,7 +554,7 @@ class Importer:
|
||||
# nothing silently vanishes, matching extract_archive's
|
||||
# fail-soft contract.
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use)
|
||||
post = self._post_for_sidecar(source, artist_use, source_row=source_row)
|
||||
self._capture_attachment(
|
||||
source, post=post, artist=artist_use, resolved=True,
|
||||
)
|
||||
@@ -554,7 +563,7 @@ class Importer:
|
||||
return ImportResult(status="attached", error=reason)
|
||||
|
||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||
post = self._post_for_sidecar(source, artist_use)
|
||||
post = self._post_for_sidecar(source, artist_use, source_row=source_row)
|
||||
member_ids: list[int] = []
|
||||
# Every member image touched (new + superseded + deduped), so the
|
||||
# from_attachment_id stamp below covers files that already existed in the
|
||||
@@ -1040,9 +1049,7 @@ class Importer:
|
||||
removal sticks. The existing catalogue is covered separately by the
|
||||
operator-triggered backfill sweep. Gated by the settings toggle, and
|
||||
best-effort: any failure is logged, never allowed to fail the import."""
|
||||
hard_on = self.settings.wip_title_tagging_enabled
|
||||
soft_on = self.settings.wip_soft_title_tagging_enabled
|
||||
if not (hard_on or soft_on):
|
||||
if not self.settings.wip_title_tagging_enabled:
|
||||
return
|
||||
if record.primary_post_id is None:
|
||||
return
|
||||
@@ -1050,21 +1057,14 @@ class Importer:
|
||||
title = self.session.execute(
|
||||
select(Post.post_title).where(Post.id == record.primary_post_id)
|
||||
).scalar_one_or_none()
|
||||
# HARD tier ("WIP"/"work in progress") wins — higher precision, and it
|
||||
# trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
|
||||
# that never trains (source wip_title_soft).
|
||||
if hard_on and matches_wip_title(title):
|
||||
source = WIP_TITLE_SOURCE
|
||||
elif soft_on and matches_soft_wip_title(title):
|
||||
source = WIP_TITLE_SOFT_SOURCE
|
||||
else:
|
||||
if not matches_wip_title(title):
|
||||
return
|
||||
if self._wip_tag_id is _UNSET:
|
||||
self._wip_tag_id = resolve_wip_tag_id(self.session)
|
||||
if self._wip_tag_id is None:
|
||||
return
|
||||
apply_wip_image_tags(
|
||||
self.session, [record.id], self._wip_tag_id, source=source
|
||||
self.session, [record.id], self._wip_tag_id, source=WIP_TITLE_SOURCE
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 — a tag must never fail an import
|
||||
log.warning(
|
||||
@@ -1141,9 +1141,51 @@ class Importer:
|
||||
if post.artist_id is None:
|
||||
post.artist_id = artist.id
|
||||
self._apply_post_fields(post, sd)
|
||||
self._redate_post_images(post)
|
||||
self.session.commit()
|
||||
return True
|
||||
|
||||
def _redate_post_images(self, post: Post) -> None:
|
||||
"""Carry a post's date onto the images already linked to it (#4431).
|
||||
|
||||
The native ingesters import a post's media BEFORE its record: the
|
||||
per-media sidecar holds only the image identity (post-first, #856), and
|
||||
the date arrives with `_post.json`. So `_attach_provenance` links each
|
||||
image to a post that has no date yet, and the image keeps its download
|
||||
time. This runs when the record lands, and applies the same two rules
|
||||
`_attach_provenance` applies: `effective_date` is the PRIMARY post's
|
||||
date, and `earliest_post_date` is the earliest date across every post
|
||||
the image is linked to. Only rows that differ are written."""
|
||||
if post.post_date is None:
|
||||
return
|
||||
self.session.flush()
|
||||
self.session.execute(
|
||||
update(ImageRecord)
|
||||
.where(ImageRecord.primary_post_id == post.id)
|
||||
.where(ImageRecord.effective_date.is_distinct_from(post.post_date))
|
||||
.values(effective_date=post.post_date)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
linked = select(ImageProvenance.image_record_id).where(
|
||||
ImageProvenance.post_id == post.id
|
||||
)
|
||||
earliest = (
|
||||
select(func.min(Post.post_date))
|
||||
.select_from(ImageProvenance)
|
||||
.join(Post, Post.id == ImageProvenance.post_id)
|
||||
.where(ImageProvenance.image_record_id == ImageRecord.id)
|
||||
.where(Post.post_date.is_not(None))
|
||||
.correlate(ImageRecord)
|
||||
.scalar_subquery()
|
||||
)
|
||||
self.session.execute(
|
||||
update(ImageRecord)
|
||||
.where(ImageRecord.id.in_(linked))
|
||||
.where(ImageRecord.earliest_post_date.is_distinct_from(earliest))
|
||||
.values(earliest_post_date=earliest)
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
|
||||
def attach_in_place(
|
||||
self,
|
||||
path: Path,
|
||||
@@ -1187,7 +1229,10 @@ class Importer:
|
||||
path, artist=artist, source_row=source,
|
||||
)
|
||||
if not is_supported(path):
|
||||
post = self._post_for_sidecar(path, artist) if artist else None
|
||||
post = (
|
||||
self._post_for_sidecar(path, artist, source_row=source)
|
||||
if artist else None
|
||||
)
|
||||
return self._capture_attachment(
|
||||
path, post=post, artist=artist, resolved=True,
|
||||
)
|
||||
|
||||
@@ -106,6 +106,22 @@ _LIVE_PROGRESS_INTERVAL = 5.0
|
||||
# recapture (the operator's schema-test flow) reaches the sample.
|
||||
_CANARY_MIN_SAMPLE = 30
|
||||
|
||||
# The walk's time budget covers only the walk, but phase 3 runs in the SAME
|
||||
# Celery task, under the same soft limit (tasks/download.py: 1350s). A walk that
|
||||
# finds a lot of work for phase 3 must stop early and leave it to the next chunk,
|
||||
# or phase 3 is killed mid-import: TamadaHeijun's recapture, 2026-09-24, walked
|
||||
# for ~2 min and then spent 20 min importing 431 orphans and relinking ~3000
|
||||
# on-disk files, and died at the soft limit.
|
||||
#
|
||||
# So the walk also stops when its elapsed time PLUS phase 3's estimated cost
|
||||
# would pass CHUNK_TOTAL_SECONDS. Costs measured on the live instance: 431
|
||||
# imports took 976s (~2.3s each: hash, pHash, sidecar, provenance); a relink is
|
||||
# a sha256 over NFS, 0.15s for the 8.8 MB average file, plus a lookup.
|
||||
# test_download_source_task pins CHUNK_TOTAL_SECONDS under the soft limit.
|
||||
CHUNK_TOTAL_SECONDS = 1200.0
|
||||
PHASE3_IMPORT_SECONDS = 2.5
|
||||
PHASE3_RELINK_SECONDS = 0.25
|
||||
|
||||
|
||||
def _parse_published(raw: object) -> datetime | None:
|
||||
"""An ISO-8601 post date from either native client, as aware UTC.
|
||||
@@ -248,6 +264,14 @@ class Ingester:
|
||||
# operator-driven "re-read every body" pass; a horizon there would be a
|
||||
# third overlapping answer to a question that has two.
|
||||
post_meta = getattr(self.client, "post_meta", None)
|
||||
# #4413: optional client seam for a source that is several feeds walked
|
||||
# one after another (a Discord server: every channel and thread). The
|
||||
# tick early-out means "this feed has nothing new", and without the
|
||||
# seam it ends the WHOLE walk — so the first quiet channel would hide
|
||||
# every channel after it. With it, the early-out asks the client to
|
||||
# move on and the walk continues. Absent → the early-out ends the walk,
|
||||
# exactly as before (Patreon and SubscribeStar are one feed each).
|
||||
skip_feed = getattr(self.client, "skip_feed", None)
|
||||
horizon: datetime | None = None
|
||||
if mode == "tick" and revisit_days > 0 and post_meta is not None:
|
||||
horizon = datetime.now(UTC) - timedelta(days=revisit_days)
|
||||
@@ -265,6 +289,11 @@ class Ingester:
|
||||
# Media handed to phase 3 for import. Marked seen by phase 3 once the
|
||||
# import has run (`mark_seen_after_import`), not here — see there.
|
||||
fetched: list[tuple[str, str]] = []
|
||||
# Post-record keys written this walk. Marked with the media, after
|
||||
# phase 3 has upserted the records — marking them at write time left a
|
||||
# walk killed before phase 3 with posts the ledger calls recorded that
|
||||
# the database never dated, and no later tick walks back to them (#4436).
|
||||
recorded: list[tuple[str, str]] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
@@ -296,6 +325,7 @@ class Ingester:
|
||||
reached_bottom = False
|
||||
budget_hit = False
|
||||
early_out = False
|
||||
feeds_caught_up = 0 # #4413: feeds a tick left early via skip_feed
|
||||
stopped = False # plan #708 B4: operator hit Stop mid-walk
|
||||
cancel_armed = False # latched once we observe a live "running" state
|
||||
|
||||
@@ -317,7 +347,9 @@ class Ingester:
|
||||
written_paths=written,
|
||||
post_record_paths=list(post_records),
|
||||
relink_source_paths=list(relink),
|
||||
mark_seen_after_import=lambda: self._mark_seen(source_id, fetched),
|
||||
mark_seen_after_import=lambda: self._mark_seen(
|
||||
source_id, fetched + recorded,
|
||||
),
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
return_code=return_code,
|
||||
@@ -393,7 +425,17 @@ class Ingester:
|
||||
|
||||
# Time-box check at the post boundary (coarse, like a gallery-dl
|
||||
# chunk). Backfill/recovery resume from emitted_cursor next chunk.
|
||||
if time.monotonic() - start >= time_budget_seconds:
|
||||
# The second half is phase 3's share of the task — see
|
||||
# CHUNK_TOTAL_SECONDS. A mid-page stop resumes the same page.
|
||||
elapsed = time.monotonic() - start
|
||||
phase3 = (
|
||||
len(written) * PHASE3_IMPORT_SECONDS
|
||||
+ len(relink) * PHASE3_RELINK_SECONDS
|
||||
)
|
||||
if (
|
||||
elapsed >= time_budget_seconds
|
||||
or elapsed + phase3 >= CHUNK_TOTAL_SECONDS
|
||||
):
|
||||
budget_hit = True
|
||||
break
|
||||
|
||||
@@ -476,7 +518,7 @@ class Ingester:
|
||||
posts_with_body += 1
|
||||
if rec.path is not None:
|
||||
post_records.append(str(rec.path))
|
||||
self._mark_seen(source_id, [(pkey, ppid)])
|
||||
recorded.append((pkey, ppid))
|
||||
# Per-post handling line in the run stdout (the existing
|
||||
# "Raw stdout" panel) — the downloader already read the
|
||||
# post; we only format its outcome here. post_type beside
|
||||
@@ -648,9 +690,15 @@ class Ingester:
|
||||
})
|
||||
|
||||
if early_out:
|
||||
break
|
||||
if skip_feed is None:
|
||||
break
|
||||
skip_feed()
|
||||
feeds_caught_up += 1
|
||||
early_out = False
|
||||
consecutive_seen = 0
|
||||
else:
|
||||
reached_bottom = True
|
||||
# A walk that left feeds early did not read to their ends.
|
||||
reached_bottom = not feeds_caught_up
|
||||
except self._error_base as exc:
|
||||
# The platform's client-error base — _failure_result (adapter)
|
||||
# maps it to a typed error.
|
||||
@@ -698,6 +746,7 @@ class Ingester:
|
||||
f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
|
||||
if revisited else ""
|
||||
)
|
||||
+ (f", {feeds_caught_up} feed(s) caught up" if feeds_caught_up else "")
|
||||
+ (", reached end" if reached_bottom else "")
|
||||
+ (", time-boxed" if budget_hit else "")
|
||||
)
|
||||
@@ -711,7 +760,13 @@ class Ingester:
|
||||
# next chunk resumes from the emitted cursor. No progress → TIMEOUT,
|
||||
# which feeds download_service's backfill stall-guard. rc<0 mirrors
|
||||
# subprocess TimeoutExpired so completion detection stays false.
|
||||
made_progress = downloaded > 0 or emitted_cursor != resume_cursor
|
||||
# Work handed to phase 3 is progress too: a recapture chunk that
|
||||
# stopped for its imports downloaded nothing, and may not have left
|
||||
# its first page.
|
||||
made_progress = (
|
||||
downloaded > 0 or bool(written) or bool(relink)
|
||||
or emitted_cursor != resume_cursor
|
||||
)
|
||||
if made_progress:
|
||||
return _result(
|
||||
success=False, return_code=-1,
|
||||
|
||||
@@ -97,8 +97,8 @@ def _sigmoid(z, np):
|
||||
|
||||
def _conflict_scores(Xn, Wc, bc, np):
|
||||
"""The presentation conflict signal (#141): per row, the MAX content-head
|
||||
probability and WHICH head produced it. Shared by the system-tag sweep's guard-2
|
||||
and the soft-wip audit — both ask "does this ALSO look like real content?"."""
|
||||
probability and WHICH head produced it — the system-tag sweep's guard-2 asks
|
||||
"does this ALSO look like real content?"."""
|
||||
cprobs = _sigmoid(Xn @ Wc.T + bc, np)
|
||||
return cprobs.max(axis=1), cprobs.argmax(axis=1)
|
||||
|
||||
@@ -106,10 +106,9 @@ def _conflict_scores(Xn, Wc, bc, np):
|
||||
def _insert_presentation_review(
|
||||
session, *, image_record_id, tag_id, conflict_tag_id, conflict_score, mode,
|
||||
):
|
||||
"""Single-source the ring-loud PresentationReview row shape so the two writers
|
||||
(system-tag sweep guard-2 + soft-wip audit) can't drift on columns or `mode` —
|
||||
they share the (image_record_id, tag_id) composite PK, so a divergent `mode`
|
||||
would be a silent first-writer-wins bug."""
|
||||
"""Single-source the ring-loud PresentationReview row shape, so every writer of
|
||||
the (image_record_id, tag_id) composite PK agrees on columns and `mode` — a
|
||||
divergent `mode` would be a silent first-writer-wins bug."""
|
||||
session.execute(
|
||||
pg_insert(PresentationReview)
|
||||
.values(
|
||||
@@ -963,68 +962,6 @@ def system_tag_auto_apply_sweep(
|
||||
}
|
||||
|
||||
|
||||
def soft_wip_conflict_audit(session: Session, dry_run: bool = False) -> dict:
|
||||
"""Ring-loud audit for the SOFT WIP-title cohort (#1474). Images auto-tagged
|
||||
`wip` from a low-precision sketch/doodle title (source='wip_title_soft') that ALSO
|
||||
score >= the process conflict threshold on a content head are probably FINISHED
|
||||
art mis-tagged as process — flag them (PresentationReview, mode='process') so the
|
||||
review strip surfaces them ("also looks like <X>", Keep tag / Remove tag). Does
|
||||
NOT remove the tag; the operator decides. No-op when there are no content heads.
|
||||
numpy-only. Returns {n_scanned, n_flagged}."""
|
||||
import numpy as np
|
||||
|
||||
from ..wip_title import WIP_TITLE_SOFT_SOURCE, resolve_wip_tag_id
|
||||
|
||||
settings = _settings(session)
|
||||
ver = settings.embedder_model_version
|
||||
conflict_thr = float(settings.process_conflict_threshold)
|
||||
conf = _conflict_heads(session, ver)
|
||||
wip_id = resolve_wip_tag_id(session)
|
||||
if not conf or wip_id is None:
|
||||
return {"n_scanned": 0, "n_flagged": 0}
|
||||
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
|
||||
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
|
||||
conf_tag_ids = [r.tag_id for r in conf]
|
||||
|
||||
soft_ids = [iid for (iid,) in session.execute(
|
||||
select(image_tag.c.image_record_id)
|
||||
.where(image_tag.c.tag_id == wip_id)
|
||||
.where(image_tag.c.source == WIP_TITLE_SOFT_SOURCE)
|
||||
)]
|
||||
# Skip images already flagged for this tag (idempotent re-runs).
|
||||
flagged = {iid for (iid,) in session.execute(
|
||||
select(PresentationReview.image_record_id)
|
||||
.where(PresentationReview.tag_id == wip_id)
|
||||
)}
|
||||
soft_ids = [i for i in soft_ids if i not in flagged]
|
||||
|
||||
n_flagged = 0
|
||||
scanned = 0
|
||||
for start in range(0, len(soft_ids), _AUTO_APPLY_CHUNK):
|
||||
chunk = soft_ids[start:start + _AUTO_APPLY_CHUNK]
|
||||
emb = _load_embeddings(session, chunk)
|
||||
cids = [i for i in chunk if i in emb]
|
||||
if not cids:
|
||||
continue
|
||||
scanned += len(cids)
|
||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
||||
max_c, arg_c = _conflict_scores(Xn, Wc, bc, np)
|
||||
for k in range(len(cids)):
|
||||
if float(max_c[k]) >= conflict_thr:
|
||||
n_flagged += 1
|
||||
if not dry_run:
|
||||
_insert_presentation_review(
|
||||
session,
|
||||
image_record_id=cids[k], tag_id=wip_id,
|
||||
conflict_tag_id=conf_tag_ids[int(arg_c[k])],
|
||||
conflict_score=float(max_c[k]),
|
||||
mode="process",
|
||||
)
|
||||
if not dry_run:
|
||||
session.commit()
|
||||
return {"n_scanned": scanned, "n_flagged": n_flagged}
|
||||
|
||||
|
||||
def retract_auto_applied_heads(session: Session) -> int:
|
||||
"""Soft auto-apply (milestone 139): re-score every standing source='head_auto'
|
||||
tag against its CURRENT head and REMOVE the ones now BELOW the head's
|
||||
|
||||
@@ -32,11 +32,8 @@ from ...models.tag import image_tag
|
||||
# `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
|
||||
# ALSO provisional — the head must learn only from title (`wip_title`) + manual
|
||||
# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
|
||||
# `wip_title_soft` (#1474): the soft title tier (sketch/doodle) is LOW-precision, so
|
||||
# it's provisional too — a finished piece titled "sketch" must not train the wip head.
|
||||
_AUTO_SOURCES = (
|
||||
"head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
|
||||
"wip_title_soft",
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -23,8 +23,10 @@ log = logging.getLogger(__name__)
|
||||
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
||||
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
||||
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
||||
# Add a platform here to cap it to a single concurrent walk.
|
||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
# Add a platform here to cap it to a single concurrent walk. Discord most of
|
||||
# all: every source walks on the operator's ONE user token, and parallel walks
|
||||
# on a user account are both how its rate limit trips and what gets it flagged.
|
||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||
|
||||
_LOCK_PREFIX = "fc:download_lock:"
|
||||
|
||||
@@ -58,3 +60,22 @@ def platform_lock(platform: str, *, ttl_seconds: int):
|
||||
except redis.RedisError as exc: # pragma: no cover - broker outage
|
||||
log.warning("platform_lock unavailable for %s: %s", platform, exc)
|
||||
return None
|
||||
|
||||
|
||||
def release_all_platform_locks() -> int:
|
||||
"""Drop every serialized platform's lock. Returns how many were held.
|
||||
|
||||
Only for the download lane's boot (#4433). A worker that restarts mid-walk
|
||||
is SIGKILLed past its stop grace, so its `finally` never releases the lock,
|
||||
and the TTL keeps every other source on that platform bouncing for up to
|
||||
27 minutes after the new worker is ready. At boot no walk of ours can be
|
||||
running, so a held lock names a dead one. Assumes one download consumer —
|
||||
the only shape FC deploys; a second replica booting would free a live
|
||||
walk's lock (not corrupt it: the walk runs on, a second walk may overlap it).
|
||||
"""
|
||||
try:
|
||||
client = _redis()
|
||||
return int(client.delete(*(f"{_LOCK_PREFIX}{p}" for p in SERIALIZED_PLATFORMS)))
|
||||
except redis.RedisError as exc: # pragma: no cover - broker outage
|
||||
log.warning("could not release platform locks at boot: %s", exc)
|
||||
return 0
|
||||
|
||||
@@ -479,6 +479,10 @@ class PostFeedService:
|
||||
# the UI uses this to explain why a post it linked to is not in the
|
||||
# stream.
|
||||
"absorbed_by_post_id": post.absorbed_by_post_id,
|
||||
# #4481. The Discord channel a message was posted in: both the native
|
||||
# post record and gallery-dl's sidecars carry it as `channel`. None
|
||||
# on every other platform, and on a Discord post that never said.
|
||||
"channel": _discord_channel(post, source),
|
||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||
"source": (
|
||||
{"id": source.id, "platform": source.platform}
|
||||
@@ -488,3 +492,13 @@ class PostFeedService:
|
||||
"thumbnails_more": thumbs_entry["more"],
|
||||
"attachments": atts_map.get(post.id, []),
|
||||
}
|
||||
|
||||
|
||||
def _discord_channel(post: Post, source: Source | None) -> str | None:
|
||||
if source is None or source.platform != "discord":
|
||||
return None
|
||||
raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {}
|
||||
channel = raw.get("channel")
|
||||
if not isinstance(channel, str):
|
||||
return None
|
||||
return channel.strip() or None
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Date the posts whose record reached the disk but never the database (#4436).
|
||||
|
||||
A native walk writes each post's record (`_post.json`, or Discord's
|
||||
`<day>_<message id>_post.json`) as it goes, and phase 3 upserts those records
|
||||
after the walk — which is when the post, and through it its images, get their
|
||||
date. Until #4436 the walk marked a record's post key seen at write time, so a
|
||||
walk killed before phase 3 (a restart, a stall) left the post undated with the
|
||||
ledger saying it was done, and no later tick walked back that far to fix it.
|
||||
|
||||
The record files are still on disk. This finds each undated native post's
|
||||
record under its artist's folder and upserts it with the post's OWN source —
|
||||
never the (artist, platform) lookup, which picks the artist's first source and
|
||||
would misfile a Discord channel's post (#4435). Idempotent: a post that is
|
||||
already dated is never looked at, and upserting a record twice changes nothing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..models import Artist, Post, Source
|
||||
from ..utils.sidecar import parse_sidecar
|
||||
from .download_backends import NATIVE_INGESTER_PLATFORMS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _is_record(path: Path) -> bool:
|
||||
return path.name == "_post.json" or path.name.endswith("_post.json")
|
||||
|
||||
|
||||
def _record_id(path: Path) -> str | None:
|
||||
try:
|
||||
data = json.loads(path.read_text("utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return parse_sidecar(data).external_post_id
|
||||
|
||||
|
||||
def date_posts_from_records(session: Session, importer, images_root: Path) -> dict:
|
||||
"""Upsert the on-disk record of every undated native post. Returns counts."""
|
||||
rows = session.execute(
|
||||
select(Post.external_post_id, Post.source_id, Post.artist_id, Source.platform)
|
||||
.join(Source, Source.id == Post.source_id)
|
||||
.where(
|
||||
Post.post_date.is_(None),
|
||||
Post.synthesized_by.is_(None),
|
||||
Source.platform.in_(NATIVE_INGESTER_PLATFORMS),
|
||||
)
|
||||
).all()
|
||||
wanted: dict[tuple[int, str], dict[str, int]] = defaultdict(dict)
|
||||
for epid, source_id, artist_id, platform in rows:
|
||||
wanted[(artist_id, platform)][epid] = source_id
|
||||
|
||||
summary = {"undated": len(rows), "dated": 0, "no_record": 0}
|
||||
for (artist_id, platform), by_epid in wanted.items():
|
||||
artist = session.get(Artist, artist_id)
|
||||
root = Path(images_root) / artist.slug / platform if artist else None
|
||||
if root is None or not root.is_dir():
|
||||
summary["no_record"] += len(by_epid)
|
||||
continue
|
||||
found = 0
|
||||
for record in root.rglob("*post.json"):
|
||||
if not _is_record(record):
|
||||
continue
|
||||
epid = _record_id(record)
|
||||
source_id = by_epid.pop(epid, None) if epid else None
|
||||
if source_id is None:
|
||||
continue
|
||||
source = session.get(Source, source_id)
|
||||
if importer.upsert_post_record(record, artist=artist, source=source):
|
||||
found += 1
|
||||
if not by_epid:
|
||||
break
|
||||
summary["dated"] += found
|
||||
summary["no_record"] += len(by_epid)
|
||||
if summary["undated"]:
|
||||
log.info("date_posts_from_records: %s", summary)
|
||||
return summary
|
||||
@@ -115,6 +115,32 @@ async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime
|
||||
return active
|
||||
|
||||
|
||||
def backfill_ready(source: Source) -> bool:
|
||||
"""A deep walk the operator started, with budget left and no failure
|
||||
backing it off — due NOW rather than at its next scheduled check.
|
||||
|
||||
A backfill runs one time-boxed chunk per download (plan #693), and nothing
|
||||
queued the next chunk: each waited for the source's regular interval. At
|
||||
the 8-hour default a freshly armed backfill sat untouched until the next
|
||||
check (the operator armed one on 2026-09-25 and saw nothing happen) and a
|
||||
five-chunk walk took most of two days. The tick's in-flight guard keeps
|
||||
one chunk at a time per source and the platform lock one walk per
|
||||
platform, so "due every tick" means "next chunk as soon as the last one
|
||||
ends".
|
||||
|
||||
The failure gate is what keeps a broken source from retrying every
|
||||
minute: any failed chunk raises `consecutive_failures`, which drops the
|
||||
source back onto its backed-off interval. A chunk that fails to progress
|
||||
twice marks the walk stalled (download_service), which ends it here too.
|
||||
"""
|
||||
co = source.config_overrides or {}
|
||||
return (
|
||||
co.get("_backfill_state") == "running"
|
||||
and (source.backfill_runs_remaining or 0) > 0
|
||||
and not (source.consecutive_failures or 0)
|
||||
)
|
||||
|
||||
|
||||
async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
"""Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
|
||||
|
||||
@@ -123,6 +149,9 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
cooldown is the preventive half of the burst-prevention pair (per-source
|
||||
consecutive_failures backoff handles the offending source itself).
|
||||
|
||||
A running backfill (`backfill_ready`) is due on every tick, and whether
|
||||
or not its artist is on auto-check — the operator started it by hand.
|
||||
|
||||
Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked
|
||||
sources go first, then the longest-since-checked, so the most overdue
|
||||
sources hit Celery's FIFO download queue first. Anti-starvation: if
|
||||
@@ -135,7 +164,6 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
.options(selectinload(Source.artist))
|
||||
.join(Artist, Source.artist_id == Artist.id)
|
||||
.where(Source.enabled.is_(True))
|
||||
.where(Artist.auto_check.is_(True))
|
||||
.order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
|
||||
)).scalars().all()
|
||||
|
||||
@@ -147,6 +175,11 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
for s in rows:
|
||||
if s.platform in cooldowns:
|
||||
continue
|
||||
if backfill_ready(s):
|
||||
due.append(s)
|
||||
continue
|
||||
if not s.artist.auto_check:
|
||||
continue
|
||||
interval = compute_effective_interval(s, s.artist, settings)
|
||||
if s.last_checked_at is None:
|
||||
due.append(s)
|
||||
@@ -163,6 +196,8 @@ def compute_next_check_at(
|
||||
"""Return the projected datetime of the next check, or None if never checked."""
|
||||
if source.last_checked_at is None:
|
||||
return None
|
||||
if backfill_ready(source):
|
||||
return datetime.now(UTC)
|
||||
interval = compute_effective_interval(source, artist, settings)
|
||||
return source.last_checked_at + timedelta(seconds=interval)
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from ..models import (
|
||||
Source,
|
||||
)
|
||||
from .db_helpers import failing_sources_clause
|
||||
from .download_backends import uses_native_ingester
|
||||
from .gallery_dl import ErrorType
|
||||
from .membership_reconcile import KEPT_KEY, STOPPED_KEY
|
||||
from .membership_roster import gated_reasons_for_sources
|
||||
@@ -67,6 +68,9 @@ class SourceRecord:
|
||||
artist_slug: str
|
||||
platform: str
|
||||
url: str
|
||||
# alembic 0115: the name the platform gives it, where the URL is opaque
|
||||
# (a Discord link is two ids). None until a walk has read it.
|
||||
display_name: str | None
|
||||
enabled: bool
|
||||
config_overrides: dict | None
|
||||
last_checked_at: str | None
|
||||
@@ -109,6 +113,7 @@ class SourceRecord:
|
||||
"artist_slug": self.artist_slug,
|
||||
"platform": self.platform,
|
||||
"url": self.url,
|
||||
"display_name": self.display_name,
|
||||
"enabled": self.enabled,
|
||||
"config_overrides": self.config_overrides,
|
||||
"last_checked_at": self.last_checked_at,
|
||||
@@ -125,6 +130,11 @@ class SourceRecord:
|
||||
"backfill_posts": self.backfill_posts,
|
||||
"tier_gated_count": self.tier_gated_count,
|
||||
"gated_reason": self.gated_reason,
|
||||
# Recover / recapture exist only on the native ingester. Sent so the
|
||||
# UI asks the backend's own predicate instead of keeping a copy of
|
||||
# the platform list — the copy said "patreon, subscribestar" for a
|
||||
# day after Discord went native (milestone 428).
|
||||
"native_ingester": uses_native_ingester(self.platform),
|
||||
}
|
||||
|
||||
|
||||
@@ -295,6 +305,7 @@ class SourceService:
|
||||
artist_slug=artist.slug,
|
||||
platform=source.platform,
|
||||
url=source.url,
|
||||
display_name=source.display_name,
|
||||
enabled=source.enabled,
|
||||
config_overrides=source.config_overrides,
|
||||
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
||||
@@ -551,8 +562,8 @@ class SourceService:
|
||||
whole source); the two flags are mutually exclusive, so arming recapture
|
||||
clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
|
||||
fresh from the top. The flag is cleared on completion (download_service)
|
||||
and on stop. Recapture is Patreon-only (the native ingester's post-record
|
||||
capture); inert elsewhere. The UI gates the action to Patreon sources."""
|
||||
and on stop. Recapture needs the native ingester's post-record capture,
|
||||
so the UI offers it on native sources only (`native_ingester`)."""
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
@@ -27,13 +27,10 @@ from .image_tag_apply import insert_image_tags
|
||||
|
||||
# image_tag.source stamped on title-heuristic WIP tags — distinct from the other
|
||||
# apply sources so provenance stays legible and a future undo can target only these.
|
||||
# HARD tier ("WIP"/"work in progress") is high-precision → trains the wip head.
|
||||
# Only the artist's own "WIP"/"work in progress" label counts — high-precision, so it
|
||||
# trains the wip head. A sketch/doodle/scribble tier (#1474) was retired in milestone
|
||||
# 430: a "sketch" is usually finished art, and its 6k tags flooded the review strip.
|
||||
WIP_TITLE_SOURCE = "wip_title"
|
||||
# SOFT tier (sketch/doodle/scribble, #1474) is LOWER-precision — a finished "sketch"
|
||||
# is often not WIP. This source is PROVISIONAL (in training_data._AUTO_SOURCES) so it
|
||||
# NEVER trains the wip head; a soft-tagged image that also looks like real content is
|
||||
# surfaced by the ring-loud audit for review.
|
||||
WIP_TITLE_SOFT_SOURCE = "wip_title_soft"
|
||||
|
||||
# A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
|
||||
# (space/underscore/hyphen separated). The letter-boundary lookarounds are what
|
||||
@@ -45,20 +42,10 @@ _WIP_RE = re.compile(
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Soft tier: sketch / doodle / scribble (+ plurals), letter-boundary anchored so
|
||||
# "sketchbook" / "kadoodle" don't trip it. Deliberately conservative — recall is
|
||||
# secondary because the soft source doesn't train the head and the ring-loud audit
|
||||
# catches false positives.
|
||||
_SOFT_WIP_RE = re.compile(
|
||||
r"(?<![A-Za-z])(?:sketch|sketches|doodle|doodles|scribble|scribbles)(?![A-Za-z])",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
# Coarse SQL prefilters for the backfill sweep — narrow the post scan to rows that
|
||||
# Coarse SQL prefilter for the backfill sweep — narrows the post scan to rows that
|
||||
# COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
|
||||
# Each MUST stay a SUPERSET of its regex or the sweep would silently miss posts.
|
||||
# It MUST stay a SUPERSET of the regex or the sweep would silently miss posts.
|
||||
WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
|
||||
SOFT_WIP_TITLE_SQL_PREFILTER = ("%sketch%", "%doodle%", "%scribble%")
|
||||
|
||||
# Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
|
||||
# ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
|
||||
@@ -66,19 +53,12 @@ _INSERT_CHUNK = 5000
|
||||
|
||||
|
||||
def matches_wip_title(title: str | None) -> bool:
|
||||
"""True when a post title explicitly marks it work-in-progress (HARD tier)."""
|
||||
"""True when a post title explicitly marks it work-in-progress."""
|
||||
if not title:
|
||||
return False
|
||||
return _WIP_RE.search(title) is not None
|
||||
|
||||
|
||||
def matches_soft_wip_title(title: str | None) -> bool:
|
||||
"""True when a title carries a SOFT WIP cue (sketch/doodle/scribble, #1474)."""
|
||||
if not title:
|
||||
return False
|
||||
return _SOFT_WIP_RE.search(title) is not None
|
||||
|
||||
|
||||
def resolve_wip_tag_id(session: Session) -> int | None:
|
||||
"""The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
|
||||
return session.execute(
|
||||
|
||||
@@ -211,11 +211,10 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
|
||||
state.reserved += len(reserved.get(hostname, []))
|
||||
state.consuming.update(q["name"] for q in queues)
|
||||
|
||||
# `pool.max-concurrency` is the number pool_grow/pool_shrink move and
|
||||
# the number the UI shows. Absent on a worker whose stats did not
|
||||
# answer, which leaves pool=None — unknown, not zero.
|
||||
pool = (stats.get(hostname) or {}).get("pool", {}).get("max-concurrency")
|
||||
if isinstance(pool, int):
|
||||
# Absent on a worker whose stats did not answer, which leaves
|
||||
# pool=None — unknown, not zero.
|
||||
pool = pool_size((stats.get(hostname) or {}).get("pool") or {})
|
||||
if pool is not None:
|
||||
state.pools[hostname] = pool
|
||||
|
||||
for state in out.values():
|
||||
@@ -223,6 +222,27 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
|
||||
return out
|
||||
|
||||
|
||||
def pool_size(pool_stats: dict) -> int | None:
|
||||
"""How many processes a prefork pool is running NOW, from `inspect stats`.
|
||||
|
||||
The length of `processes`, not `max-concurrency`. Celery reports
|
||||
`max-concurrency` as the pool's `limit`, set once at boot; `pool_grow` and
|
||||
`pool_shrink` hand straight to billiard and never touch it (celery 5.6
|
||||
`concurrency/prefork.py`: `self.grow = P.grow`). So it read 1 forever on a
|
||||
lane booted at 1, and the sizing sweep, computing `target - 1` on every
|
||||
tick, grew a scheduler capped at 2 to six processes while the System tab
|
||||
showed "6 / 1" (2026-09-24) — and could never shrink one, since 1 - 1 is 0.
|
||||
|
||||
`max-concurrency` is the fallback only for a pool that lists no processes
|
||||
(a non-prefork pool), where it is the best number there is.
|
||||
"""
|
||||
procs = pool_stats.get("processes")
|
||||
if isinstance(procs, list):
|
||||
return len(procs)
|
||||
limit = pool_stats.get("max-concurrency")
|
||||
return limit if isinstance(limit, int) else None
|
||||
|
||||
|
||||
def effective_slots(target: int) -> int:
|
||||
"""What a pool can actually be set to. Never below one process.
|
||||
|
||||
|
||||
@@ -137,7 +137,13 @@ IMPORT_BATCH_KEEP_DAYS = 30
|
||||
# (the import queue itself stays at the 5-min default for single
|
||||
# files); time_limit=2100.
|
||||
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"ml": 25,
|
||||
# ml: the scheduled auto-apply sweeps and refresh_character_prototypes run
|
||||
# to a 35-min hard limit (2100s); 25 swept them mid-run (#4432). The two
|
||||
# 65-min jobs have their own entries below.
|
||||
"ml": 40,
|
||||
# import: import_media_file's hard limit is 6 min (360s), one past the
|
||||
# 5-min default this queue fell to (#4432).
|
||||
"import": 10,
|
||||
# download_source legitimately walks 5-25 min (Patreon/gallery-dl
|
||||
# deep creators); its hard time_limit is DOWNLOAD_HARD_TIME_LIMIT
|
||||
# (1500s = 25m). The 5-min default flagged healthy in-flight walks as
|
||||
@@ -154,6 +160,12 @@ QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
# overrides below cover the outliers (backups, library audit).
|
||||
"maintenance": 75,
|
||||
"scan": 75,
|
||||
# The long lane (#4432). Until TaskRun.queue asked the router, nothing was
|
||||
# recorded here: these runs read as `maintenance` (75) or, for
|
||||
# translation, `default` (5 — which failed healthy 35-min runs). The
|
||||
# longest task without its own entry below is the admin family at a
|
||||
# 40-min hard limit; 45 = 40 + 5.
|
||||
"maintenance_long": 45,
|
||||
}
|
||||
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||
@@ -179,6 +191,10 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||
# external-fetch entry above — without an override a healthy in-flight walk
|
||||
# is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
|
||||
"backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
|
||||
# Head training and the manual head apply run to 65 min (3900s) — past the
|
||||
# ml queue's threshold (#4432). 70 = 65 + 5.
|
||||
"backend.app.tasks.ml.train_heads": 70,
|
||||
"backend.app.tasks.ml.apply_head_tags": 70,
|
||||
}
|
||||
|
||||
|
||||
@@ -473,6 +489,10 @@ def prune_task_runs() -> dict:
|
||||
(recover_stalled_task_runs) is the mechanism that flips them to
|
||||
terminal state; prune doesn't touch in-flight state.
|
||||
- 'retry' rows: treated as failures (>7d).
|
||||
- The NEWEST row of each task is never deleted, whatever its age: it is
|
||||
what the beat scheduler reads to know when a job last ran (#4408).
|
||||
Without it a weekly job's last run would be pruned after a day, and beat
|
||||
would think it had never run and fire it on every restart.
|
||||
|
||||
Returns dict of how many rows were deleted in each bucket.
|
||||
"""
|
||||
@@ -480,16 +500,19 @@ def prune_task_runs() -> dict:
|
||||
now = datetime.now(UTC)
|
||||
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
|
||||
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
|
||||
newest = select(func.max(TaskRun.id)).group_by(TaskRun.task_name)
|
||||
with SessionLocal() as session:
|
||||
ok_deleted = session.execute(
|
||||
delete(TaskRun)
|
||||
.where(TaskRun.status == "ok")
|
||||
.where(TaskRun.finished_at < ok_cutoff)
|
||||
.where(TaskRun.id.not_in(newest))
|
||||
).rowcount or 0
|
||||
fail_deleted = session.execute(
|
||||
delete(TaskRun)
|
||||
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
|
||||
.where(TaskRun.finished_at < fail_cutoff)
|
||||
.where(TaskRun.id.not_in(newest))
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||
@@ -708,6 +731,32 @@ def recover_stalled_download_events() -> int:
|
||||
return events_recovered
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.maintenance.date_posts_from_records",
|
||||
soft_time_limit=1500,
|
||||
time_limit=1800,
|
||||
)
|
||||
def date_posts_from_records() -> dict:
|
||||
"""Date undated native posts from the records their walk left on disk
|
||||
(#4436). Hourly and self-limiting: once every post is dated it is one
|
||||
empty query."""
|
||||
from ..services.importer import Importer
|
||||
from ..services.post_record_repair import date_posts_from_records as _repair
|
||||
from ..services.thumbnailer import Thumbnailer
|
||||
|
||||
images_root = IMAGES_ROOT
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
importer = Importer(
|
||||
session=session,
|
||||
images_root=images_root,
|
||||
import_root=images_root,
|
||||
thumbnailer=Thumbnailer(images_root=images_root),
|
||||
settings=ImportSettings.load_sync(session),
|
||||
)
|
||||
return _repair(session, importer, images_root)
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
|
||||
def recover_stalled_backup_runs() -> int:
|
||||
"""Flip BackupRun rows stuck in running/restoring past the hard limit
|
||||
@@ -1034,8 +1083,7 @@ def cleanup_old_download_events() -> int:
|
||||
|
||||
def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
||||
"""One keyset-paginated pass over posts whose title matches a WIP tier, applying
|
||||
`tag_id` (stamped `source`) to their images. Shared by the hard + soft tiers
|
||||
(#1458 / #1474). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
||||
`tag_id` (stamped `source`) to their images (#1458). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
||||
`matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
|
||||
count newly applied."""
|
||||
from ..models import Post
|
||||
@@ -1075,26 +1123,17 @@ def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
||||
)
|
||||
def backfill_wip_title_tags() -> int:
|
||||
"""Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
|
||||
images — the operator-triggered back-catalogue catch-up (task #1458 hard tier +
|
||||
#1474 soft tier). New imports are tagged live by the importer; this covers the
|
||||
existing library.
|
||||
|
||||
HARD tier ("WIP"/"work in progress") always runs (the operator triggered the
|
||||
scan); the SOFT tier (sketch/doodle, provisional source) runs only when
|
||||
wip_soft_title_tagging_enabled, AFTER hard so a title matching both keeps the
|
||||
trained hard tag (ON CONFLICT DO NOTHING). Keyset-paginated, restart-safe.
|
||||
images — the operator-triggered back-catalogue catch-up (task #1458). New
|
||||
imports are tagged live by the importer; this covers the existing library.
|
||||
Keyset-paginated, restart-safe.
|
||||
|
||||
Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
|
||||
posts and silently undo a manual WIP removal, so it stays an explicit operator
|
||||
action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
|
||||
"""
|
||||
from ..models import ImportSettings
|
||||
from ..services.wip_title import (
|
||||
SOFT_WIP_TITLE_SQL_PREFILTER,
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
WIP_TITLE_SOURCE,
|
||||
WIP_TITLE_SQL_PREFILTER,
|
||||
matches_soft_wip_title,
|
||||
matches_wip_title,
|
||||
resolve_wip_tag_id,
|
||||
)
|
||||
@@ -1107,16 +1146,10 @@ def backfill_wip_title_tags() -> int:
|
||||
"backfill_wip_title_tags: no `wip` system tag present; nothing to do"
|
||||
)
|
||||
return 0
|
||||
settings = ImportSettings.load_sync(session)
|
||||
applied = _backfill_wip_tier(
|
||||
session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
|
||||
WIP_TITLE_SOURCE,
|
||||
)
|
||||
if settings.wip_soft_title_tagging_enabled:
|
||||
applied += _backfill_wip_tier(
|
||||
session, tag_id, SOFT_WIP_TITLE_SQL_PREFILTER, matches_soft_wip_title,
|
||||
WIP_TITLE_SOFT_SOURCE,
|
||||
)
|
||||
if applied:
|
||||
log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
|
||||
return applied
|
||||
|
||||
@@ -620,24 +620,6 @@ def scheduled_process_auto_apply() -> str:
|
||||
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
||||
|
||||
|
||||
@celery.task(
|
||||
name="backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
||||
soft_time_limit=1800, time_limit=2100,
|
||||
)
|
||||
def scheduled_soft_wip_conflict_audit() -> str:
|
||||
"""Ring-loud audit over the SOFT WIP-title cohort (#1474) — flag sketch/doodle
|
||||
auto-tags that ALSO look like real content for review. No-op when there are no
|
||||
content heads; idempotent (already-flagged images skipped). Runs regardless of
|
||||
the process-sweep toggle, since soft-title tags come from the importer, not that
|
||||
sweep. Wall-clock bounded by the task time limits."""
|
||||
from ..services.ml.heads import soft_wip_conflict_audit
|
||||
|
||||
SessionLocal = _sync_session_factory()
|
||||
with SessionLocal() as session:
|
||||
result = soft_wip_conflict_audit(session)
|
||||
return f"scanned={result['n_scanned']} flagged={result['n_flagged']}"
|
||||
|
||||
|
||||
@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
|
||||
def prune_presentation_reviews() -> str:
|
||||
"""Retention (rule 89): drop RESOLVED presentation-review flags older than 30
|
||||
|
||||
+7
-4
@@ -15,7 +15,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
|
||||
## Secondary runtime image
|
||||
|
||||
node:24-bookworm-slim — `.forgejo/workflows/extension.yml` only.
|
||||
node:24-bookworm-slim — `build.yml`'s `extension-test` job only.
|
||||
|
||||
`.forgejo/workflows/release.yml` runs on `ci-python:3.14` like everything else
|
||||
and installs nothing: it needs git and stdlib python, and builds no image.
|
||||
@@ -29,8 +29,8 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
||||
|
||||
- `pip install -r requirements.txt pytest pytest-asyncio` — in `backend-lint-and-test` and `integration` jobs
|
||||
- `npm install --no-audit --no-fund` — in `frontend-build` job
|
||||
- `npm install --no-audit --no-fund` — in `extension.yml`'s `lint` job (web-ext + vitest)
|
||||
- `unzip` — in `extension.yml`'s "Verify XPI contents" step, installed via apt
|
||||
- `npm install --no-audit --no-fund` — in `build.yml`'s `extension-test` job (web-ext + vitest)
|
||||
- `unzip` — in `extension-test`'s "Verify XPI contents" step, installed via apt
|
||||
only when absent (`node:24-bookworm-slim` may or may not carry it). Debian
|
||||
package, ~2s. Not worth baking into a shared image for a single consumer, per
|
||||
`docs/process.md`'s ">1 project" rule.
|
||||
@@ -180,7 +180,10 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
||||
BUILD_REF` that every checkout in the file takes, rather than per job —
|
||||
otherwise `sign-extension` would derive dev's extension version while
|
||||
`build-web` bundled main's, and the release download would 404 on a version
|
||||
that exists perfectly well. Every job then ASSERTS its checkout is `main`
|
||||
that exists perfectly well. On every other trigger `BUILD_REF` is the
|
||||
triggering COMMIT (`github.sha`), not the branch: a branch is re-resolved
|
||||
per job, so a push landing mid-run used to move the publishing jobs onto a
|
||||
commit the run's lanes never tested (run 7499, #4427). Every job then ASSERTS its checkout is `main`
|
||||
before doing anything, because `env` inside `with:` is not a context this
|
||||
runner is known to evaluate — if it silently resolved to empty, checkout
|
||||
would fall back to the triggering ref and the refresh would publish dev's
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ x-celery-healthcheck: &celery_healthcheck
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
|
||||
@@ -88,7 +88,12 @@ async function checkForUpdateInfo() {
|
||||
currentVersion,
|
||||
latestVersion,
|
||||
channel,
|
||||
xpiUrl: info && info.latest_url ? `${base}${info.latest_url}` : null,
|
||||
// Where the Update button sends the operator: FC's own install card, not
|
||||
// the XPI. Firefox refuses an add-on install whose navigation an extension
|
||||
// started (tabs.create on the .xpi dies with NS_ERROR_FAILURE — operator-
|
||||
// flagged 2026-09-25); it accepts one from a user click on a web page,
|
||||
// which is exactly what the card's Install button is.
|
||||
installPageUrl: base ? `${base}/subscriptions?tab=settings` : null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -128,7 +133,8 @@ browser.webRequest.onBeforeSendHeaders.addListener(
|
||||
saveDiscordToken(auth.value);
|
||||
}
|
||||
},
|
||||
{ urls: ['https://discord.com/api/*'] },
|
||||
// ptb/canary are Discord's beta clients; their API calls carry the same token.
|
||||
{ urls: ['https://discord.com/api/*', 'https://*.discord.com/api/*'] },
|
||||
['requestHeaders'],
|
||||
);
|
||||
|
||||
@@ -213,7 +219,17 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
if (key === 'discord') {
|
||||
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
||||
await api.uploadCredentials('discord', 'token', discordToken);
|
||||
return { success: true };
|
||||
// Then have FC try it against a Discord source, so a token Discord
|
||||
// has already revoked shows up here rather than at the next check.
|
||||
// A failed verify never undoes the upload: valid=null means FC could
|
||||
// not test (no Discord source yet), not that the token is bad.
|
||||
let verify = null;
|
||||
try {
|
||||
verify = await api.verifyCredential('discord');
|
||||
} catch (e) {
|
||||
verify = { valid: null, reason: e.message };
|
||||
}
|
||||
return { success: true, verify };
|
||||
}
|
||||
return { error: 'Unsupported platform.' };
|
||||
} catch (e) {
|
||||
@@ -256,14 +272,40 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
|
||||
case 'ADD_AS_SOURCE':
|
||||
try {
|
||||
return await api.quickAddSource(msg.url);
|
||||
return await api.quickAddSource(msg.url, {
|
||||
artistId: msg.artistId ?? null,
|
||||
artistName: msg.artistName ?? null,
|
||||
usePlatformName: msg.usePlatformName === true,
|
||||
discordAuthors: Array.isArray(msg.discordAuthors) ? msg.discordAuthors : null,
|
||||
});
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'DISCORD_POSTERS':
|
||||
try {
|
||||
return await api.getDiscordPosters(msg.url);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'SET_DISCORD_POSTERS':
|
||||
try {
|
||||
return await api.setDiscordPosters(msg.url, msg.authors || []);
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'SEARCH_ARTISTS':
|
||||
try {
|
||||
return { artists: await api.searchArtists(msg.q || '') };
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'PROBE_SOURCE':
|
||||
try {
|
||||
return await api.probeSource(msg.url);
|
||||
return await api.probeSource(msg.url, { names: msg.names === true });
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
@@ -40,3 +40,67 @@
|
||||
from { transform: translateY(20px); opacity: 0; }
|
||||
to { transform: translateY(0); opacity: 1; }
|
||||
}
|
||||
|
||||
/* Discord Add panel — sits above the chip. Same slate/parchment palette. */
|
||||
.fc-panel {
|
||||
all: revert;
|
||||
position: fixed; bottom: 76px; right: 24px; z-index: 2147483647;
|
||||
box-sizing: border-box; width: 320px; max-width: calc(100vw - 48px);
|
||||
padding: 14px 16px; border-radius: 10px;
|
||||
background: rgb(20, 23, 26); color: rgb(232, 228, 216);
|
||||
border: 1px solid rgb(60, 64, 70);
|
||||
font: 14px/1.4 system-ui, sans-serif;
|
||||
box-shadow: 0 8px 28px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
.fc-panel__title { font-weight: 600; color: rgb(244, 186, 122); }
|
||||
.fc-panel__sub { color: rgb(170, 166, 156); font-size: 12px; margin-bottom: 8px; }
|
||||
.fc-panel__label {
|
||||
margin: 10px 0 4px; font-size: 11px; letter-spacing: 0.06em;
|
||||
text-transform: uppercase; color: rgb(170, 166, 156);
|
||||
}
|
||||
.fc-panel__radio { display: flex; gap: 8px; align-items: center; padding: 2px 0; cursor: pointer; }
|
||||
.fc-panel__radio input { margin: 0; accent-color: rgb(244, 186, 122); }
|
||||
.fc-panel__input {
|
||||
all: revert; box-sizing: border-box; width: 100%;
|
||||
padding: 7px 9px; border-radius: 6px;
|
||||
border: 1px solid rgb(70, 74, 80); background: rgb(12, 14, 16); color: inherit;
|
||||
font: inherit;
|
||||
}
|
||||
.fc-panel__input:focus { outline: 2px solid rgb(244, 186, 122); outline-offset: -1px; }
|
||||
.fc-panel__results { display: flex; flex-direction: column; max-height: 160px; overflow-y: auto; }
|
||||
.fc-panel__result {
|
||||
all: revert; text-align: left; padding: 6px 9px; border: none; border-radius: 4px;
|
||||
background: transparent; color: inherit; font: inherit; cursor: pointer;
|
||||
}
|
||||
.fc-panel__result:hover, .fc-panel__result:focus { background: rgb(36, 40, 46); }
|
||||
.fc-panel__hint { margin-top: 8px; font-size: 12px; color: rgb(170, 166, 156); }
|
||||
.fc-panel__actions { display: flex; justify-content: flex-end; gap: 8px; margin-top: 12px; }
|
||||
.fc-panel__btn {
|
||||
all: revert; padding: 6px 14px; border-radius: 999px; cursor: pointer;
|
||||
border: 1px solid rgb(70, 74, 80); background: transparent; color: inherit;
|
||||
font: 500 13px/1.2 system-ui, sans-serif;
|
||||
}
|
||||
.fc-panel__btn--primary { border-color: rgb(244, 186, 122); background: rgb(244, 186, 122); color: rgb(20, 23, 26); }
|
||||
.fc-panel__btn:disabled { opacity: 0.5; cursor: default; }
|
||||
|
||||
/* Artist autocomplete — the list sits under the field, inside the panel. */
|
||||
.fc-panel__combo { position: relative; }
|
||||
.fc-panel__results {
|
||||
margin-top: 4px; border-radius: 6px;
|
||||
background: rgb(12, 14, 16);
|
||||
}
|
||||
.fc-panel__results:empty { display: none; }
|
||||
.fc-panel__results:not(:empty) { border: 1px solid rgb(70, 74, 80); padding: 3px; }
|
||||
.fc-panel__result { display: flex; justify-content: space-between; align-items: center; width: 100%; box-sizing: border-box; }
|
||||
.fc-panel__result--active { background: rgb(52, 44, 32); outline: 1px solid rgb(244, 186, 122); }
|
||||
.fc-panel__result--new { color: rgb(244, 186, 122); }
|
||||
.fc-panel__result-tag { font-size: 11px; color: rgb(140, 220, 160); }
|
||||
.fc-panel__empty { padding: 6px 9px; font-size: 12px; color: rgb(170, 166, 156); }
|
||||
.fc-panel__hint--match { color: rgb(140, 220, 160); }
|
||||
/* The panel's own [hidden] — its rows are display:flex, which beats the UA's. */
|
||||
.fc-panel [hidden] { display: none !important; }
|
||||
.fc-panel__rename { margin-top: 8px; font-size: 13px; }
|
||||
/* Poster picker (#4488): who posts in the channel, the likely creator ticked. */
|
||||
.fc-panel__posters { max-height: 220px; overflow-y: auto; }
|
||||
.fc-panel__poster { flex-wrap: wrap; }
|
||||
.fc-panel__poster-meta { flex-basis: 100%; padding-left: 21px; font-size: 11px; color: rgb(170, 166, 156); }
|
||||
|
||||
@@ -5,42 +5,61 @@
|
||||
// Cached probe result for the current URL so click-handlers know which
|
||||
// action to dispatch without round-tripping again.
|
||||
let currentProbe = null;
|
||||
// Bumped on every evaluate(): a probe that answers after the operator has
|
||||
// navigated on is for a page they've left, and must not repaint the chip.
|
||||
let generation = 0;
|
||||
let lastUrl = window.location.href;
|
||||
|
||||
evaluate();
|
||||
|
||||
const reEval = () => evaluate();
|
||||
window.addEventListener('popstate', reEval);
|
||||
const origPush = history.pushState;
|
||||
history.pushState = function () { origPush.apply(this, arguments); reEval(); };
|
||||
// SPA navigation. Patreon, SubscribeStar and above all Discord change
|
||||
// channel/page without a reload. Patching history.pushState from here never
|
||||
// worked: a content script runs in an isolated world, so the page's own
|
||||
// pushState is not the function we'd replace. Polling the URL is the one
|
||||
// signal that sees every navigation, and costs a string compare.
|
||||
window.addEventListener('popstate', () => onUrlMaybeChanged());
|
||||
setInterval(onUrlMaybeChanged, 500);
|
||||
|
||||
function onUrlMaybeChanged() {
|
||||
if (window.location.href === lastUrl) return;
|
||||
lastUrl = window.location.href;
|
||||
closePanel();
|
||||
evaluate();
|
||||
}
|
||||
|
||||
async function evaluate() {
|
||||
const mine = ++generation;
|
||||
const url = window.location.href;
|
||||
const platform = getPlatformFromUrl(url);
|
||||
const onArtist = platform && isArtistPage(url, platform);
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
if (!onArtist) {
|
||||
if (btn) btn.remove();
|
||||
removeButton();
|
||||
currentProbe = null;
|
||||
return;
|
||||
}
|
||||
// On artist pages, ask the backend what state the URL is in BEFORE
|
||||
// injecting the button — so the chip can render the right state on
|
||||
// first paint instead of flashing the generic "Add" copy and
|
||||
// updating afterwards.
|
||||
// Ask the backend what state the URL is in BEFORE drawing the button, so
|
||||
// the chip renders the right state on first paint instead of flashing the
|
||||
// generic "Add" copy and updating afterwards.
|
||||
let probe;
|
||||
try {
|
||||
probe = await browser.runtime.sendMessage({ type: 'PROBE_SOURCE', url });
|
||||
} catch (e) {
|
||||
probe = { error: e?.message || 'probe failed' };
|
||||
}
|
||||
if (mine !== generation) return;
|
||||
currentProbe = probe;
|
||||
if (probe?.state === 'unknown_platform') {
|
||||
if (btn) btn.remove();
|
||||
removeButton();
|
||||
return;
|
||||
}
|
||||
renderButton(probe);
|
||||
}
|
||||
|
||||
function removeButton() {
|
||||
document.getElementById('fc-add-source-btn')?.remove();
|
||||
closePanel();
|
||||
}
|
||||
|
||||
function renderButton(probe) {
|
||||
let btn = document.getElementById('fc-add-source-btn');
|
||||
if (!btn) {
|
||||
@@ -51,79 +70,44 @@
|
||||
}
|
||||
// Reset state classes so re-renders (SPA navigation) don't stack.
|
||||
btn.className = 'fc-add-source-btn';
|
||||
btn.classList.add(`fc-add-source-btn--${stateModifier(probe)}`);
|
||||
btn.textContent = labelFor(probe);
|
||||
btn.classList.add(`fc-add-source-btn--${chipState(probe)}`);
|
||||
btn.textContent = chipLabel(probe, PLATFORMS[probe?.platform]?.name || probe?.platform || '');
|
||||
btn.disabled = false;
|
||||
}
|
||||
|
||||
function stateModifier(probe) {
|
||||
if (!probe || probe.error) return 'new';
|
||||
return ({
|
||||
source_match: 'source-match',
|
||||
artist_match: 'artist-match',
|
||||
new: 'new',
|
||||
})[probe.state] || 'new';
|
||||
}
|
||||
|
||||
function labelFor(probe) {
|
||||
if (!probe || probe.error) return '+ Add to FabledCurator';
|
||||
const platformName = platformDisplayName(probe.platform);
|
||||
const artistName = probe.artist?.name;
|
||||
switch (probe.state) {
|
||||
case 'source_match':
|
||||
return `✓ In FabledCurator · ${platformName}`;
|
||||
case 'artist_match':
|
||||
return `+ Add ${platformName} source to ${artistName || 'artist'}`;
|
||||
case 'new':
|
||||
default:
|
||||
return '+ Add to FabledCurator';
|
||||
}
|
||||
}
|
||||
|
||||
function platformDisplayName(key) {
|
||||
return PLATFORMS[key]?.name || key || '';
|
||||
}
|
||||
|
||||
async function onClick() {
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
if (!btn) return;
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
const probe = currentProbe;
|
||||
|
||||
if (probe?.state === 'source_match') {
|
||||
btn.textContent = 'Opening…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'OPEN_ARTIST_PAGE',
|
||||
slug: probe.artist?.slug,
|
||||
});
|
||||
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
btn.textContent = 'Adding…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'ADD_AS_SOURCE',
|
||||
url: window.location.href,
|
||||
});
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
} else {
|
||||
const verb = r.created_source ? 'Added' : 'Already a source for';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
||||
// Re-probe so the chip flips green without waiting for the next
|
||||
// navigation.
|
||||
evaluate();
|
||||
// A followed Discord channel opens its poster list (#4488); the artist
|
||||
// page is one click further, in that panel.
|
||||
if (probe.platform === 'discord' && probe.discord?.channel_id) {
|
||||
if (document.getElementById('fc-add-panel')) closePanel();
|
||||
else openPostersPanel(btn, probe);
|
||||
return;
|
||||
}
|
||||
await openArtist(btn, probe.artist?.slug);
|
||||
return;
|
||||
}
|
||||
// Every add goes through the panel, so the operator can match the page to
|
||||
// an artist FabledCurator already has (the same creator is often spelled
|
||||
// differently per platform) instead of minting a duplicate.
|
||||
if (document.getElementById('fc-add-panel')) {
|
||||
closePanel();
|
||||
return;
|
||||
}
|
||||
await openAddPanel(btn, probe);
|
||||
}
|
||||
|
||||
async function openArtist(btn, slug) {
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
btn.textContent = 'Opening…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'OPEN_ARTIST_PAGE', slug });
|
||||
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
@@ -132,6 +116,425 @@
|
||||
}
|
||||
}
|
||||
|
||||
// One add, shared by the one-click chip and the Discord panel. Resolves
|
||||
// true on success.
|
||||
async function add(btn, request) {
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
btn.textContent = 'Adding…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'ADD_AS_SOURCE', ...request });
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
return false;
|
||||
}
|
||||
const verb = r.created_source ? 'Added to' : 'Already a source for';
|
||||
const renamed = r.renamed_from ? ` — renamed from “${r.renamed_from}”` : '';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})${renamed}`, 'success');
|
||||
// Re-probe so the chip flips green without waiting for a navigation.
|
||||
evaluate();
|
||||
return true;
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
return false;
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Add panel ----
|
||||
// Who: the suggested artist, one found by search, or a new one by name —
|
||||
// on every platform. Where (Discord only): this channel or the whole
|
||||
// server. On Patreon, joining an artist known by another name offers the
|
||||
// Patreon name, which the operator treats as canon. Built with
|
||||
// createElement only — server, channel and artist names are other people's
|
||||
// text.
|
||||
|
||||
function el(tag, props = {}, children = []) {
|
||||
const node = document.createElement(tag);
|
||||
const { class: className, text, ...rest } = props;
|
||||
if (className) node.className = className;
|
||||
if (text != null) node.textContent = text;
|
||||
Object.assign(node, rest);
|
||||
for (const c of children) node.appendChild(c);
|
||||
return node;
|
||||
}
|
||||
|
||||
function closePanel() {
|
||||
document.getElementById('fc-add-panel')?.remove();
|
||||
}
|
||||
|
||||
// ---- Poster picker (#4488) ----
|
||||
// Who posted in this channel lately, read by FabledCurator with its Discord
|
||||
// token, the creator the server is named for (or owned by) ticked. The
|
||||
// source keeps their ids: a username can change on a whim, an id can't, so
|
||||
// the list never quietly stops matching. Names are other people's text —
|
||||
// createElement only.
|
||||
|
||||
function posterSection() {
|
||||
const list = el('div', { class: 'fc-panel__posters' }, [
|
||||
el('div', { class: 'fc-panel__hint', text: 'Reading who posts here…' }),
|
||||
]);
|
||||
const node = el('div', {}, [
|
||||
el('div', { class: 'fc-panel__label', text: 'Only posts by' }),
|
||||
list,
|
||||
]);
|
||||
const state = { data: null, picked: new Set() };
|
||||
|
||||
function render() {
|
||||
const posters = withSelected(state.data);
|
||||
if (!posters.length) {
|
||||
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: 'No one has posted here lately.' }));
|
||||
return;
|
||||
}
|
||||
const rows = posters.map((p) => {
|
||||
const id = String(p.id);
|
||||
const box = el('input', { type: 'checkbox', checked: state.picked.has(id) });
|
||||
box.addEventListener('change', () => {
|
||||
if (box.checked) state.picked.add(id);
|
||||
else state.picked.delete(id);
|
||||
hint.textContent = pickHint();
|
||||
});
|
||||
const bits = [`${p.images} image${p.images === 1 ? '' : 's'}`, `${p.messages} message${p.messages === 1 ? '' : 's'}`];
|
||||
if (p.reasons?.length) bits.unshift(p.reasons.join(', '));
|
||||
return el('label', { class: 'fc-panel__radio fc-panel__poster' }, [
|
||||
box,
|
||||
el('span', { text: posterLabel(p) }),
|
||||
el('span', { class: 'fc-panel__poster-meta', text: bits.join(' · ') }),
|
||||
]);
|
||||
});
|
||||
const hint = el('div', { class: 'fc-panel__hint', text: pickHint() });
|
||||
list.replaceChildren(...rows, hint);
|
||||
}
|
||||
|
||||
function pickHint() {
|
||||
return state.picked.size
|
||||
? 'Only the ticked people’s posts are taken.'
|
||||
: 'Nothing ticked: everyone’s posts are taken.';
|
||||
}
|
||||
|
||||
async function load() {
|
||||
let r;
|
||||
try {
|
||||
r = await browser.runtime.sendMessage({ type: 'DISCORD_POSTERS', url: window.location.href });
|
||||
} catch (e) {
|
||||
r = { error: e.message };
|
||||
}
|
||||
if (r?.error) {
|
||||
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: `Couldn’t list posters — ${r.error}` }));
|
||||
return;
|
||||
}
|
||||
state.data = r;
|
||||
state.picked = initialPosterPick(r);
|
||||
render();
|
||||
}
|
||||
|
||||
// null until the list has loaded: an add made before then leaves the
|
||||
// source's list as it is instead of clearing it.
|
||||
const request = () => (state.data ? postersRequest(state.data.posters, state.picked) : null);
|
||||
return { node, load, request };
|
||||
}
|
||||
|
||||
function openPostersPanel(btn, probe) {
|
||||
closePanel();
|
||||
const d = probe.discord || {};
|
||||
const posters = posterSection();
|
||||
const saveBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Save' });
|
||||
const artistBtn = el('button', { class: 'fc-panel__btn', text: `Open ${probe.artist?.name || 'artist'}` });
|
||||
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Close' });
|
||||
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||
el('div', { class: 'fc-panel__title', text: 'Discord source' }),
|
||||
el('div', { class: 'fc-panel__sub', text: d.channel_id ? channelLabel(d) : serverLabel(d) }),
|
||||
posters.node,
|
||||
el('div', { class: 'fc-panel__actions' }, [artistBtn, cancelBtn, saveBtn]),
|
||||
]);
|
||||
panel.addEventListener('keydown', (e) => {
|
||||
e.stopPropagation();
|
||||
if (e.key === 'Escape') closePanel();
|
||||
});
|
||||
cancelBtn.addEventListener('click', closePanel);
|
||||
artistBtn.addEventListener('click', () => { closePanel(); openArtist(btn, probe.artist?.slug); });
|
||||
saveBtn.addEventListener('click', async () => {
|
||||
const authors = posters.request();
|
||||
if (!authors) return;
|
||||
saveBtn.disabled = true;
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'SET_DISCORD_POSTERS', url: window.location.href, authors,
|
||||
});
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
return;
|
||||
}
|
||||
showToast(authors.length ? `Only posts by ${authors.map((a) => a.name || a.id).join(', ')}` : 'Taking everyone’s posts', 'success');
|
||||
closePanel();
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
saveBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
document.body.appendChild(panel);
|
||||
posters.load();
|
||||
}
|
||||
|
||||
async function openAddPanel(btn, probe) {
|
||||
closePanel();
|
||||
const platformName = PLATFORMS[probe.platform]?.name || probe.platform;
|
||||
// Discord's probe already carries its names. Patreon/SubscribeStar read the
|
||||
// creator's display name only now, when the panel needs it — a request to
|
||||
// the platform the chip's own probe deliberately doesn't make.
|
||||
if (probe.platform !== 'discord') {
|
||||
const original = btn.textContent;
|
||||
btn.disabled = true;
|
||||
btn.textContent = `Reading the ${platformName} name…`;
|
||||
try {
|
||||
const named = await browser.runtime.sendMessage({
|
||||
type: 'PROBE_SOURCE', url: window.location.href, names: true,
|
||||
});
|
||||
if (named && !named.error) probe = named;
|
||||
} catch { /* fall back to the chip's probe: the URL handle */ }
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
if (probe.state === 'source_match') {
|
||||
renderButton(probe);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const d = probe.discord || {};
|
||||
const discord = probe.platform === 'discord';
|
||||
const choice = panelDefaults(probe, window.location.href);
|
||||
// #4488: on a channel page, pick whose posts the new source takes.
|
||||
const posters = discord && d.channel_id ? posterSection() : null;
|
||||
|
||||
const scopeRow = (value, label, disabled) => {
|
||||
const input = el('input', {
|
||||
type: 'radio', name: 'fc-discord-scope', value,
|
||||
checked: choice.scope === value, disabled,
|
||||
});
|
||||
input.addEventListener('change', () => { choice.scope = value; refresh(); });
|
||||
return el('label', { class: 'fc-panel__radio' }, [input, el('span', { text: label })]);
|
||||
};
|
||||
|
||||
// The artist field is an autocomplete over FC's artists: it searches as
|
||||
// soon as the panel opens (with the prefilled name) and on every keystroke,
|
||||
// lists the matches under the field, fills in the rest of the top match as
|
||||
// you type (Tab or Enter accepts it, typing on replaces it), and picks an
|
||||
// artist whose name IS the text, spacing and case aside, without being
|
||||
// asked. ↑/↓ walk the list; the last row creates a new artist instead.
|
||||
const nameInput = el('input', {
|
||||
type: 'text', class: 'fc-panel__input', value: choice.artistName,
|
||||
placeholder: 'Search artists or type a new name',
|
||||
autocomplete: 'off', spellcheck: false,
|
||||
role: 'combobox',
|
||||
});
|
||||
nameInput.setAttribute('aria-autocomplete', 'both');
|
||||
nameInput.setAttribute('aria-expanded', 'false');
|
||||
const results = el('div', { class: 'fc-panel__results', role: 'listbox' });
|
||||
const hint = el('div', { class: 'fc-panel__hint' });
|
||||
const addBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Add' });
|
||||
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Cancel' });
|
||||
|
||||
// Patreon is canon: joining an artist known by another name takes the
|
||||
// Patreon name unless this is unticked. Shown only when it would rename.
|
||||
const renameBox = el('input', { type: 'checkbox', checked: choice.adoptPlatformName });
|
||||
const renameText = el('span');
|
||||
const renameRow = el('label', { class: 'fc-panel__radio fc-panel__rename' }, [renameBox, renameText]);
|
||||
renameBox.addEventListener('change', () => { choice.adoptPlatformName = renameBox.checked; refresh(); });
|
||||
|
||||
const sub = discord
|
||||
? serverLabel(d)
|
||||
: [probe.display_name, probe.slug].filter(Boolean).filter((v, i, a) => a.indexOf(v) === i).join(' · ');
|
||||
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||
el('div', { class: 'fc-panel__title', text: `Add ${platformName} source` }),
|
||||
el('div', { class: 'fc-panel__sub', text: sub }),
|
||||
...(discord ? [
|
||||
el('div', { class: 'fc-panel__label', text: 'Follow' }),
|
||||
scopeRow('channel', d.channel_id ? channelLabel(d) : 'this channel', !d.channel_id),
|
||||
scopeRow('server', `Every channel in ${serverLabel(d)}`, false),
|
||||
] : []),
|
||||
el('div', { class: 'fc-panel__label', text: 'Artist' }),
|
||||
el('div', { class: 'fc-panel__combo' }, [nameInput, results]),
|
||||
renameRow,
|
||||
...(posters ? [posters.node] : []),
|
||||
hint,
|
||||
el('div', { class: 'fc-panel__actions' }, [cancelBtn, addBtn]),
|
||||
]);
|
||||
|
||||
// Search state: the rows on screen, which one ↑/↓ has highlighted (-1 =
|
||||
// none), and whether the list is open.
|
||||
let rows = [];
|
||||
let active = -1;
|
||||
let listOpen = false;
|
||||
|
||||
function refresh() {
|
||||
const req = addRequest(choice);
|
||||
addBtn.disabled = !req;
|
||||
if (!req) hint.textContent = 'Pick an artist or type a name.';
|
||||
else if (req.artistId != null) hint.textContent = `✓ Connects to ${choice.artist.name}, already in FabledCurator.`;
|
||||
else if (req.artistName) hint.textContent = `Creates a new artist “${req.artistName}”.`;
|
||||
else hint.textContent = `Creates a new artist, named from the ${platformName} page.`;
|
||||
hint.classList.toggle('fc-panel__hint--match', !!req && req.artistId != null);
|
||||
const offer = renameOffer(choice);
|
||||
renameRow.hidden = !offer;
|
||||
if (offer) renameText.textContent = `Rename “${offer.from}” to the Patreon name “${offer.to}”`;
|
||||
}
|
||||
|
||||
function pick(artist) {
|
||||
choice.artist = artist ? { id: artist.id, name: artist.name } : null;
|
||||
if (artist) {
|
||||
choice.artistName = artist.name;
|
||||
nameInput.value = artist.name;
|
||||
}
|
||||
closeList();
|
||||
refresh();
|
||||
}
|
||||
|
||||
function closeList() {
|
||||
listOpen = false;
|
||||
active = -1;
|
||||
results.replaceChildren();
|
||||
nameInput.setAttribute('aria-expanded', 'false');
|
||||
}
|
||||
|
||||
function renderList() {
|
||||
const typed = nameInput.value.trim();
|
||||
const exact = exactArtistMatch(typed, rows);
|
||||
const items = rows.map((a, i) => {
|
||||
const row = el('button', { type: 'button', class: 'fc-panel__result', role: 'option' }, [
|
||||
el('span', { text: a.name }),
|
||||
]);
|
||||
if (choice.artist && choice.artist.id === a.id) {
|
||||
row.appendChild(el('span', { class: 'fc-panel__result-tag', text: 'selected' }));
|
||||
}
|
||||
row.classList.toggle('fc-panel__result--active', i === active);
|
||||
// mousedown, not click: it fires before the input's blur closes the list.
|
||||
row.addEventListener('mousedown', (e) => { e.preventDefault(); pick(a); });
|
||||
return row;
|
||||
});
|
||||
if (typed && !exact) {
|
||||
const i = rows.length;
|
||||
const create = el('button', { type: 'button', class: 'fc-panel__result fc-panel__result--new', role: 'option',
|
||||
text: `+ New artist “${typed}”` });
|
||||
create.classList.toggle('fc-panel__result--active', i === active);
|
||||
create.addEventListener('mousedown', (e) => { e.preventDefault(); pick(null); choice.artistName = typed; refresh(); });
|
||||
items.push(create);
|
||||
}
|
||||
if (typed && !rows.length) {
|
||||
items.unshift(el('div', { class: 'fc-panel__empty', text: 'No FabledCurator artist matches.' }));
|
||||
}
|
||||
results.replaceChildren(...items);
|
||||
listOpen = items.length > 0;
|
||||
nameInput.setAttribute('aria-expanded', String(listOpen));
|
||||
results.querySelector('.fc-panel__result--active')?.scrollIntoView({ block: 'nearest' });
|
||||
}
|
||||
|
||||
let debounce = null;
|
||||
let searchSeq = 0;
|
||||
// `autofill` is false for deletions: filling the name back in as you
|
||||
// backspace would make it impossible to delete.
|
||||
function search(autofill) {
|
||||
clearTimeout(debounce);
|
||||
const q = nameInput.value.trim();
|
||||
if (!q) { rows = []; closeList(); return; }
|
||||
debounce = setTimeout(async () => {
|
||||
const mine = ++searchSeq;
|
||||
let r;
|
||||
try {
|
||||
r = await browser.runtime.sendMessage({ type: 'SEARCH_ARTISTS', q });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
// Stale: a later keystroke has its own search coming.
|
||||
if (mine !== searchSeq || r?.error || nameInput.value.trim() !== q) return;
|
||||
rows = r.artists || [];
|
||||
active = -1;
|
||||
const exact = exactArtistMatch(q, rows);
|
||||
if (exact && !choice.artist) {
|
||||
choice.artist = { id: exact.id, name: exact.name };
|
||||
} else if (autofill && document.activeElement === nameInput) {
|
||||
const hit = inlineCompletion(nameInput.value, rows);
|
||||
const caret = nameInput.value.length;
|
||||
if (hit && nameInput.selectionStart === caret) {
|
||||
nameInput.value = nameInput.value + hit.name.slice(caret);
|
||||
nameInput.setSelectionRange(caret, hit.name.length);
|
||||
choice.artist = { id: hit.id, name: hit.name };
|
||||
choice.artistName = hit.name;
|
||||
}
|
||||
}
|
||||
renderList();
|
||||
refresh();
|
||||
}, 150);
|
||||
}
|
||||
|
||||
nameInput.addEventListener('input', (e) => {
|
||||
choice.artistName = nameInput.value;
|
||||
// Any edit un-picks: the artist is whatever the field now says.
|
||||
choice.artist = null;
|
||||
refresh();
|
||||
search(!String(e.inputType || '').startsWith('delete'));
|
||||
});
|
||||
nameInput.addEventListener('focus', () => { if (rows.length) renderList(); });
|
||||
nameInput.addEventListener('blur', () => closeList());
|
||||
|
||||
// Keep Discord's global shortcuts from eating keystrokes meant for us.
|
||||
panel.addEventListener('keydown', (e) => {
|
||||
e.stopPropagation();
|
||||
const count = results.querySelectorAll('.fc-panel__result').length;
|
||||
if (e.target === nameInput && listOpen && count) {
|
||||
if (e.key === 'ArrowDown' || e.key === 'ArrowUp') {
|
||||
e.preventDefault();
|
||||
active = e.key === 'ArrowDown' ? (active + 1) % count : (active <= 0 ? count - 1 : active - 1);
|
||||
renderList();
|
||||
return;
|
||||
}
|
||||
if ((e.key === 'Enter' || e.key === 'Tab') && active >= 0) {
|
||||
e.preventDefault();
|
||||
results.querySelectorAll('.fc-panel__result')[active]
|
||||
.dispatchEvent(new MouseEvent('mousedown', { cancelable: true }));
|
||||
return;
|
||||
}
|
||||
if ((e.key === 'Tab' || e.key === 'Enter') && choice.artist
|
||||
&& nameInput.selectionEnd > nameInput.selectionStart) {
|
||||
// Accept the inline autofill — Enter too, but only to accept: the
|
||||
// add itself takes a second Enter, once the hint names the artist.
|
||||
e.preventDefault();
|
||||
pick(choice.artist);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (e.key === 'Escape') {
|
||||
if (listOpen) closeList();
|
||||
else closePanel();
|
||||
return;
|
||||
}
|
||||
if (e.key === 'Enter' && e.target === nameInput && !addBtn.disabled) addBtn.click();
|
||||
});
|
||||
|
||||
cancelBtn.addEventListener('click', closePanel);
|
||||
addBtn.addEventListener('click', async () => {
|
||||
const req = addRequest(choice);
|
||||
if (!req) return;
|
||||
const authors = posters?.request();
|
||||
if (authors) req.discordAuthors = authors;
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
addBtn.disabled = true;
|
||||
const ok = await add(btn || addBtn, req);
|
||||
if (ok) closePanel();
|
||||
else refresh();
|
||||
});
|
||||
|
||||
document.body.appendChild(panel);
|
||||
refresh();
|
||||
nameInput.focus();
|
||||
posters?.load();
|
||||
nameInput.select();
|
||||
// Search what the field opens with — the server's name, usually — so an
|
||||
// artist it already matches is picked before the operator types anything.
|
||||
if (!choice.artist && nameInput.value.trim()) search(false);
|
||||
}
|
||||
|
||||
function showToast(text, kind) {
|
||||
const t = document.createElement('div');
|
||||
t.className = `fc-toast fc-toast--${kind}`;
|
||||
|
||||
+41
-5
@@ -80,6 +80,17 @@ class FabledCuratorAPI {
|
||||
getCredentials() {
|
||||
return this.request('GET', '/credentials');
|
||||
}
|
||||
// Test the STORED credential against one of the platform's sources — the
|
||||
// same check the web UI's Verify button runs. {valid: true|false|null, reason}.
|
||||
verifyCredential(platform) {
|
||||
return this.request('POST', `/credentials/${encodeURIComponent(platform)}/verify`);
|
||||
}
|
||||
|
||||
// Artist search for the Discord Add panel — the web UI's autocomplete.
|
||||
searchArtists(q, limit = 8) {
|
||||
const qs = new URLSearchParams({ q, limit: String(limit) }).toString();
|
||||
return this.request('GET', `/artists/autocomplete?${qs}`);
|
||||
}
|
||||
|
||||
// FC-3a — sources.
|
||||
listSources() {
|
||||
@@ -90,15 +101,40 @@ class FabledCuratorAPI {
|
||||
}
|
||||
|
||||
// FC-3g — extension-specific.
|
||||
quickAddSource(url) {
|
||||
return this.request('POST', '/extension/quick-add-source', { url });
|
||||
// artistId connects the source to an existing artist, artistName to the
|
||||
// artist of that name (created if new); with neither the server derives the
|
||||
// artist from the URL. A Discord channel always sends one.
|
||||
// usePlatformName: a Patreon source joining an existing artist renames it
|
||||
// to the Patreon display name (Patreon is canon; name only, never the slug).
|
||||
// discordAuthors ([{id, name}], #4488): a Discord source's poster list, set
|
||||
// in the same step.
|
||||
quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false, discordAuthors = null } = {}) {
|
||||
const body = { url };
|
||||
if (discordAuthors) body.discord_authors = discordAuthors;
|
||||
if (artistId != null) body.artist_id = artistId;
|
||||
else if (artistName) body.artist_name = artistName;
|
||||
if (usePlatformName) body.use_platform_name = true;
|
||||
return this.request('POST', '/extension/quick-add-source', body);
|
||||
}
|
||||
probeSource(url) {
|
||||
probeSource(url, { names = false } = {}) {
|
||||
// Read-only existence check. Drives the content-script chip's
|
||||
// color/copy BEFORE the operator clicks Add.
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
// color/copy BEFORE the operator clicks Add. `names` also reads the
|
||||
// creator's display name from the platform — the Add panel asks for it,
|
||||
// the chip doesn't, so a plain page view never costs a platform request.
|
||||
const params = { url };
|
||||
if (names) params.names = '1';
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
return this.request('GET', `/extension/probe?${qs}`);
|
||||
}
|
||||
// #4488: who posts in the Discord channel `url` shows, lately, with the
|
||||
// likely creator marked; and setting the list on the source that follows it.
|
||||
getDiscordPosters(url) {
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
return this.request('GET', `/extension/discord/posters?${qs}`);
|
||||
}
|
||||
setDiscordPosters(url, authors) {
|
||||
return this.request('POST', '/extension/discord/posters', { url, authors });
|
||||
}
|
||||
// Latest published extension version on this instance — drives the in-app
|
||||
// update prompt. Public endpoint (no key needed, but request() sends it
|
||||
// harmlessly). Returns {version, xpi_url, latest_url, sha256}.
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
/**
|
||||
* The content script's decisions, kept apart from its DOM so the specs can
|
||||
* load them (test/chip.spec.js): which state the chip shows, what it says,
|
||||
* and what the Add panel starts out proposing and finally sends.
|
||||
*
|
||||
* `probe` is /api/extension/probe's answer; `platformName` is the display
|
||||
* name (PLATFORMS[key].name), passed in so this file needs no other lib.
|
||||
*/
|
||||
|
||||
function chipState(probe) {
|
||||
if (!probe || probe.error) return 'new';
|
||||
return ({ source_match: 'source-match', artist_match: 'artist-match', new: 'new' })[probe.state] || 'new';
|
||||
}
|
||||
|
||||
function chipLabel(probe, platformName) {
|
||||
if (!probe || probe.error) return '+ Add to FabledCurator';
|
||||
const artist = probe.artist?.name || 'artist';
|
||||
if (probe.platform === 'discord') {
|
||||
const d = probe.discord || {};
|
||||
if (probe.state === 'source_match') {
|
||||
return probe.covered_by_server
|
||||
? `✓ Whole server in FabledCurator · ${artist}`
|
||||
: `✓ In FabledCurator · ${artist}`;
|
||||
}
|
||||
// Every other Discord state opens the panel: the artist is always chosen.
|
||||
return d.channel_id ? `+ Add ${channelLabel(d)} to FabledCurator` : '+ Add server to FabledCurator';
|
||||
}
|
||||
switch (probe.state) {
|
||||
case 'source_match':
|
||||
return `✓ In FabledCurator · ${platformName}`;
|
||||
case 'artist_match':
|
||||
return `+ Add ${platformName} source to ${artist}`;
|
||||
default:
|
||||
return '+ Add to FabledCurator';
|
||||
}
|
||||
}
|
||||
|
||||
/** `#name` when the probe could read it, else a neutral "this channel". */
|
||||
function channelLabel(d) {
|
||||
return d.channel_name ? `#${d.channel_name}` : 'this channel';
|
||||
}
|
||||
|
||||
/** `name` when the probe could read it, else "this server". */
|
||||
function serverLabel(d) {
|
||||
return d.server_name || 'this server';
|
||||
}
|
||||
|
||||
/**
|
||||
* What the Add panel opens with, on any platform.
|
||||
*
|
||||
* Discord: the channel is the default scope when there is one — a server
|
||||
* source walks every channel the token can read, which is rarely what a
|
||||
* single art channel wants. The artist is the probe's suggestion (the owner
|
||||
* of another source on this server), else a new one named after the server.
|
||||
*
|
||||
* Patreon / SubscribeStar: the page URL is the source. The artist is the one
|
||||
* whose slug the URL already names (artist_match), else a new one under the
|
||||
* creator's display name — `probe.display_name`, from the probe the panel
|
||||
* makes with names=1 — falling back to the URL handle, which the server
|
||||
* resolves on its own when it is left untouched (`nameIsHandle`).
|
||||
*/
|
||||
function panelDefaults(probe, pageUrl) {
|
||||
const d = probe?.discord || {};
|
||||
const suggested = probe?.state === 'artist_match' && probe.artist ? probe.artist : null;
|
||||
const discord = probe?.platform === 'discord';
|
||||
const shown = probe?.display_name || null;
|
||||
let artistName = '';
|
||||
if (suggested) artistName = suggested.name;
|
||||
else if (discord) artistName = d.server_name || '';
|
||||
else artistName = shown || probe?.slug || '';
|
||||
return {
|
||||
platform: probe?.platform || null,
|
||||
scope: discord ? (d.channel_id ? 'channel' : 'server') : 'page',
|
||||
pageUrl: pageUrl || null,
|
||||
channelUrl: d.channel_url || null,
|
||||
serverUrl: d.server_url || null,
|
||||
artist: suggested ? { id: suggested.id, name: suggested.name } : null,
|
||||
artistName,
|
||||
nameIsHandle: !discord && !suggested && !shown,
|
||||
handle: probe?.slug || '',
|
||||
displayName: shown,
|
||||
// Patreon is canon: joining an artist known by another name takes the
|
||||
// Patreon name, unless the operator unticks it.
|
||||
adoptPlatformName: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The rename the panel offers, or null: only on Patreon (the canon name),
|
||||
* only when joining an existing artist, only with a name actually read from
|
||||
* Patreon, and only when it differs from what the artist is called now.
|
||||
*/
|
||||
function renameOffer(choice) {
|
||||
if (choice.platform !== 'patreon' || !choice.artist || !choice.displayName) return null;
|
||||
if (choice.artist.name === choice.displayName) return null;
|
||||
return { from: choice.artist.name, to: choice.displayName };
|
||||
}
|
||||
|
||||
/**
|
||||
* The quick-add body for the panel's current choice. A picked artist goes by
|
||||
* id — names can collide once slugified — and a typed name creates (or
|
||||
* finds) that artist. The URL handle left untouched sends no name, so the
|
||||
* server resolves the display name itself. null when there is nothing valid.
|
||||
*/
|
||||
function addRequest(choice) {
|
||||
let url = choice.pageUrl;
|
||||
if (choice.scope === 'server') url = choice.serverUrl;
|
||||
else if (choice.scope === 'channel') url = choice.channelUrl;
|
||||
if (!url) return null;
|
||||
if (choice.artist && choice.artist.id != null && choice.artist.name === choice.artistName) {
|
||||
const req = { url, artistId: choice.artist.id };
|
||||
if (choice.adoptPlatformName && renameOffer(choice)) req.usePlatformName = true;
|
||||
return req;
|
||||
}
|
||||
const name = (choice.artistName || '').trim();
|
||||
if (!name) return null;
|
||||
if (choice.nameIsHandle && name === choice.handle) return { url };
|
||||
return { url, artistName: name };
|
||||
}
|
||||
|
||||
/**
|
||||
* An artist name reduced to what identifies it: lowercase letters and digits
|
||||
* of any script, nothing else — so "Tamada Heijun", "tamada_heijun" and
|
||||
* "TamadaHeijun" are one name. Mirrors the server's autocomplete (#429).
|
||||
*/
|
||||
function squashName(name) {
|
||||
return String(name || '').toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
|
||||
}
|
||||
|
||||
/** The search result that IS the query, spacing aside, else null. */
|
||||
function exactArtistMatch(query, results) {
|
||||
const q = squashName(query);
|
||||
if (!q) return null;
|
||||
return (results || []).find((a) => squashName(a.name) === q) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inline autofill: the first result whose name extends what was typed
|
||||
* (case-insensitive), so the panel can fill in the rest and select it —
|
||||
* typing on overwrites it, Tab or Enter accepts it. null when none does.
|
||||
*/
|
||||
function inlineCompletion(typed, results) {
|
||||
const t = String(typed || '').toLowerCase();
|
||||
if (!t) return null;
|
||||
return (results || []).find((a) => a.name.length > t.length && a.name.toLowerCase().startsWith(t)) || null;
|
||||
}
|
||||
|
||||
// ---- Discord poster picker (#4488) ----
|
||||
// /api/extension/discord/posters lists who posted in the channel lately, with
|
||||
// the creator the server is named for (or owned by) marked `suggested`. The
|
||||
// picker ticks posters; the source keeps their ids, since a username can
|
||||
// change on a whim and an id can't.
|
||||
|
||||
/** "Todding (@todding)" — the display name, then the handle when it differs. */
|
||||
function posterLabel(p) {
|
||||
const shown = p?.global_name || p?.username || p?.id || '';
|
||||
const handle = p?.username && p.username !== shown ? ` (@${p.username})` : '';
|
||||
return `${shown}${handle}`;
|
||||
}
|
||||
|
||||
/** What the picker starts with: the source's current list when it has one,
|
||||
* else the suggested creator(s). A set of ids. */
|
||||
function initialPosterPick(data) {
|
||||
const current = (data?.selected || []).map(String);
|
||||
if (current.length) return new Set(current);
|
||||
return new Set((data?.posters || []).filter((p) => p.suggested).map((p) => String(p.id)));
|
||||
}
|
||||
|
||||
/** The checklist's rows: who posted lately, then anyone already on the
|
||||
* source's list who hasn't — by the name they were picked under — so they
|
||||
* can still be unticked. */
|
||||
function withSelected(data) {
|
||||
const posters = [...(data?.posters || [])];
|
||||
const shown = new Set(posters.map((p) => String(p.id)));
|
||||
const labels = data?.selected_labels || {};
|
||||
for (const id of (data?.selected || []).map(String)) {
|
||||
if (shown.has(id)) continue;
|
||||
posters.push({ id, global_name: labels[id] || null, username: null,
|
||||
messages: 0, images: 0, reasons: ['on the list, not posting lately'] });
|
||||
}
|
||||
return posters;
|
||||
}
|
||||
|
||||
/** The picked posters as the API takes them: [{id, name}], in list order.
|
||||
* An id already on the source but no longer posting lately is kept. */
|
||||
function postersRequest(posters, picked) {
|
||||
const byId = new Map((posters || []).map((p) => [String(p.id), p]));
|
||||
return [...picked].map((id) => ({
|
||||
id,
|
||||
name: byId.has(id) ? (byId.get(id).global_name || byId.get(id).username || id) : null,
|
||||
}));
|
||||
}
|
||||
@@ -57,7 +57,8 @@ const PLATFORMS = {
|
||||
domains: ['.discord.com', 'discord.com'],
|
||||
authType: 'token',
|
||||
color: '#5865F2',
|
||||
urlPattern: /^https?:\/\/(www\.)?discord\.com/,
|
||||
// ptb/canary are Discord's beta clients — same channels, same token.
|
||||
urlPattern: /^https?:\/\/((www|ptb|canary)\.)?discord\.com/,
|
||||
note: 'Open Discord in browser to capture token',
|
||||
},
|
||||
};
|
||||
@@ -80,8 +81,23 @@ const PLATFORM_ARTIST_PATTERNS = {
|
||||
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i,
|
||||
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
||||
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
||||
// A Discord URL names a server or channel, not a creator: the backend's slug
|
||||
// is `<server>[/<channel>]` and the Add panel asks which artist it belongs
|
||||
// to. A message jump link still names its channel; DMs (@me) and thread
|
||||
// links don't match. Mirrors extension_service._PLATFORM_PATTERNS.
|
||||
discord: /^https?:\/\/(?:www\.|ptb\.|canary\.)?discord\.com\/channels\/\d+(?:\/\d+)?(?:\/\d+)?\/?(?:[?#].*)?$/i,
|
||||
};
|
||||
|
||||
/**
|
||||
* `{serverId, channelId}` from a Discord channel/server URL the artist
|
||||
* pattern accepts, else null. channelId is null for a whole-server URL.
|
||||
*/
|
||||
function parseDiscordUrl(url) {
|
||||
if (!PLATFORM_ARTIST_PATTERNS.discord.test(url || '')) return null;
|
||||
const m = /\/channels\/(\d+)(?:\/(\d+))?/.exec(url);
|
||||
return m ? { serverId: m[1], channelId: m[2] || null } : null;
|
||||
}
|
||||
|
||||
function getPlatformFromUrl(url) {
|
||||
for (const [key, platform] of Object.entries(PLATFORMS)) {
|
||||
if (platform.urlPattern.test(url)) return key;
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* The popup's wording, kept apart from its DOM so the specs can load it
|
||||
* (test/popup-format.spec.js). Classic script, like the rest of lib/.
|
||||
*/
|
||||
|
||||
/**
|
||||
* One line of state for a source row, from /api/sources' fields, with the
|
||||
* status class the popup colours it by ('ready' | 'error' | 'no-cookies' for
|
||||
* a warning | '' for plain). The most actionable fact wins: an error before
|
||||
* a running backfill, a backfill before the last-checked time.
|
||||
*/
|
||||
function sourceStatus(src, now = Date.now()) {
|
||||
if (!src.enabled) return { text: 'Disabled', kind: '' };
|
||||
if (src.last_error) {
|
||||
const first = String(src.last_error).split('\n')[0].trim();
|
||||
const short = first.length > 90 ? `${first.slice(0, 89)}…` : first;
|
||||
return { text: `Error — ${short}`, kind: 'error' };
|
||||
}
|
||||
if (src.backfill_state === 'running') {
|
||||
const n = src.backfill_chunks || 0;
|
||||
return { text: n ? `Backfilling — ${n} chunk${n === 1 ? '' : 's'} done` : 'Backfill queued', kind: 'ready' };
|
||||
}
|
||||
if (src.backfill_state === 'stalled') return { text: 'Backfill stalled', kind: 'no-cookies' };
|
||||
if (!src.last_checked_at) return { text: 'Not checked yet', kind: '' };
|
||||
return { text: `Checked ${relativeTime(src.last_checked_at, now)}`, kind: '' };
|
||||
}
|
||||
|
||||
// Same buckets and wording as the web UI's canonical formatRelative
|
||||
// (frontend/src/utils/date.js, snippet #3959) — the extension can't import it (classic
|
||||
// scripts, separate package), so it mirrors it: "42s ago", "5m ago", "3h ago",
|
||||
// "2d ago", and "Never" for a missing or unreadable time.
|
||||
function relativeTime(iso, now = Date.now()) {
|
||||
const t = iso ? Date.parse(iso) : NaN;
|
||||
if (Number.isNaN(t)) return 'Never';
|
||||
const abs = Math.abs(now - t) / 1000;
|
||||
let body;
|
||||
if (abs < 60) body = `${Math.floor(abs)}s`;
|
||||
else if (abs < 3600) body = `${Math.floor(abs / 60)}m`;
|
||||
else if (abs < 86400) body = `${Math.floor(abs / 3600)}h`;
|
||||
else body = `${Math.floor(abs / 86400)}d`;
|
||||
return `${body} ago`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The popup message after a Discord token export, from FC's verify of the
|
||||
* stored token: {text, kind} with kind 'success' | 'warning' | 'error'.
|
||||
* valid=null is "FC couldn't test it" (no Discord source yet, or a network
|
||||
* hiccup) — a warning with FC's reason, never a failure.
|
||||
*/
|
||||
function tokenExportMessage(verify) {
|
||||
if (!verify) return { text: 'Discord: token exported', kind: 'success' };
|
||||
if (verify.valid === true) return { text: `Discord: token exported and verified ✓ — ${verify.reason}`, kind: 'success' };
|
||||
if (verify.valid === false) return { text: `Discord: token exported, but FC's check failed — ${verify.reason}`, kind: 'error' };
|
||||
return { text: `Discord: token exported (not verified — ${verify.reason})`, kind: 'warning' };
|
||||
}
|
||||
@@ -56,9 +56,10 @@
|
||||
"*://*.patreon.com/*",
|
||||
"*://*.subscribestar.com/*",
|
||||
"*://*.subscribestar.adult/*",
|
||||
"*://*.hentai-foundry.com/*"
|
||||
"*://*.hentai-foundry.com/*",
|
||||
"*://*.discord.com/*"
|
||||
],
|
||||
"js": ["lib/platforms.js", "content/content-script.js"],
|
||||
"js": ["lib/platforms.js", "lib/chip.js", "content/content-script.js"],
|
||||
"css": ["content/content-script.css"],
|
||||
"run_at": "document_idle"
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
</section>
|
||||
|
||||
<script src="../lib/platforms.js"></script>
|
||||
<script src="../lib/popup-format.js"></script>
|
||||
<script src="popup.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -76,7 +76,7 @@ function updateConnectionDot(connected) {
|
||||
async function checkForUpdate() {
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_UPDATE' });
|
||||
if (r && r.updateAvailable && r.xpiUrl) showUpdateBanner(r);
|
||||
if (r && r.updateAvailable && r.installPageUrl) showUpdateBanner(r);
|
||||
} catch { /* non-fatal */ }
|
||||
}
|
||||
|
||||
@@ -86,10 +86,14 @@ function showUpdateBanner(r) {
|
||||
// exactly as it did before the field existed.
|
||||
const channel = r.channel ? ` (${r.channel})` : '';
|
||||
document.getElementById('update-text').textContent =
|
||||
`Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion})`;
|
||||
// Opening the signed XPI triggers Firefox's native install prompt.
|
||||
`Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion}). ` +
|
||||
'Opens FabledCurator — click “Install Firefox extension” there.';
|
||||
// Opens FC's install card rather than the XPI: Firefox only installs an
|
||||
// add-on from a user click on a web page, never from a tab an extension
|
||||
// opened on the .xpi itself.
|
||||
document.getElementById('update-btn').addEventListener('click', () => {
|
||||
browser.tabs.create({ url: r.xpiUrl });
|
||||
browser.tabs.create({ url: r.installPageUrl });
|
||||
window.close();
|
||||
});
|
||||
document.getElementById('update-banner').classList.remove('hidden');
|
||||
}
|
||||
@@ -159,7 +163,11 @@ async function exportPlatformCookies(key, card) {
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'EXPORT_COOKIES', platform: key });
|
||||
if (r.error) showError(r.error);
|
||||
else {
|
||||
else if (key === 'discord') {
|
||||
const m = tokenExportMessage(r.verify);
|
||||
showStatusMessage(m.text, m.kind);
|
||||
await loadPlatformStatus();
|
||||
} else {
|
||||
const n = r.cookieCount ?? null;
|
||||
const verifiedSuffix = r.verified ? ' (verified ✓)' : '';
|
||||
const msg = n !== null
|
||||
@@ -205,7 +213,10 @@ async function loadSources() {
|
||||
c.appendChild(mutedNote('No sources yet.'));
|
||||
return;
|
||||
}
|
||||
for (const src of r.sources) c.appendChild(createSourceRow(src));
|
||||
// Grouped by artist so a creator's Patreon and Discord sit together.
|
||||
const sorted = [...r.sources].sort((a, b) =>
|
||||
(a.artist_name || '').localeCompare(b.artist_name || '') || a.id - b.id);
|
||||
for (const src of sorted) c.appendChild(createSourceRow(src));
|
||||
}
|
||||
|
||||
function createSourceRow(src) {
|
||||
@@ -215,11 +226,16 @@ function createSourceRow(src) {
|
||||
info.className = 'info';
|
||||
const name = document.createElement('div');
|
||||
name.className = 'name';
|
||||
name.textContent = `${src.platform} · #${src.id}`;
|
||||
const platformName = PLATFORMS[src.platform]?.name || src.platform;
|
||||
name.textContent = `${src.artist_name || `Source #${src.id}`} · ${platformName}`;
|
||||
const state = sourceStatus(src);
|
||||
const st = document.createElement('div');
|
||||
st.className = `status ${state.kind}`;
|
||||
st.textContent = state.text;
|
||||
const url = document.createElement('div');
|
||||
url.className = 'url';
|
||||
url.textContent = src.url;
|
||||
info.appendChild(name); info.appendChild(url);
|
||||
info.appendChild(name); info.appendChild(st); info.appendChild(url);
|
||||
const play = document.createElement('button');
|
||||
play.className = 'play';
|
||||
play.textContent = '▶';
|
||||
@@ -229,7 +245,7 @@ function createSourceRow(src) {
|
||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_SOURCE', sourceId: src.id });
|
||||
play.disabled = false;
|
||||
if (r.error) showError(r.error);
|
||||
else showSuccess(`Triggered check for source #${src.id}`);
|
||||
else showSuccess(`Check queued for ${src.artist_name || `source #${src.id}`} (${platformName})`);
|
||||
});
|
||||
row.appendChild(info); row.appendChild(play);
|
||||
return row;
|
||||
|
||||
@@ -134,5 +134,42 @@
|
||||
{ "url": "https://www.hentai-foundry.com/pictures/popular", "why": "gallery listing, not a user" },
|
||||
{ "url": "https://www.hentai-foundry.com/", "why": "site root" }
|
||||
]
|
||||
},
|
||||
|
||||
"discord": {
|
||||
"match": [
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111/222222222222222222",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "a channel: the slug is server/channel -- a Discord URL names a place, not a creator, so the artist is chosen in the Add panel"
|
||||
},
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111",
|
||||
"slug": "111111111111111111",
|
||||
"why": "a whole server"
|
||||
},
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111/222222222222222222/333333333333333333",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "a message jump link still names its channel"
|
||||
},
|
||||
{
|
||||
"url": "https://ptb.discord.com/channels/111111111111111111/222222222222222222",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "the ptb and canary clients serve the same channels"
|
||||
},
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111/222222222222222222/",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "trailing slash is tolerated"
|
||||
}
|
||||
],
|
||||
"no_match": [
|
||||
{ "url": "https://discord.com/channels/@me", "why": "the DM list is not a source" },
|
||||
{ "url": "https://discord.com/channels/@me/222222222222222222", "why": "a DM is not a source" },
|
||||
{ "url": "https://discord.com/app", "why": "the app shell, no server open" },
|
||||
{ "url": "https://discord.com/channels/111111111111111111/222222222222222222/threads/444444444444444444", "why": "thread links are left to the manual Add form" },
|
||||
{ "url": "https://discord.com/servers/111111111111111111", "why": "a server-discovery page, not a channel" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { loadLib } from './helpers/loadLib.js'
|
||||
|
||||
const { chipState, chipLabel, panelDefaults, addRequest, renameOffer } = loadLib('chip.js', [
|
||||
'chipState',
|
||||
'chipLabel',
|
||||
'panelDefaults',
|
||||
'addRequest',
|
||||
'renameOffer'
|
||||
])
|
||||
|
||||
const discord = (extra = {}) => ({
|
||||
platform: 'discord',
|
||||
slug: '111/222',
|
||||
discord: {
|
||||
server_id: '111',
|
||||
channel_id: '222',
|
||||
server_name: 'Studio',
|
||||
channel_name: 'drops',
|
||||
server_url: 'https://discord.com/channels/111',
|
||||
channel_url: 'https://discord.com/channels/111/222'
|
||||
},
|
||||
...extra
|
||||
})
|
||||
|
||||
describe('chip state and label', () => {
|
||||
it('keeps the one-click wording on creator platforms', () => {
|
||||
expect(chipLabel({ state: 'new', platform: 'patreon' }, 'Patreon')).toBe('+ Add to FabledCurator')
|
||||
expect(
|
||||
chipLabel({ state: 'artist_match', platform: 'patreon', artist: { name: 'Atole' } }, 'Patreon')
|
||||
).toBe('+ Add Patreon source to Atole')
|
||||
expect(chipLabel({ state: 'source_match', platform: 'patreon' }, 'Patreon')).toBe(
|
||||
'✓ In FabledCurator · Patreon'
|
||||
)
|
||||
})
|
||||
|
||||
it('offers the channel by name on Discord, whatever the suggestion', () => {
|
||||
expect(chipLabel(discord({ state: 'new' }), 'Discord')).toBe('+ Add #drops to FabledCurator')
|
||||
expect(chipLabel(discord({ state: 'artist_match', artist: { name: 'A' } }), 'Discord')).toBe(
|
||||
'+ Add #drops to FabledCurator'
|
||||
)
|
||||
})
|
||||
|
||||
it('says "this channel" when the token could not read the name', () => {
|
||||
const p = discord({ state: 'new' })
|
||||
p.discord.channel_name = null
|
||||
expect(chipLabel(p, 'Discord')).toBe('+ Add this channel to FabledCurator')
|
||||
})
|
||||
|
||||
it('says whose source a Discord channel already is, and when the server covers it', () => {
|
||||
const artist = { name: 'Tamada', slug: 'tamada' }
|
||||
expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: false }), 'Discord')).toBe(
|
||||
'✓ In FabledCurator · Tamada'
|
||||
)
|
||||
expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: true }), 'Discord')).toBe(
|
||||
'✓ Whole server in FabledCurator · Tamada'
|
||||
)
|
||||
})
|
||||
|
||||
it('falls back to the generic add when the probe failed', () => {
|
||||
expect(chipState({ error: 'x' })).toBe('new')
|
||||
expect(chipLabel({ error: 'x' }, '')).toBe('+ Add to FabledCurator')
|
||||
expect(chipState({ state: 'source_match' })).toBe('source-match')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Add panel on Discord', () => {
|
||||
it('opens on the channel with the suggested artist preselected', () => {
|
||||
const d = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
expect(d.scope).toBe('channel')
|
||||
expect(d.artist).toEqual({ id: 7, name: 'Tamada' })
|
||||
expect(d.artistName).toBe('Tamada')
|
||||
})
|
||||
|
||||
it('proposes a new artist named after the server when nothing is suggested', () => {
|
||||
const d = panelDefaults(discord({ state: 'new' }))
|
||||
expect(d.artist).toBe(null)
|
||||
expect(d.artistName).toBe('Studio')
|
||||
})
|
||||
|
||||
it('sends a picked artist by id', () => {
|
||||
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
expect(addRequest(choice)).toEqual({
|
||||
url: 'https://discord.com/channels/111/222',
|
||||
artistId: 7
|
||||
})
|
||||
})
|
||||
|
||||
it('sends a typed name once the picked artist has been edited away', () => {
|
||||
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
choice.artistName = 'Tamada Alt'
|
||||
expect(addRequest(choice)).toEqual({
|
||||
url: 'https://discord.com/channels/111/222',
|
||||
artistName: 'Tamada Alt'
|
||||
})
|
||||
})
|
||||
|
||||
it('adds the whole server when the operator picks it', () => {
|
||||
const choice = panelDefaults(discord({ state: 'new' }))
|
||||
choice.scope = 'server'
|
||||
expect(addRequest(choice).url).toBe('https://discord.com/channels/111')
|
||||
})
|
||||
|
||||
it('has nothing to send without an artist', () => {
|
||||
const choice = panelDefaults(discord({ state: 'new' }))
|
||||
choice.artistName = ' '
|
||||
expect(addRequest(choice)).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
const { squashName, exactArtistMatch, inlineCompletion } = loadLib('chip.js', [
|
||||
'squashName',
|
||||
'exactArtistMatch',
|
||||
'inlineCompletion'
|
||||
])
|
||||
|
||||
describe('artist matching for the Add panel', () => {
|
||||
const results = [
|
||||
{ id: 1, name: 'TamadaHeijun' },
|
||||
{ id: 2, name: 'Tamago' },
|
||||
{ id: 3, name: 'Sabu Art' }
|
||||
]
|
||||
|
||||
it('treats spacing, case and punctuation as the same name', () => {
|
||||
expect(squashName('Tamada Heijun')).toBe('tamadaheijun')
|
||||
expect(squashName('sabu_art!')).toBe('sabuart')
|
||||
expect(squashName('玉田 平順')).toBe('玉田平順')
|
||||
})
|
||||
|
||||
it('finds the result that is the query, spacing aside', () => {
|
||||
expect(exactArtistMatch('tamada heijun', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||
expect(exactArtistMatch('Tama', results)).toBe(null)
|
||||
expect(exactArtistMatch(' ', results)).toBe(null)
|
||||
})
|
||||
|
||||
it('autofills the first name that extends what was typed', () => {
|
||||
expect(inlineCompletion('tamad', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||
expect(inlineCompletion('Sab', results).name).toBe('Sabu Art')
|
||||
// Nothing to add once the name is complete, or when nothing extends it.
|
||||
expect(inlineCompletion('Sabu Art', results)).toBe(null)
|
||||
expect(inlineCompletion('heijun', results)).toBe(null)
|
||||
expect(inlineCompletion('', results)).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Add panel on Patreon and SubscribeStar', () => {
|
||||
const PAGE = 'https://www.patreon.com/cw/tamadaheijun'
|
||||
const patreon = (extra = {}) => ({ platform: 'patreon', slug: 'tamadaheijun', ...extra })
|
||||
|
||||
it('opens on the Patreon display name as a new artist', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
expect(c.scope).toBe('page')
|
||||
expect(c.artistName).toBe('Tamada Heijun')
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Tamada Heijun' })
|
||||
})
|
||||
|
||||
it('leaves an untouched URL handle to the server to resolve', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||
expect(c.artistName).toBe('tamadaheijun')
|
||||
expect(addRequest(c)).toEqual({ url: PAGE })
|
||||
c.artistName = 'Someone Else'
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Someone Else' })
|
||||
})
|
||||
|
||||
it('offers the Patreon name when joining an artist known by another name', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
c.artist = { id: 4, name: 'tamada' }
|
||||
c.artistName = 'tamada'
|
||||
expect(renameOffer(c)).toEqual({ from: 'tamada', to: 'Tamada Heijun' })
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4, usePlatformName: true })
|
||||
c.adoptPlatformName = false
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4 })
|
||||
})
|
||||
|
||||
it('offers no rename when the names agree or the name was not read', () => {
|
||||
const same = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
same.artist = { id: 4, name: 'Tamada Heijun' }
|
||||
expect(renameOffer(same)).toBe(null)
|
||||
const unread = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||
unread.artist = { id: 4, name: 'tamada' }
|
||||
expect(renameOffer(unread)).toBe(null)
|
||||
})
|
||||
|
||||
it('never renames from SubscribeStar or Discord — Patreon is the canon', () => {
|
||||
const ss = panelDefaults(
|
||||
{ platform: 'subscribestar', slug: 'tamada', state: 'new', display_name: 'SS Tamada' },
|
||||
'https://subscribestar.adult/tamada'
|
||||
)
|
||||
ss.artist = { id: 4, name: 'Tamada Heijun' }
|
||||
ss.artistName = 'Tamada Heijun'
|
||||
expect(renameOffer(ss)).toBe(null)
|
||||
expect(addRequest(ss)).toEqual({ url: 'https://subscribestar.adult/tamada', artistId: 4 })
|
||||
})
|
||||
|
||||
it('preselects the artist the URL already names', () => {
|
||||
const c = panelDefaults(patreon({ state: 'artist_match', artist: { id: 9, name: 'Tamada' } }), PAGE)
|
||||
expect(c.artist).toEqual({ id: 9, name: 'Tamada' })
|
||||
expect(c.artistName).toBe('Tamada')
|
||||
})
|
||||
})
|
||||
|
||||
const { posterLabel, initialPosterPick, postersRequest, withSelected } = loadLib('chip.js', [
|
||||
'posterLabel', 'initialPosterPick', 'postersRequest', 'withSelected',
|
||||
])
|
||||
|
||||
describe('the poster checklist rows', () => {
|
||||
it('adds someone on the list who has not posted lately, by their saved name', () => {
|
||||
const rows = withSelected({
|
||||
posters: [{ id: '7', username: 'todding', global_name: 'Todding' }],
|
||||
selected: ['7', '42'],
|
||||
selected_labels: { 42: 'Old Name' },
|
||||
})
|
||||
expect(rows.map((r) => r.id)).toEqual(['7', '42'])
|
||||
expect(posterLabel(rows[1])).toBe('Old Name')
|
||||
expect(rows[1].reasons).toEqual(['on the list, not posting lately'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('the Discord poster picker (#4488)', () => {
|
||||
const posters = [
|
||||
{ id: '7', username: 'todding', global_name: 'Todding', suggested: true },
|
||||
{ id: '8', username: 'jakeboii', global_name: 'Jake Boii', suggested: false },
|
||||
{ id: '9', username: 'plain', global_name: null, suggested: false },
|
||||
]
|
||||
|
||||
it('labels a poster by display name, then handle', () => {
|
||||
expect(posterLabel(posters[0])).toBe('Todding (@todding)')
|
||||
expect(posterLabel(posters[2])).toBe('plain')
|
||||
})
|
||||
|
||||
it('starts from the source list when it has one, else the suggestion', () => {
|
||||
expect([...initialPosterPick({ posters, selected: [] })]).toEqual(['7'])
|
||||
expect([...initialPosterPick({ posters, selected: ['8'] })]).toEqual(['8'])
|
||||
expect([...initialPosterPick({ posters: [] })]).toEqual([])
|
||||
})
|
||||
|
||||
it('sends ids with the names they were picked under, keeping unknown ids', () => {
|
||||
expect(postersRequest(posters, new Set(['7', '42']))).toEqual([
|
||||
{ id: '7', name: 'Todding' },
|
||||
{ id: '42', name: null },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -7,10 +7,14 @@ import { loadLib } from './helpers/loadLib.js'
|
||||
const EXT_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const manifest = JSON.parse(readFileSync(path.join(EXT_DIR, 'manifest.json'), 'utf8'))
|
||||
|
||||
const { getPlatformFromUrl, isArtistPage, PLATFORMS, PLATFORM_ARTIST_PATTERNS } = loadLib(
|
||||
'platforms.js',
|
||||
['getPlatformFromUrl', 'isArtistPage', 'PLATFORMS', 'PLATFORM_ARTIST_PATTERNS']
|
||||
)
|
||||
const { getPlatformFromUrl, isArtistPage, parseDiscordUrl, PLATFORMS, PLATFORM_ARTIST_PATTERNS } =
|
||||
loadLib('platforms.js', [
|
||||
'getPlatformFromUrl',
|
||||
'isArtistPage',
|
||||
'parseDiscordUrl',
|
||||
'PLATFORMS',
|
||||
'PLATFORM_ARTIST_PATTERNS'
|
||||
])
|
||||
|
||||
describe('getPlatformFromUrl', () => {
|
||||
it('identifies each platform from a domain URL', () => {
|
||||
@@ -88,8 +92,11 @@ describe('isArtistPage', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('returns false for a platform with no artist pattern (discord)', () => {
|
||||
it('matches Discord server and channel pages, not DMs (milestone 429)', () => {
|
||||
expect(isArtistPage('https://discord.com/channels/111/222', 'discord')).toBe(true)
|
||||
expect(isArtistPage('https://discord.com/channels/111', 'discord')).toBe(true)
|
||||
expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false)
|
||||
expect(isArtistPage('https://discord.com/channels/@me/222', 'discord')).toBe(false)
|
||||
})
|
||||
|
||||
it('returns false for an unknown platform key', () => {
|
||||
@@ -100,8 +107,8 @@ describe('isArtistPage', () => {
|
||||
describe('platform table integrity', () => {
|
||||
it('gives every artist pattern a corresponding platform entry', () => {
|
||||
// A pattern keyed to a platform that no longer exists is dead code that
|
||||
// silently never fires; the reverse (a platform with no pattern) is the
|
||||
// legitimate discord case, so only this direction is an error.
|
||||
// silently never fires; the reverse (a platform with no pattern) would be
|
||||
// a platform the button never offers, a product choice rather than an error.
|
||||
for (const key of Object.keys(PLATFORM_ARTIST_PATTERNS)) {
|
||||
expect(Object.keys(PLATFORMS)).toContain(key)
|
||||
}
|
||||
@@ -125,7 +132,8 @@ describe('platform table integrity', () => {
|
||||
const samples = {
|
||||
patreon: 'https://www.patreon.com/cw/Atole',
|
||||
subscribestar: 'https://subscribestar.adult/someone',
|
||||
hentaifoundry: 'https://www.hentai-foundry.com/user/someone'
|
||||
hentaifoundry: 'https://www.hentai-foundry.com/user/someone',
|
||||
discord: 'https://ptb.discord.com/channels/111/222'
|
||||
}
|
||||
for (const [key, url] of Object.entries(samples)) {
|
||||
expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true)
|
||||
@@ -152,7 +160,7 @@ describe('manifest.json agrees with the platform table', () => {
|
||||
)
|
||||
expect(owner, `no platform claims content-script match "${m}"`).toBeTruthy()
|
||||
// The content script exists to draw the Add-as-source button, so a
|
||||
// platform with no artist pattern (discord) has no business here.
|
||||
// platform with no artist pattern has no business here.
|
||||
expect(
|
||||
PLATFORM_ARTIST_PATTERNS[owner[0]],
|
||||
`"${m}" injects for ${owner[0]}, which has no artist pattern`
|
||||
@@ -232,9 +240,8 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => {
|
||||
|
||||
it('has samples for every platform that has an artist pattern', () => {
|
||||
// The guard's own coverage check: without it, deleting a platform's
|
||||
// samples would make this block pass by testing less. Discord is
|
||||
// deliberately in neither — it is channel-based, with no creator page to
|
||||
// put a button on, so it has no artist pattern on either side.
|
||||
// samples would make this block pass by testing less. Discord joined at
|
||||
// milestone 429 — its slug is server/channel and the artist is chosen.
|
||||
expect(Object.keys(samples).sort()).toEqual(Object.keys(PLATFORM_ARTIST_PATTERNS).sort())
|
||||
})
|
||||
|
||||
@@ -247,3 +254,26 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseDiscordUrl', () => {
|
||||
it('reads the server and channel ids the Add panel offers', () => {
|
||||
expect(parseDiscordUrl('https://discord.com/channels/111/222')).toEqual({
|
||||
serverId: '111',
|
||||
channelId: '222'
|
||||
})
|
||||
expect(parseDiscordUrl('https://discord.com/channels/111/222/333')).toEqual({
|
||||
serverId: '111',
|
||||
channelId: '222'
|
||||
})
|
||||
expect(parseDiscordUrl('https://discord.com/channels/111')).toEqual({
|
||||
serverId: '111',
|
||||
channelId: null
|
||||
})
|
||||
})
|
||||
|
||||
it('returns null for anything the artist pattern rejects', () => {
|
||||
expect(parseDiscordUrl('https://discord.com/channels/@me/222')).toBe(null)
|
||||
expect(parseDiscordUrl('https://discord.com/app')).toBe(null)
|
||||
expect(parseDiscordUrl('')).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { loadLib } from './helpers/loadLib.js'
|
||||
|
||||
const { sourceStatus, relativeTime, tokenExportMessage } = loadLib('popup-format.js', [
|
||||
'sourceStatus',
|
||||
'relativeTime',
|
||||
'tokenExportMessage'
|
||||
])
|
||||
|
||||
const NOW = Date.parse('2026-09-25T12:00:00Z')
|
||||
const src = (extra = {}) => ({ enabled: true, last_error: null, backfill_state: null, ...extra })
|
||||
|
||||
describe('sourceStatus', () => {
|
||||
it('puts an error first, trimmed to its first line', () => {
|
||||
const s = sourceStatus(src({ last_error: 'Discord rejected the token\nstack…', backfill_state: 'running' }), NOW)
|
||||
expect(s).toEqual({ text: 'Error — Discord rejected the token', kind: 'error' })
|
||||
})
|
||||
|
||||
it('shows a running backfill and its progress', () => {
|
||||
expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 0 }), NOW).text).toBe('Backfill queued')
|
||||
expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 3 }), NOW).text).toBe(
|
||||
'Backfilling — 3 chunks done'
|
||||
)
|
||||
})
|
||||
|
||||
it('says when a source was last checked, or that it never was', () => {
|
||||
expect(sourceStatus(src({ last_checked_at: '2026-09-25T11:55:00Z' }), NOW).text).toBe('Checked 5m ago')
|
||||
expect(sourceStatus(src({ last_checked_at: null }), NOW).text).toBe('Not checked yet')
|
||||
})
|
||||
|
||||
it('shows a disabled source as disabled, whatever else it carries', () => {
|
||||
expect(sourceStatus(src({ enabled: false, last_error: 'x' }), NOW).text).toBe('Disabled')
|
||||
})
|
||||
})
|
||||
|
||||
describe('relativeTime', () => {
|
||||
it('uses the web UI formatRelative buckets', () => {
|
||||
expect(relativeTime('2026-09-25T11:59:18Z', NOW)).toBe('42s ago')
|
||||
expect(relativeTime('2026-09-25T09:00:00Z', NOW)).toBe('3h ago')
|
||||
expect(relativeTime('2026-09-23T12:00:00Z', NOW)).toBe('2d ago')
|
||||
expect(relativeTime('garbage', NOW)).toBe('Never')
|
||||
})
|
||||
})
|
||||
|
||||
describe('tokenExportMessage', () => {
|
||||
it('distinguishes verified, rejected and untested tokens', () => {
|
||||
expect(tokenExportMessage({ valid: true, reason: 'Token valid (me)' }).kind).toBe('success')
|
||||
expect(tokenExportMessage({ valid: false, reason: 'Discord rejected the token' }).kind).toBe('error')
|
||||
const untested = tokenExportMessage({ valid: null, reason: 'No enabled source' })
|
||||
expect(untested.kind).toBe('warning')
|
||||
expect(untested.text).toContain('No enabled source')
|
||||
})
|
||||
})
|
||||
@@ -153,6 +153,14 @@ const health = computed(() => {
|
||||
label: (worst?.detail || 'A part has stopped') + suffix,
|
||||
}
|
||||
}
|
||||
// Running but slow — a GPU agent that fell back to the CPU (#4410). Worth
|
||||
// the amber dot: nothing else anywhere says so.
|
||||
if (overall === 'degraded') {
|
||||
return {
|
||||
icon: 'mdi-speedometer-slow', color: 'warning',
|
||||
label: (worst?.detail || 'A part is running degraded') + suffix,
|
||||
}
|
||||
}
|
||||
if (overall === 'stale') {
|
||||
return {
|
||||
icon: 'mdi-alert', color: 'warning',
|
||||
|
||||
@@ -103,9 +103,9 @@ const stats = computed(() => {
|
||||
padding: 0.5rem 1rem;
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
rgba(20, 23, 26, 0.92) 0%,
|
||||
rgba(20, 23, 26, 0.65) 60%,
|
||||
rgba(20, 23, 26, 0) 100%
|
||||
rgba(var(--fc-chrome-rgb), 0.92) 0%,
|
||||
rgba(var(--fc-chrome-rgb), 0.65) 60%,
|
||||
rgba(var(--fc-chrome-rgb), 0) 100%
|
||||
);
|
||||
backdrop-filter: blur(2px);
|
||||
-webkit-backdrop-filter: blur(2px);
|
||||
|
||||
@@ -101,7 +101,7 @@ import MaintenanceTile from '../common/MaintenanceTile.vue'
|
||||
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||
|
||||
const store = useCleanupStore()
|
||||
const threshold = ref(0.95)
|
||||
const threshold = ref(0.995)
|
||||
const tolerance = ref(30)
|
||||
const audit = ref(null)
|
||||
const busy = ref(false)
|
||||
|
||||
@@ -329,7 +329,7 @@ function pushFilter(mutate) {
|
||||
opaque than the bar/nav so the controls stay legible. */
|
||||
.fc-filterbar-wrap :deep(.v-field),
|
||||
.fc-filterbar-wrap :deep(.v-btn-group) {
|
||||
background-color: rgba(20, 23, 26, 0.72);
|
||||
background-color: rgba(var(--v-theme-background), 0.72);
|
||||
}
|
||||
/* Media toggle (All / Images / Videos) as ONE cohesive segmented control.
|
||||
FC's global VBtn { rounded: 'pill' } default made Vuetify 4 pill-round each
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
<!-- System-tag auto-applies (chrome hides / process WIP tags) that ALSO looked
|
||||
like real content — surfaced PROACTIVELY atop the gallery whenever there's
|
||||
something to review (NOT gated on the Show-hidden toggle, so misfires can't
|
||||
go unnoticed), most-concerning first, with keep / remove (#141, #1464).
|
||||
go unnoticed), most-concerning first (#141, #1464). Each card asks one
|
||||
question — is this image a <tag>? — and its buttons answer it (#4424).
|
||||
Renders nothing when there's nothing to review. -->
|
||||
<section v-if="items.length" class="fc-review" aria-label="Auto-tagged images to review">
|
||||
<div class="fc-review__head">
|
||||
<v-icon size="18" color="warning">mdi-alert-outline</v-icon>
|
||||
<span class="fc-review__title">
|
||||
{{ items.length }} auto-tagged {{ items.length === 1 ? 'image' : 'images' }}
|
||||
may be real content — review
|
||||
{{ items.length }} {{ items.length === 1 ? 'auto-tag' : 'auto-tags' }} to check
|
||||
</span>
|
||||
</div>
|
||||
<div class="fc-review__cards">
|
||||
@@ -21,22 +21,23 @@
|
||||
class="fc-review-card__thumb" loading="lazy"
|
||||
>
|
||||
<div class="fc-review-card__body">
|
||||
<div class="fc-review-card__question">{{ question(it) }}</div>
|
||||
<div
|
||||
class="fc-review-card__conflict"
|
||||
:title="`Scored ${Math.round(it.conflict_score * 100)}% on “${it.conflict_name || 'a content tag'}”`"
|
||||
class="fc-review-card__reason"
|
||||
:title="reasonTitle(it)"
|
||||
>
|
||||
also looks like <strong>{{ it.conflict_name || 'content' }}</strong>
|
||||
also {{ Math.round(it.conflict_score * 100) }}%
|
||||
<strong>{{ it.conflict_name || 'content' }}</strong>
|
||||
</div>
|
||||
<div class="fc-review-card__tag">{{ tagLine(it) }}</div>
|
||||
<div class="fc-review-card__acts">
|
||||
<button
|
||||
type="button" class="fc-review-btn fc-review-btn--keep"
|
||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')"
|
||||
>{{ keepLabel(it) }}</button>
|
||||
>Is {{ withArticle(it) }}</button>
|
||||
<button
|
||||
type="button" class="fc-review-btn fc-review-btn--unhide"
|
||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')"
|
||||
>{{ removeLabel(it) }}</button>
|
||||
>Is not {{ withArticle(it) }}</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -55,11 +56,18 @@ const items = ref([])
|
||||
const busy = ref([])
|
||||
|
||||
function keyOf(it) { return `${it.image_id}:${it.tag_id}` }
|
||||
// Chrome flags hide the image (keep-hidden / un-hide); process flags leave it
|
||||
// visible and just tagged (keep-tag / remove-tag). Same endpoints, different words.
|
||||
function tagLine(it) { return (it.mode === 'process' ? 'auto-tagged ' : 'hidden as ') + it.tag_name }
|
||||
function keepLabel(it) { return it.mode === 'process' ? 'Keep tag' : 'Keep hidden' }
|
||||
function removeLabel(it) { return it.mode === 'process' ? 'Remove tag' : 'Un-hide' }
|
||||
// The card asks whether the image IS the auto-applied system tag, and the buttons
|
||||
// answer that (operator, #4424: "is a <tag>" / "is not a <tag>"). "Is" keeps the
|
||||
// tag ('keep'); "Is not" removes it, un-hiding a chrome image ('unhide'). The
|
||||
// content tag it also scored on is the reason it was flagged, not the question.
|
||||
function noun(it) { return it.tag_name === 'wip' ? 'WIP' : it.tag_name }
|
||||
function withArticle(it) { return (/^[aeiou]/i.test(noun(it)) ? 'an ' : 'a ') + noun(it) }
|
||||
function question(it) { return `Is this ${withArticle(it)}?` }
|
||||
function reasonTitle(it) {
|
||||
const hidden = it.mode === 'process' ? '' : ' It is hidden from the gallery until you answer.'
|
||||
return `Auto-tagged “${it.tag_name}”, but it also scored ${Math.round(it.conflict_score * 100)}% `
|
||||
+ `on “${it.conflict_name || 'a content tag'}”, so it may be finished art.${hidden}`
|
||||
}
|
||||
|
||||
async function load() {
|
||||
// Fetched unconditionally on mount — the strip prompts for pending misfires
|
||||
@@ -77,14 +85,11 @@ async function resolve(it, action) {
|
||||
await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`)
|
||||
items.value = items.value.filter((x) => keyOf(x) !== k)
|
||||
if (action === 'unhide') {
|
||||
const verb = it.mode === 'process' ? 'Removed' : 'Un-hidden'
|
||||
toast({ text: `${verb} — “${it.tag_name}” removed; it'll train the head`, type: 'success' })
|
||||
const shown = it.mode === 'process' ? '' : ', back in the gallery'
|
||||
toast({ text: `Not ${withArticle(it)} — “${it.tag_name}” removed${shown}; the tagger learns from it`, type: 'success' })
|
||||
}
|
||||
} catch (e) {
|
||||
toast({
|
||||
text: `Could not ${action === 'keep' ? 'keep hidden' : 'un-hide'}: ${e.message}`,
|
||||
type: 'error',
|
||||
})
|
||||
toast({ text: `Could not save your answer: ${e.message}`, type: 'error' })
|
||||
} finally {
|
||||
busy.value = busy.value.filter((x) => x !== k)
|
||||
}
|
||||
@@ -112,7 +117,7 @@ onMounted(load)
|
||||
display: flex; gap: 10px; overflow-x: auto; padding-bottom: 4px;
|
||||
}
|
||||
.fc-review-card {
|
||||
flex: 0 0 auto; width: 150px;
|
||||
flex: 0 0 auto; width: 170px;
|
||||
display: flex; flex-direction: column;
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
border-radius: 6px; overflow: hidden;
|
||||
@@ -123,16 +128,16 @@ onMounted(load)
|
||||
background: rgb(var(--v-theme-surface-light));
|
||||
}
|
||||
.fc-review-card__body { padding: 6px 8px; }
|
||||
.fc-review-card__conflict {
|
||||
font-size: 11px; color: rgb(var(--v-theme-on-surface));
|
||||
.fc-review-card__question {
|
||||
font-size: 12px; font-weight: 600; color: rgb(var(--v-theme-on-surface));
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-review-card__conflict strong { color: rgb(var(--v-theme-warning)); }
|
||||
.fc-review-card__tag {
|
||||
.fc-review-card__reason {
|
||||
font-size: 10px; color: rgb(var(--v-theme-on-surface-variant));
|
||||
margin: 1px 0 6px;
|
||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||
}
|
||||
.fc-review-card__reason strong { color: rgb(var(--v-theme-warning)); font-weight: 600; }
|
||||
.fc-review-card__acts { display: flex; gap: 4px; }
|
||||
.fc-review-btn {
|
||||
flex: 1; font-size: 11px; padding: 3px 4px; border-radius: 4px;
|
||||
|
||||
@@ -213,7 +213,7 @@ function nextFrame() {
|
||||
--fc-side-w: 320px;
|
||||
/* Obsidian haze (#14171A = 20,23,26) — same palette as TopNav,
|
||||
mid-opacity + blur so the page behind shows through faintly. */
|
||||
background: rgba(20, 23, 26, 0.65);
|
||||
background: rgba(var(--v-theme-background), 0.65);
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
outline: none;
|
||||
@@ -221,7 +221,7 @@ function nextFrame() {
|
||||
}
|
||||
.fc-viewer__close, .fc-viewer__nav {
|
||||
position: absolute; top: 50%;
|
||||
background: rgba(20, 23, 26, 0.7);
|
||||
background: rgba(var(--v-theme-background), 0.7);
|
||||
color: rgb(var(--v-theme-parchment, 232 228 216));
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
border-radius: 50%;
|
||||
@@ -359,7 +359,7 @@ function nextFrame() {
|
||||
z-index: 1;
|
||||
/* Opaque obsidian so the scrolling panel never bleeds through the
|
||||
haze behind the pinned image. */
|
||||
background: rgb(20, 23, 26);
|
||||
background: rgb(var(--v-theme-background));
|
||||
}
|
||||
.fc-viewer__side {
|
||||
width: 100%;
|
||||
|
||||
@@ -20,12 +20,13 @@
|
||||
class="fc-post-card__synthetic" :title="synthesisTitle"
|
||||
>
|
||||
<v-icon icon="mdi-auto-fix" size="x-small" start />
|
||||
grouped by FabledCurator
|
||||
{{ synthesisChip }}
|
||||
</v-chip>
|
||||
<RouterLink
|
||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||
class="fc-post-card__artist"
|
||||
>{{ post.artist.name }}</RouterLink>
|
||||
<span v-if="post.channel" class="fc-post-card__meta">#{{ post.channel }}</span>
|
||||
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
||||
<span v-if="totalImages" class="fc-post-card__meta">
|
||||
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
||||
@@ -270,11 +271,18 @@ const plainTitle = computed(() => toPlainText(props.post.post_title))
|
||||
// post dict by hand, must degrade to "not synthetic" rather than throw.
|
||||
const synthesized = computed(() => Boolean(props.post.synthesized_by))
|
||||
const messageCount = computed(() => props.post.synthesis?.message_count ?? 0)
|
||||
// A drop of one message stays a synthetic post (teaser matching only looks at
|
||||
// drops), but there is nothing grouped in it — #4390: "Grouped from 1 Discord
|
||||
// message" described a wrapper, not a grouping. Say what it is instead.
|
||||
const synthesisTitle = computed(() => {
|
||||
const n = messageCount.value
|
||||
if (!n) return 'Grouped from Discord'
|
||||
return `Grouped from ${n} Discord message${n === 1 ? '' : 's'}`
|
||||
if (n === 1) return 'Discord message'
|
||||
return `Grouped from ${n} Discord messages`
|
||||
})
|
||||
const synthesisChip = computed(() =>
|
||||
messageCount.value === 1 ? 'from Discord' : 'grouped by FabledCurator'
|
||||
)
|
||||
|
||||
const hero = computed(() => images.value[0])
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
<v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now
|
||||
</v-btn>
|
||||
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
||||
<QueueStatusBar queue="maintenance_long" queue-label="Long maintenance" />
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
|
||||
@@ -49,16 +49,20 @@
|
||||
sometimes triggered nothing instead of the install dialog
|
||||
(operator-flagged 2026-05-26). No `download` attribute —
|
||||
that would force a save dialog instead of install. -->
|
||||
<!-- The VERSIONED file, not the `latest` alias: a versioned URL can
|
||||
only ever be this build's bytes, so a browser cache can't hand
|
||||
back the previous build (operator-flagged 2026-09-25: a cached
|
||||
alias reinstalled the old version and the update never took). -->
|
||||
<v-btn
|
||||
v-if="isFirefox"
|
||||
color="accent" variant="flat" rounded="pill"
|
||||
prepend-icon="mdi-firefox"
|
||||
:href="manifest.latest_url"
|
||||
:href="manifest.xpi_url"
|
||||
>Install Firefox extension</v-btn>
|
||||
|
||||
<v-btn
|
||||
variant="outlined" rounded="pill"
|
||||
:href="manifest.latest_url" download
|
||||
:href="manifest.xpi_url" download
|
||||
prepend-icon="mdi-download"
|
||||
>Download XPI</v-btn>
|
||||
|
||||
|
||||
@@ -77,7 +77,7 @@
|
||||
<v-col cols="12" sm="6">
|
||||
<v-slider
|
||||
v-model="local.single_color_threshold" label="Single-color threshold"
|
||||
min="0.5" max="1" step="0.05" thumb-label hide-details color="accent"
|
||||
min="0.9" max="1" step="0.005" thumb-label hide-details color="accent"
|
||||
:disabled="!local.skip_single_color" @end="save"
|
||||
/>
|
||||
</v-col>
|
||||
@@ -103,17 +103,6 @@
|
||||
the Explore browse. Applies to new imports; run the scan below to catch
|
||||
posts already in your library.
|
||||
</div>
|
||||
<v-switch
|
||||
v-model="local.wip_soft_title_tagging_enabled"
|
||||
label="Also tag “sketch” / “doodle” titles (lower precision)"
|
||||
density="compact" hide-details color="primary" @change="save"
|
||||
/>
|
||||
<div class="fc-help mb-3">
|
||||
Extends the above to softer cues (<code>sketch</code>, <code>doodle</code>,
|
||||
<code>scribble</code>). These stay <strong>visible</strong> and never train
|
||||
the tagging model — a daily audit flags any that actually look like finished
|
||||
art for review. Off by default.
|
||||
</div>
|
||||
<v-btn
|
||||
variant="tonal" color="primary" size="small"
|
||||
:loading="store.wipScanBusy" prepend-icon="mdi-magnify"
|
||||
@@ -153,10 +142,9 @@ const PHASH_TICKS = { 0: 'Exact', 12: 'Strict', 24: 'Default', 48: 'Loose' }
|
||||
const local = reactive({
|
||||
min_width: 0, min_height: 0,
|
||||
skip_transparent: false, transparency_threshold: 0.9,
|
||||
skip_single_color: false, single_color_threshold: 0.95,
|
||||
skip_single_color: false, single_color_threshold: 0.995,
|
||||
phash_threshold: 24,
|
||||
wip_title_tagging_enabled: true,
|
||||
wip_soft_title_tagging_enabled: false,
|
||||
})
|
||||
|
||||
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
<v-icon start>mdi-file-remove-outline</v-icon> Repair missing-file records
|
||||
</v-btn>
|
||||
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
||||
<QueueStatusBar queue="maintenance_long" queue-label="Long maintenance" />
|
||||
</MaintenanceTile>
|
||||
</template>
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ const props = defineProps({
|
||||
|
||||
const QUEUE_NAMES = [
|
||||
'default', 'import', 'thumbnail', 'ml',
|
||||
'download', 'scan', 'maintenance',
|
||||
'download', 'scan', 'maintenance', 'maintenance_long',
|
||||
]
|
||||
|
||||
function formatDepth(name) {
|
||||
|
||||
@@ -368,6 +368,7 @@ function step(lane, delta) {
|
||||
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
|
||||
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
|
||||
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
|
||||
.fc-sys__dot--degraded { background: rgb(var(--v-theme-warning)); }
|
||||
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
|
||||
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
|
||||
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
<template>
|
||||
<!-- #4486. A Discord source's posts by people outside its "Only posts by"
|
||||
list, taken before the list existed. Preview first, always: nothing is
|
||||
deleted until the operator has seen the per-poster breakdown and typed
|
||||
the token that preview returned. -->
|
||||
<v-dialog :model-value="modelValue" max-width="560"
|
||||
@update:model-value="$emit('update:modelValue', $event)">
|
||||
<v-card>
|
||||
<v-card-title>Remove posts from other posters</v-card-title>
|
||||
<v-card-text>
|
||||
<div class="text-caption mb-2">{{ source.display_name || source.url }}</div>
|
||||
<v-progress-linear v-if="loading" indeterminate color="accent" class="mb-3" />
|
||||
<v-alert v-else-if="error" type="warning" variant="tonal" density="compact">
|
||||
{{ error }}
|
||||
</v-alert>
|
||||
<template v-else-if="result">
|
||||
<v-alert type="success" variant="tonal" density="compact">
|
||||
Removed {{ result.posts_deleted }} post(s), {{ result.images_deleted }} image(s),
|
||||
{{ result.attachments_deleted }} attachment(s) and {{ result.drops_deleted }}
|
||||
grouped post(s). Grouping rebuilds what remains on its next sweep.
|
||||
</v-alert>
|
||||
</template>
|
||||
<template v-else-if="preview">
|
||||
<p class="mb-2">
|
||||
Keeping posts by <strong>{{ preview.authors.join(', ') }}</strong>.
|
||||
</p>
|
||||
<p v-if="!preview.posts" class="fc-dim">
|
||||
Nothing to remove: every post with a known poster is by someone on the list.
|
||||
</p>
|
||||
<template v-else>
|
||||
<v-table density="compact" class="mb-3">
|
||||
<thead><tr><th>Poster</th><th class="text-right">Posts</th></tr></thead>
|
||||
<tbody>
|
||||
<tr v-for="p in preview.posters" :key="p.poster">
|
||||
<td>{{ p.poster }}</td>
|
||||
<td class="text-right">{{ p.posts }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</v-table>
|
||||
<p class="mb-1">
|
||||
{{ preview.posts }} post(s), {{ preview.images }} image(s) found only on them,
|
||||
{{ preview.attachments }} attachment(s).
|
||||
</p>
|
||||
<p v-if="preview.drops" class="mb-1">
|
||||
{{ preview.drops }} grouped post(s) that include them are ungrouped; the
|
||||
rest regroup on the next sweep.
|
||||
</p>
|
||||
<p class="fc-dim">
|
||||
Images that also belong to a kept post stay. If a name you want to keep
|
||||
appears above, add it to the source's "Only posts by" list first.
|
||||
</p>
|
||||
</template>
|
||||
<p v-if="preview.unknown_posts" class="fc-dim mt-2">
|
||||
{{ preview.unknown_posts }} older post(s) don't record who posted them and
|
||||
are left alone.
|
||||
</p>
|
||||
</template>
|
||||
</v-card-text>
|
||||
<v-card-actions>
|
||||
<v-spacer />
|
||||
<v-btn variant="text" @click="$emit('update:modelValue', false)">
|
||||
{{ result ? 'Close' : 'Cancel' }}
|
||||
</v-btn>
|
||||
<v-btn
|
||||
v-if="preview && preview.posts && !result"
|
||||
color="error" variant="flat" :loading="busy"
|
||||
@click="confirmOpen = true"
|
||||
>Remove {{ preview.posts }} post(s)</v-btn>
|
||||
</v-card-actions>
|
||||
</v-card>
|
||||
<DestructiveConfirmModal
|
||||
v-if="preview"
|
||||
v-model="confirmOpen"
|
||||
action="delete" kind="other posters' posts" tier="C"
|
||||
:expected-token-override="preview.confirm_token"
|
||||
:projected-counts="{ posts: preview.posts, images: preview.images,
|
||||
attachments: preview.attachments, grouped: preview.drops }"
|
||||
@confirm="onConfirm"
|
||||
/>
|
||||
</v-dialog>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { ref, watch } from 'vue'
|
||||
import { useSourcesStore } from '../../stores/sources.js'
|
||||
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||
|
||||
const props = defineProps({
|
||||
modelValue: { type: Boolean, default: false },
|
||||
source: { type: Object, required: true },
|
||||
})
|
||||
defineEmits(['update:modelValue'])
|
||||
const store = useSourcesStore()
|
||||
|
||||
const loading = ref(false)
|
||||
const busy = ref(false)
|
||||
const error = ref('')
|
||||
const preview = ref(null)
|
||||
const result = ref(null)
|
||||
const confirmOpen = ref(false)
|
||||
|
||||
function message(e) {
|
||||
return e?.body?.detail || e?.body?.error || e?.message || String(e)
|
||||
}
|
||||
|
||||
watch(() => props.modelValue, async (open) => {
|
||||
if (!open) return
|
||||
preview.value = null
|
||||
result.value = null
|
||||
error.value = ''
|
||||
loading.value = true
|
||||
try {
|
||||
preview.value = await store.previewOtherPosters(props.source.id)
|
||||
} catch (e) {
|
||||
error.value = message(e)
|
||||
} finally {
|
||||
loading.value = false
|
||||
}
|
||||
})
|
||||
|
||||
async function onConfirm(token) {
|
||||
busy.value = true
|
||||
try {
|
||||
result.value = await store.removeOtherPosters(props.source.id, token)
|
||||
} catch (e) {
|
||||
error.value = message(e)
|
||||
} finally {
|
||||
busy.value = false
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.fc-dim { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||
</style>
|
||||
@@ -50,6 +50,17 @@
|
||||
library — without re-downloading media
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-list-item
|
||||
v-if="hasPosterList && !running"
|
||||
prepend-icon="mdi-account-remove-outline"
|
||||
@click="cleanupOpen = true"
|
||||
>
|
||||
<v-list-item-title>Remove posts from other posters</v-list-item-title>
|
||||
<v-list-item-subtitle>
|
||||
Preview, then delete the posts taken from people outside this
|
||||
source's "Only posts by" list
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-divider v-if="isNative && !running" />
|
||||
<v-list-item
|
||||
base-color="error"
|
||||
@@ -59,12 +70,14 @@
|
||||
<v-list-item-title>Remove source</v-list-item-title>
|
||||
</v-list-item>
|
||||
</KebabMenu>
|
||||
<PosterCleanupDialog v-if="hasPosterList" v-model="cleanupOpen" :source="source" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import { computed } from 'vue'
|
||||
import { computed, ref } from 'vue'
|
||||
import KebabMenu from '../common/KebabMenu.vue'
|
||||
import PosterCleanupDialog from './PosterCleanupDialog.vue'
|
||||
|
||||
const props = defineProps({
|
||||
source: { type: Object, required: true },
|
||||
@@ -76,10 +89,14 @@ const running = computed(() => props.source.backfill_state === 'running')
|
||||
const recovering = computed(() => !!props.source.backfill_bypass_seen)
|
||||
const recapturing = computed(() => !!props.source.backfill_recapture)
|
||||
// Recover / recapture are native-ingester features (ledger-bypass re-walk and
|
||||
// post-text re-grab), available to every native platform — not just Patreon.
|
||||
// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS.
|
||||
const NATIVE_PLATFORMS = ['patreon', 'subscribestar']
|
||||
const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform))
|
||||
// post-text re-grab), available on every native platform. The backend says
|
||||
// which those are (`native_ingester`); a copied list here went stale when
|
||||
// Discord moved over.
|
||||
const isNative = computed(() => !!props.source.native_ingester)
|
||||
// #4486: only a Discord source with an "Only posts by" list has other posters.
|
||||
const hasPosterList = computed(() => props.source.platform === 'discord'
|
||||
&& (props.source.config_overrides?.discord_authors || []).length > 0)
|
||||
const cleanupOpen = ref(false)
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
<a
|
||||
:href="source.url" target="_blank" rel="noopener"
|
||||
class="fc-source-card__url" @click.stop
|
||||
>{{ source.url }}</a>
|
||||
>{{ source.display_name || source.url }}</a>
|
||||
<v-btn
|
||||
icon="mdi-pencil" size="x-small" variant="text"
|
||||
@click.stop="$emit('edit', source)"
|
||||
@@ -39,18 +39,18 @@
|
||||
</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'running'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
size="x-small" color="info" variant="flat" label
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||
}}{{ source.backfill_posts
|
||||
? ` · ${source.backfill_posts} posts`
|
||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'complete'"
|
||||
size="x-small" color="success" variant="tonal" label
|
||||
size="x-small" color="success" variant="flat" label
|
||||
>Backfilled</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'stalled'"
|
||||
size="x-small" color="warning" variant="tonal" label
|
||||
size="x-small" color="warning" variant="flat" label
|
||||
>Stalled</v-chip>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -44,6 +44,17 @@
|
||||
v-model="structuredSince" label="Skip posts older than (YYYY-MM-DD)"
|
||||
placeholder="2024-01-01" hide-details class="mt-2"
|
||||
/>
|
||||
<!-- A creator's server is full of other members posting their own
|
||||
pictures; the source is subscribed to the creator (#4481). -->
|
||||
<v-text-field
|
||||
v-if="platform === 'discord'"
|
||||
v-model="structuredAuthors" label="Only posts by (Discord names or ids)"
|
||||
placeholder="Todding" hint="Comma-separated. Empty takes everyone's."
|
||||
persistent-hint class="mt-2"
|
||||
/>
|
||||
<div v-if="platform === 'discord' && authorNames.length" class="text-caption mt-2">
|
||||
{{ authorNames.join(', ') }}
|
||||
</div>
|
||||
<p class="text-caption mt-2" style="opacity: 0.75">
|
||||
More per-platform fields land here over time. Use Advanced JSON for everything else.
|
||||
</p>
|
||||
@@ -98,6 +109,33 @@ const urlError = ref('')
|
||||
const configTab = ref('structured')
|
||||
const structuredVideos = ref(true)
|
||||
const structuredSince = ref('')
|
||||
const structuredAuthors = ref('')
|
||||
// Keys the structured view has no field for, carried through its saves so
|
||||
// switching tabs never drops what the JSON view set.
|
||||
const otherConfig = ref({})
|
||||
const STRUCTURED_KEYS = ['videos', 'since', 'discord_authors']
|
||||
|
||||
// The extension's poster picker stores ids (#4488) and the name each was
|
||||
// picked under, so a list of numbers still says who it means.
|
||||
const authorNames = computed(() => {
|
||||
const labels = otherConfig.value.discord_author_labels || {}
|
||||
return splitAuthors(structuredAuthors.value)
|
||||
.map(a => (labels[a] ? `${labels[a]} (${a})` : null))
|
||||
.filter(Boolean)
|
||||
})
|
||||
|
||||
function splitAuthors(txt) {
|
||||
return (txt || '').split(',').map(s => s.trim()).filter(Boolean)
|
||||
}
|
||||
|
||||
function takeConfig(co) {
|
||||
structuredVideos.value = co.videos !== false
|
||||
structuredSince.value = co.since ?? ''
|
||||
structuredAuthors.value = Array.isArray(co.discord_authors) ? co.discord_authors.join(', ') : ''
|
||||
otherConfig.value = Object.fromEntries(
|
||||
Object.entries(co).filter(([k]) => !STRUCTURED_KEYS.includes(k)),
|
||||
)
|
||||
}
|
||||
const jsonText = ref('{}')
|
||||
const jsonError = ref('')
|
||||
|
||||
@@ -105,13 +143,15 @@ const busy = ref(false)
|
||||
|
||||
// Sync config_overrides between the two views.
|
||||
const config = computed(() => {
|
||||
const out = {}
|
||||
const out = { ...otherConfig.value }
|
||||
if (!structuredVideos.value) out.videos = false
|
||||
if (structuredSince.value) out.since = structuredSince.value
|
||||
const authors = splitAuthors(structuredAuthors.value)
|
||||
if (authors.length) out.discord_authors = authors
|
||||
return out
|
||||
})
|
||||
|
||||
watch([structuredVideos, structuredSince], () => {
|
||||
watch([structuredVideos, structuredSince, structuredAuthors], () => {
|
||||
if (configTab.value === 'structured') {
|
||||
jsonText.value = JSON.stringify(config.value, null, 2)
|
||||
jsonError.value = ''
|
||||
@@ -128,8 +168,7 @@ watch(jsonText, (txt) => {
|
||||
}
|
||||
jsonError.value = ''
|
||||
// Reflect recognized keys into the structured view.
|
||||
structuredVideos.value = parsed.videos !== false
|
||||
structuredSince.value = parsed.since ?? ''
|
||||
takeConfig(parsed)
|
||||
} catch {
|
||||
jsonError.value = 'Invalid JSON'
|
||||
}
|
||||
@@ -146,14 +185,13 @@ watch(() => props.modelValue, async (open) => {
|
||||
url.value = props.source.url
|
||||
enabled.value = props.source.enabled
|
||||
const co = props.source.config_overrides || {}
|
||||
structuredVideos.value = co.videos !== false
|
||||
structuredSince.value = co.since ?? ''
|
||||
takeConfig(co)
|
||||
jsonText.value = JSON.stringify(co, null, 2)
|
||||
artistChoice.value = { id: props.source.artist_id, name: props.source.artist_name }
|
||||
} else {
|
||||
platform.value = platformsStore.list[0]?.key || 'patreon'
|
||||
url.value = ''; enabled.value = true
|
||||
structuredVideos.value = true; structuredSince.value = ''
|
||||
takeConfig({})
|
||||
jsonText.value = '{}'
|
||||
artistChoice.value = props.initialArtist
|
||||
? { id: props.initialArtist.id, name: props.initialArtist.name }
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
<div class="fc-source-row__url-wrap">
|
||||
<a :href="source.url" target="_blank" rel="noopener" class="fc-source-row__url"
|
||||
@click.stop>
|
||||
{{ source.url }}
|
||||
{{ source.display_name || source.url }}
|
||||
</a>
|
||||
<!-- Edit sits next to the source identity (operator-requested), not in
|
||||
the action cluster where it was easy to fat-finger Remove. -->
|
||||
@@ -72,7 +72,7 @@
|
||||
worked, the content just isn't ours. -->
|
||||
<v-chip
|
||||
v-else-if="source.error_type === 'tier_limited'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
size="x-small" color="info" variant="flat" label
|
||||
prepend-icon="mdi-lock-outline"
|
||||
>{{ source.tier_gated_count ? `${source.tier_gated_count} gated` : 'No access' }}
|
||||
<v-tooltip activator="parent" location="top" max-width="480">
|
||||
@@ -81,18 +81,18 @@
|
||||
</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'running'"
|
||||
size="x-small" color="info" variant="tonal" label
|
||||
size="x-small" color="info" variant="flat" label
|
||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||
}}{{ source.backfill_posts
|
||||
? ` · ${source.backfill_posts} posts`
|
||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'complete'"
|
||||
size="x-small" color="success" variant="tonal" label
|
||||
size="x-small" color="success" variant="flat" label
|
||||
>Backfilled</v-chip>
|
||||
<v-chip
|
||||
v-else-if="source.backfill_state === 'stalled'"
|
||||
size="x-small" color="warning" variant="tonal" label
|
||||
size="x-small" color="warning" variant="flat" label
|
||||
>Stalled</v-chip>
|
||||
<span v-else class="fc-source-row__zero">0</span>
|
||||
</td>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
v-if="item.singleSource"
|
||||
:href="item.singleSource.url" target="_blank" rel="noopener"
|
||||
class="fc-subs__sub-url" @click.stop
|
||||
>{{ item.singleSource.url }}</a>
|
||||
>{{ item.singleSource.display_name || item.singleSource.url }}</a>
|
||||
</template>
|
||||
|
||||
<template #item.platforms="{ item }">
|
||||
@@ -517,6 +517,7 @@ const filteredGroups = computed(() => {
|
||||
g.sources.some(
|
||||
(s) =>
|
||||
(s.url || '').toLowerCase().includes(q) ||
|
||||
(s.display_name || '').toLowerCase().includes(q) ||
|
||||
(s.platform || '').toLowerCase().includes(q),
|
||||
),
|
||||
)
|
||||
@@ -679,7 +680,7 @@ async function onRecover(source) {
|
||||
}
|
||||
}
|
||||
|
||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||
// #830: arm a recapture walk (native platforms) — re-grab every post's body +
|
||||
// external links and localize on-disk inline images, without re-downloading
|
||||
// media. Reuses the backfill lifecycle/badge; stop via the same Stop control.
|
||||
async function onRecapture(source) {
|
||||
|
||||
+17
-1
@@ -94,9 +94,25 @@ const routes = [
|
||||
const history =
|
||||
typeof window !== 'undefined' ? createWebHistory() : createMemoryHistory()
|
||||
|
||||
// Where the window lands after a navigation. Without this the router keeps the
|
||||
// window's scroll offset, so a view entered from halfway down another one opened
|
||||
// halfway down itself, its heading and tab strip under the nav (operator-flagged
|
||||
// 2026-09-24, Settings entered from a scrolled feed).
|
||||
// - back/forward returns to where that entry was left;
|
||||
// - the SAME view with a new query (a gallery filter, a Browse or
|
||||
// Subscriptions tab) keeps its place — that is not a new page;
|
||||
// - a new view opens at its top.
|
||||
export function scrollBehavior(to, from, savedPosition) {
|
||||
if (savedPosition) return savedPosition
|
||||
if (from.matched.length && to.path === from.path) return false
|
||||
if (to.hash) return { el: to.hash }
|
||||
return { top: 0 }
|
||||
}
|
||||
|
||||
const router = createRouter({
|
||||
history,
|
||||
routes
|
||||
routes,
|
||||
scrollBehavior
|
||||
})
|
||||
|
||||
const DEFAULT_TITLE = 'FabledCurator'
|
||||
|
||||
@@ -119,7 +119,7 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
||||
|
||||
/**
|
||||
* Polls /api/system/activity/runs?queue=maintenance every 3s,
|
||||
* Polls /api/system/activity/runs?celery_task_id=<id> every 3s,
|
||||
* resolves when a task_run row with the given celery task_id
|
||||
* reaches a terminal status (ok / error / timeout). Returns the
|
||||
* row. Times out after 30 min by default.
|
||||
@@ -129,7 +129,9 @@ export const useAdminStore = defineStore('admin', () => {
|
||||
while (Date.now() < deadline) {
|
||||
const body = await api.get(
|
||||
'/api/system/activity/runs',
|
||||
{ params: { queue: 'maintenance', limit: 20 } },
|
||||
// By id, not by lane: these jobs run on `maintenance_long`, and a
|
||||
// lane filter here is one more copy of the routing table (#4432).
|
||||
{ params: { celery_task_id: taskId, limit: 1 } },
|
||||
)
|
||||
const row = (body.runs || []).find(r => r.celery_task_id === taskId)
|
||||
if (row && ['ok', 'error', 'timeout'].includes(row.status)) {
|
||||
|
||||
@@ -12,7 +12,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
||||
min_width: 0,
|
||||
min_height: 0,
|
||||
transparency_threshold: 0.9,
|
||||
single_color_threshold: 0.95,
|
||||
single_color_threshold: 0.995,
|
||||
single_color_tolerance: 30,
|
||||
})
|
||||
|
||||
@@ -24,7 +24,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
||||
min_width: s.min_width ?? 0,
|
||||
min_height: s.min_height ?? 0,
|
||||
transparency_threshold: s.transparency_threshold ?? 0.9,
|
||||
single_color_threshold: s.single_color_threshold ?? 0.95,
|
||||
single_color_threshold: s.single_color_threshold ?? 0.995,
|
||||
single_color_tolerance: s.single_color_tolerance ?? 30,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
_patchSource(body)
|
||||
return body
|
||||
}
|
||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||
// #830: arm a recapture walk (native platforms) — re-grab every post's body +
|
||||
// external links and localize on-disk inline images, WITHOUT re-downloading
|
||||
// media. Shares the backfill lifecycle/badge; stop via stopBackfill.
|
||||
async function recaptureSource(id, artistIdHint = null) {
|
||||
@@ -149,6 +149,15 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
return body
|
||||
}
|
||||
|
||||
// #4486: a Discord source's posts by people outside its poster list.
|
||||
// The preview touches nothing and hands back the token the remove must echo.
|
||||
async function previewOtherPosters(id) {
|
||||
return api.get(`/api/sources/${id}/discord/other-posters`)
|
||||
}
|
||||
async function removeOtherPosters(id, confirm) {
|
||||
return api.post(`/api/sources/${id}/discord/other-posters/remove`, { body: { confirm } })
|
||||
}
|
||||
|
||||
function sourcesByArtistGrouped() {
|
||||
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
||||
const arr = byArtist.value.get(null) ?? []
|
||||
@@ -179,6 +188,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
stopBackfill,
|
||||
recoverSource,
|
||||
recaptureSource,
|
||||
previewOtherPosters, removeOtherPosters,
|
||||
findOrCreateArtist, autocompleteArtist, reassign,
|
||||
loadScheduleStatus,
|
||||
sourcesByArtistGrouped,
|
||||
|
||||
@@ -24,7 +24,8 @@ export function queueKey(queues) {
|
||||
}
|
||||
|
||||
// Worst first. A stopped datastore is why someone opened this tab.
|
||||
export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 }
|
||||
// `degraded`: checking in, but slow — a GPU agent on the CPU (#4410).
|
||||
export const SEVERITY = { down: 4, stale: 3, degraded: 2, unknown: 1, ok: 0 }
|
||||
|
||||
export function kindLabel(kind) {
|
||||
if (kind === 'celery') return 'worker lane'
|
||||
|
||||
@@ -369,7 +369,7 @@ onUnmounted(() => {
|
||||
|
||||
/* Center viewer. */
|
||||
.fc-ex__viewer {
|
||||
background: rgb(20, 23, 26);
|
||||
background: rgb(var(--v-theme-background));
|
||||
display: flex; flex-direction: column; min-width: 0; min-height: 0;
|
||||
}
|
||||
.fc-ex__canvas { flex: 1 1 auto; display: flex; min-height: 0; min-width: 0; }
|
||||
|
||||
@@ -276,7 +276,7 @@ onUnmounted(() => {
|
||||
.fc-reader__thumb img { width: 100%; height: auto; display: block; }
|
||||
.fc-reader__thumbnum {
|
||||
position: absolute; bottom: 4px; right: 4px;
|
||||
background: rgba(20, 23, 26, 0.8); color: rgb(var(--v-theme-on-surface));
|
||||
background: rgba(var(--v-theme-background), 0.8); color: rgb(var(--v-theme-on-surface));
|
||||
padding: 1px 5px; border-radius: 4px; font-size: 0.7rem;
|
||||
}
|
||||
.fc-reader__content {
|
||||
@@ -302,7 +302,7 @@ onUnmounted(() => {
|
||||
.fc-reader__indicator {
|
||||
position: fixed; bottom: 1.5rem; right: 1.5rem;
|
||||
padding: 4px 10px; border-radius: 999px;
|
||||
background: rgba(20, 23, 26, 0.6);
|
||||
background: rgba(var(--v-theme-background), 0.6);
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
font-size: 0.8rem; color: rgb(var(--v-theme-on-surface-variant));
|
||||
z-index: 100; pointer-events: none;
|
||||
|
||||
@@ -338,18 +338,18 @@ onMounted(() => {
|
||||
position: absolute; top: 6px; left: 6px;
|
||||
display: inline-flex; align-items: center; gap: 2px;
|
||||
padding: 1px 6px; border-radius: 999px; font-size: 11px;
|
||||
background: rgba(20, 23, 26, 0.75);
|
||||
background: rgba(var(--v-theme-background), 0.75);
|
||||
color: rgb(var(--v-theme-warning, var(--v-theme-accent)));
|
||||
}
|
||||
/* Kebab sits top-right of the cover, opposite the gap badge. Tinted backing so
|
||||
it stays legible over any cover image. */
|
||||
.fc-sbcard__kebab {
|
||||
position: absolute; top: 4px; right: 4px;
|
||||
background: rgba(20, 23, 26, 0.6) !important;
|
||||
background: rgba(var(--v-theme-background), 0.6) !important;
|
||||
border-radius: 50%;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
.fc-sbcard__kebab:hover { background: rgba(20, 23, 26, 0.85) !important; }
|
||||
.fc-sbcard__kebab:hover { background: rgba(var(--v-theme-background), 0.85) !important; }
|
||||
.fc-sbcard__body { padding: 8px 10px; }
|
||||
.fc-sbcard__name {
|
||||
font-family: 'Fraunces', Georgia, serif; font-size: 15px; font-weight: 600;
|
||||
|
||||
@@ -1,23 +1,12 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useAdminStore } from '../src/stores/admin.js'
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
// Covers the two helpers the admin store actions route through (DRY Finding C,
|
||||
// #753): _dryRunPost (URL + dry_run body, sourceId→source_id) and _guard
|
||||
// (lastError capture + rethrow). The store had no frontend test before.
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function lastCallBody(calls) {
|
||||
return JSON.parse(calls.at(-1).init.body)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user