Compare commits
23
Commits
+204
-76
@@ -242,7 +242,7 @@ jobs:
|
||||
# rather than left running beside the new mechanism (rule 22).
|
||||
#
|
||||
# Two things are still worth asserting, and this is the only lane that can:
|
||||
# the extension.yml suite runs on node:24-slim, which is exactly why
|
||||
# the extension-test lane runs on node:24-slim, which is exactly why
|
||||
# version.spec.js sticks to packaging.sh's git-free subcommands.
|
||||
# 1. the derivation actually resolves on this commit
|
||||
# 2. the derived string is one AMO will accept, checked against Mozilla's
|
||||
@@ -378,6 +378,76 @@ jobs:
|
||||
- run: npm run test:unit
|
||||
- run: npm run build
|
||||
|
||||
# The extension's lane: web-ext lint, the vitest suite, and the check that
|
||||
# asks web-ext what it ACTUALLY packaged. Moved in from `extension.yml`
|
||||
# (deleted) at milestone 429 — the same move `ci.yml` made on 2026-09-23 and
|
||||
# for the same reason: a separate workflow cannot gate this one, so a red
|
||||
# extension suite still let `sign-extension` sign and `build-web` ship the
|
||||
# XPI. It is a lane in THE GATE now. Its old path filter is gone with it; a
|
||||
# filter here would make the lane skip, and a skipped lane blocks the publish.
|
||||
#
|
||||
# The vitest suite is also the JS half of the #3093 artist-pattern mirror —
|
||||
# the Python half runs in backend-lint-and-test — so until this move one half
|
||||
# of that guard could fail without stopping anything.
|
||||
extension-test:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:24-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Not --no-save: vitest and web-ext are both real devDependencies now,
|
||||
# and the suite needs vitest resolvable from node_modules.
|
||||
- name: Install dev dependencies
|
||||
run: cd extension && npm install --no-audit --no-fund
|
||||
- name: Lint
|
||||
run: cd extension && npm run lint
|
||||
# Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
|
||||
# package version-consistency checks. No browser, no network.
|
||||
- name: Unit tests
|
||||
run: cd extension && npm run test:unit
|
||||
|
||||
# Everything else about packaging is asserted against our own declaration
|
||||
# of what ships. This is the only check that asks web-ext what it ACTUALLY
|
||||
# put in the archive. Until now that was an unverified assumption about
|
||||
# glob semantics — and a fragile one: `test/**` reaches web-ext intact
|
||||
# only because callers `set -f` first, so losing that quoting would
|
||||
# silently start shipping dev files with no other signal.
|
||||
- name: Verify XPI contents
|
||||
run: |
|
||||
set -eu
|
||||
command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
|
||||
cd extension
|
||||
npm run build
|
||||
ZIP=$(ls web-ext-artifacts/*.zip | head -1)
|
||||
echo "=== packaged entries in $ZIP ==="
|
||||
unzip -Z1 "$ZIP" | sort
|
||||
echo "=== end ==="
|
||||
ENTRIES=$(unzip -Z1 "$ZIP")
|
||||
fail=0
|
||||
# Must NOT ship: repo infrastructure with no business in a user's browser.
|
||||
for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
|
||||
if echo "$ENTRIES" | grep -q "^$pat"; then
|
||||
echo "ERROR: '$pat' was packaged into the XPI but must not be"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
# Must ship: if an exclusion pattern ever over-matches, the extension
|
||||
# breaks at runtime rather than at build time, so assert presence too.
|
||||
for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js' 'lib/chip.js' 'lib/popup-format.js'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$req$"; then
|
||||
echo "ERROR: '$req' is missing from the XPI"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$dir"; then
|
||||
echo "ERROR: nothing from '$dir' was packaged"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
[ "$fail" -eq 0 ] || exit 1
|
||||
echo "XPI contents verified."
|
||||
|
||||
# Single integration job — collapsed from a 3-way shard split on 2026-06-04.
|
||||
# The shards existed to parallelize ~8.5min of integration tests; once the
|
||||
# throwaway Postgres runs with fsync OFF (the durability step below) the whole
|
||||
@@ -419,7 +489,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
@@ -434,7 +504,7 @@ jobs:
|
||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||
echo "=== end landscape ==="
|
||||
PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:8-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
@@ -521,7 +591,8 @@ jobs:
|
||||
# itself through a job-level `if:` that could not read `env`, and a design
|
||||
# where only a FAILED gate blocks would have published unverified images
|
||||
# while reporting success.
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
|
||||
integration]
|
||||
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
||||
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
||||
# it is stated on each publishing job rather than inferred from a `needs`
|
||||
@@ -850,13 +921,16 @@ jobs:
|
||||
# reuse hit — deliberately NOT folded into `digest`, which the :c-<sha>
|
||||
# repoint reads and which must keep meaning "what this run built" (#4290).
|
||||
published_digest: ${{ steps.reuse.outputs.published_digest }}
|
||||
# Every tag this commit should end up under (channel, plus :c-<sha> on
|
||||
# main). `promote` writes them from `digest` once the smoke has passed.
|
||||
tags: ${{ steps.tag.outputs.tags }}
|
||||
# A plain `needs` — no `always()`. That expression existed to let a
|
||||
# SKIPPED sign-extension through on a tag push while still blocking a
|
||||
# FAILED one. With no tag trigger, sign-extension always runs, so the
|
||||
# default behaviour is exactly what we want: a failed sign skips build-web
|
||||
# rather than shipping an image without its XPI.
|
||||
needs: [sign-extension, lint, extension-version, backend-lint-and-test,
|
||||
frontend-build, integration]
|
||||
frontend-build, extension-test, integration]
|
||||
# The lanes in that list are THE GATE (2026-09-23) — see sign-extension's
|
||||
# copy of this comment for why, including the property that a SKIPPED lane
|
||||
# blocks as firmly as a failing one. They are repeated here rather than
|
||||
@@ -1108,11 +1182,13 @@ jobs:
|
||||
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||
# whether the channel then has to be written separately.
|
||||
#
|
||||
# On a push the build writes the channel tag directly: the bytes came
|
||||
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
||||
# it behind.
|
||||
# Every build writes a CANDIDATE tag, never the channel (#4310):
|
||||
# :dev-candidate / :latest-candidate on a push, :refresh-candidate on
|
||||
# the refresh. `promote` moves the channel only after smoke-web has
|
||||
# booted the bytes. The lanes prove the SOURCE; only the smoke proves
|
||||
# the BYTES, and the two are not the same claim.
|
||||
#
|
||||
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
||||
# The refresh is the case that made this obvious. A
|
||||
# refresh rebuilds against freshly resolved base images, and the web
|
||||
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||
@@ -1142,7 +1218,10 @@ jobs:
|
||||
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||
echo "promote=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||
# A push builds to a per-channel candidate too (#4310). The channel
|
||||
# tag moves in `promote`, after smoke-web has booted these bytes —
|
||||
# so a broken image never reaches the tag deployments follow.
|
||||
echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
|
||||
echo "promote=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
@@ -1418,6 +1497,16 @@ jobs:
|
||||
TAGS: ${{ steps.tag.outputs.tags }}
|
||||
run: |
|
||||
set -euf
|
||||
# A BUILD publishes nothing from here (#4310). Its bytes sit on the
|
||||
# candidate tag until smoke-web has booted them; `promote` then
|
||||
# writes the channel tag AND :c-<sha> from this run's digest. Writing
|
||||
# :c-<sha> here would publish an immutable rollback tag for bytes
|
||||
# that might then fail the smoke.
|
||||
if [ -n "${BUILT_DIGEST:-}" ]; then
|
||||
echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
|
||||
echo "repoint: after the smoke; nothing to write here."
|
||||
exit 0
|
||||
fi
|
||||
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
||||
#
|
||||
# This step used to copy from the channel tag by NAME. Nothing
|
||||
@@ -1511,14 +1600,9 @@ jobs:
|
||||
# source tree, and not a static inspection: `ffmpeg -version` exiting 0 would
|
||||
# pass while a codec removal broke every thumbnail in the library.
|
||||
#
|
||||
# Refresh-only. On a push the bytes came from a commit, and a commit is what
|
||||
# the lanes above already test — and since 2026-09-23 they gate the build, so
|
||||
# those bytes could not exist without them having passed.
|
||||
#
|
||||
# Reports a verdict; it does not yet gate the promote (milestone 362 step 4).
|
||||
# Landing the gate and the thing it gates in one change would mean the first
|
||||
# time anyone saw this job run would also be the first time it could stop a
|
||||
# publish.
|
||||
# It gates `promote` on every trigger (#4310). The lanes above prove the
|
||||
# source; they cannot see the bytes, and a Dockerfile or base change breaks
|
||||
# the bytes without touching the source.
|
||||
smoke-web:
|
||||
needs: [build-web]
|
||||
# Every run that actually BUILT something, not just the weekly refresh.
|
||||
@@ -1530,13 +1614,10 @@ jobs:
|
||||
# and were smoked when they were built. Re-smoking them would burn two
|
||||
# minutes to re-learn a fact.
|
||||
#
|
||||
# HONEST LIMIT, and it is the reason #4299 exists: on a push this runs
|
||||
# AFTER build-web has written the channel tag, so it detects rather than
|
||||
# gates. Rule 164's verify_with asks for the check BETWEEN build and push.
|
||||
# Closing that needs the push path to adopt the candidate-then-promote
|
||||
# shape the refresh already has — per-channel candidate tags, promote
|
||||
# learning its channel, and the :c-<sha> repoint moving after the gate.
|
||||
# That is a redesign of the production publish path and is its own task.
|
||||
# This GATES the publish on every trigger (#4310): builds land on a
|
||||
# candidate tag, and `promote` needs this job, so :dev / :latest and a
|
||||
# built :c-<sha> move only after it passes. Rule 164's verify_with — the
|
||||
# check BETWEEN build and push.
|
||||
#
|
||||
# NO `if:` — this job always runs (#4323). It used to be gated on the
|
||||
# build having published something, which skipped it on a reuse hit. That
|
||||
@@ -1583,7 +1664,7 @@ jobs:
|
||||
--health-timeout 5s
|
||||
--health-retries 10
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
@@ -1609,7 +1690,7 @@ jobs:
|
||||
# in a container against a mounted docker socket, so the services are
|
||||
# SIBLINGS reachable by IP, not by hostname.
|
||||
PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
||||
RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:8-alpine" -q | head -n1)
|
||||
test -n "$PG" && test -n "$RD"
|
||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||
@@ -1954,88 +2035,113 @@ jobs:
|
||||
# smoked it, and it should not be read as if it were.
|
||||
promote:
|
||||
needs: [build-web, build-agent, smoke-web]
|
||||
# Only a refresh publishes through a candidate; a push writes its channel
|
||||
# tag directly from the build. Reads the same reuse-step decision the build
|
||||
# took, via a job output — a job's `if:` cannot see the `env` context.
|
||||
if: needs.build-web.outputs.candidate == 'true'
|
||||
# THE PUBLISH (#4310). Both image jobs build to a candidate tag — a push to
|
||||
# :dev-candidate / :latest-candidate, the weekly refresh to
|
||||
# :refresh-candidate — and nothing names those bytes under a tag anybody
|
||||
# deploys until this job runs. It runs only when every job in `needs`
|
||||
# SUCCEEDED (a job-level `if:` without a status function implies
|
||||
# success()), so a failed smoke — or a skipped one, which is not the same
|
||||
# as a passing one (run 5290) — leaves :dev / :latest on the last build
|
||||
# that worked. Rule 164's verify_with: the check sits BETWEEN build and
|
||||
# push.
|
||||
#
|
||||
# Per image, from the DIGEST that image's job built (#4290): a tag can
|
||||
# move between the build and this job, a digest cannot. An image whose job
|
||||
# hit reuse built nothing and has nothing to promote — its build job
|
||||
# already wrote its :c-<sha> from the channel tag, which "hit" proved
|
||||
# carries this commit.
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
steps:
|
||||
- name: Point the channel tags at the smoked candidates
|
||||
- name: Point this commit's tags at the smoked builds
|
||||
env:
|
||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||
ACTOR: ${{ github.actor }}
|
||||
WEB_DIGEST: ${{ needs.build-web.outputs.digest }}
|
||||
WEB_TAGS: ${{ needs.build-web.outputs.tags }}
|
||||
AGENT_DIGEST: ${{ needs.build-agent.outputs.digest }}
|
||||
AGENT_TAGS: ${{ needs.build-agent.outputs.tags }}
|
||||
run: |
|
||||
set -eu
|
||||
# `latest` is not a guess: a refresh always builds `main` (BUILD_REF),
|
||||
# and the "must have checked out main" guard in every build job fails
|
||||
# the run if that did not hold. So the channel is main's.
|
||||
TAG=latest
|
||||
FAILED=""
|
||||
MOVED=0
|
||||
|
||||
for NAME in fabledcurator fabledcurator-agent; do
|
||||
promote() {
|
||||
NAME="$1"; DIGEST="$2"; TAGS="$3"
|
||||
REPO="bvandeusen/$NAME"
|
||||
echo "promote: $REPO"
|
||||
if [ -z "$DIGEST" ]; then
|
||||
echo "promote: $NAME — no build this run (reuse hit); nothing to publish"
|
||||
return 0
|
||||
fi
|
||||
echo "promote: $NAME $DIGEST"
|
||||
|
||||
# Registry auth is its own token exchange — `docker login`
|
||||
# authenticates the docker client, not curl. Deadline on every call
|
||||
# (rule 156): a registry that stops answering must fail this step,
|
||||
# not hang the weekly refresh until the job times out.
|
||||
# not hang the run until the job times out.
|
||||
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
|
||||
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
|
||||
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
|
||||
|
||||
# Ask for the IMAGE manifest media types only. Offering the index
|
||||
# types too would let the registry hand back an index if one ever
|
||||
# existed at this tag, and we would faithfully copy the thing this
|
||||
# whole approach exists to avoid creating.
|
||||
# types too would let the registry hand back an index, and we would
|
||||
# faithfully copy the thing this approach exists to avoid creating.
|
||||
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
|
||||
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
|
||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$DIGEST" \
|
||||
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
|
||||
test -n "$CT"
|
||||
SRC=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||
echo "promote: candidate $SRC ($CT)"
|
||||
|
||||
# NOT `imagetools create`. That wraps its source in an INDEX, and
|
||||
# `.Image.Config.Labels` does not resolve through one — the
|
||||
# fc.revision the reuse check reads off the channel tag would come
|
||||
# back empty, every later push would miss and rebuild, and nothing
|
||||
# would go red (#3183, run 4751). A manifest PUT is what "make this
|
||||
# tag name that image" means at the registry: same bytes, same media
|
||||
# type, same digest, no layer transfer.
|
||||
curl -fsS --max-time 120 -X PUT \
|
||||
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
|
||||
--data-binary @manifest.json \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
|
||||
# tag name that image" means at the registry: same bytes, same
|
||||
# media type, same digest, no layer transfer. (It also makes a
|
||||
# built :c-<sha> a plain image rather than the index the old
|
||||
# repoint left.)
|
||||
IFS=,
|
||||
for REF in $TAGS; do
|
||||
TAG="${REF##*:}"
|
||||
curl -fsS --max-time 120 -X PUT \
|
||||
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
|
||||
--data-binary @manifest.json \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
|
||||
|
||||
# Read it back. A PUT that returned 2xx but landed something else is
|
||||
# exactly the silent-and-plausible failure this pipeline keeps
|
||||
# producing, and the check costs one request.
|
||||
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
|
||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
|
||||
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||
if [ "$NOW" != "$SRC" ]; then
|
||||
echo "promote: FAILED — $NAME:$TAG is $NOW, expected $SRC" >&2
|
||||
FAILED="$FAILED $NAME"
|
||||
continue
|
||||
fi
|
||||
echo "promote: $NAME:$TAG now names $NOW"
|
||||
done
|
||||
# Read it back. A PUT that returned 2xx but landed something else
|
||||
# is exactly the silent-and-plausible failure this pipeline keeps
|
||||
# producing, and the check costs one request.
|
||||
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
|
||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
|
||||
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||
if [ "$NOW" != "$DIGEST" ]; then
|
||||
echo "promote: FAILED — $NAME:$TAG is $NOW, expected $DIGEST" >&2
|
||||
FAILED="$FAILED $NAME:$TAG"
|
||||
continue
|
||||
fi
|
||||
echo "promote: $NAME:$TAG now names $NOW"
|
||||
MOVED=$((MOVED + 1))
|
||||
done
|
||||
unset IFS
|
||||
}
|
||||
|
||||
promote fabledcurator "$WEB_DIGEST" "$WEB_TAGS"
|
||||
promote fabledcurator-agent "$AGENT_DIGEST" "$AGENT_TAGS"
|
||||
|
||||
if [ -n "$FAILED" ]; then
|
||||
echo "" >&2
|
||||
echo "promote: FAILED for:$FAILED" >&2
|
||||
echo "promote: the channel tags are now INCONSISTENT — some images" >&2
|
||||
echo "promote: moved and some did not. Re-run this refresh; the" >&2
|
||||
echo "promote: candidates are still published and the promote is" >&2
|
||||
echo "promote: idempotent." >&2
|
||||
echo "promote: the tags are now INCONSISTENT — some moved and some" >&2
|
||||
echo "promote: did not. Re-run this workflow; the builds are still" >&2
|
||||
echo "promote: published by digest and the promote is idempotent." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "promote: both channel tags moved"
|
||||
echo "promote: $MOVED tag(s) written"
|
||||
build-agent:
|
||||
# THE GATE (2026-09-23). Every lane above must have PASSED before this job
|
||||
# exists at all — so a red suite does not produce an image, let alone push
|
||||
@@ -2048,7 +2154,14 @@ jobs:
|
||||
# itself through a job-level `if:` that could not read `env`, and a design
|
||||
# where only a FAILED gate blocks would have published unverified images
|
||||
# while reporting success.
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
|
||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
|
||||
integration]
|
||||
# What `promote` needs to publish this image once the smoke has passed:
|
||||
# the manifest this run built (empty on a reuse hit) and every tag it
|
||||
# belongs under. Same meaning as build-web's outputs of the same names.
|
||||
outputs:
|
||||
digest: ${{ steps.build.outputs.digest }}
|
||||
tags: ${{ steps.tag.outputs.tags }}
|
||||
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
||||
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
||||
# it is stated on each publishing job rather than inferred from a `needs`
|
||||
@@ -2240,11 +2353,13 @@ jobs:
|
||||
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||
# whether the channel then has to be written separately.
|
||||
#
|
||||
# On a push the build writes the channel tag directly: the bytes came
|
||||
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
||||
# it behind.
|
||||
# Every build writes a CANDIDATE tag, never the channel (#4310):
|
||||
# :dev-candidate / :latest-candidate on a push, :refresh-candidate on
|
||||
# the refresh. `promote` moves the channel only after smoke-web has
|
||||
# booted the bytes. The lanes prove the SOURCE; only the smoke proves
|
||||
# the BYTES, and the two are not the same claim.
|
||||
#
|
||||
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
||||
# The refresh is the case that made this obvious. A
|
||||
# refresh rebuilds against freshly resolved base images, and the web
|
||||
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||
@@ -2273,7 +2388,10 @@ jobs:
|
||||
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
||||
# A push builds to a per-channel candidate too (#4310). The channel
|
||||
# tag moves in `promote`, after smoke-web has booted these bytes —
|
||||
# so a broken image never reaches the tag deployments follow.
|
||||
echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
||||
@@ -2464,6 +2582,16 @@ jobs:
|
||||
TAGS: ${{ steps.tag.outputs.tags }}
|
||||
run: |
|
||||
set -euf
|
||||
# A BUILD publishes nothing from here (#4310). Its bytes sit on the
|
||||
# candidate tag until smoke-web has booted them; `promote` then
|
||||
# writes the channel tag AND :c-<sha> from this run's digest. Writing
|
||||
# :c-<sha> here would publish an immutable rollback tag for bytes
|
||||
# that might then fail the smoke.
|
||||
if [ -n "${BUILT_DIGEST:-}" ]; then
|
||||
echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
|
||||
echo "repoint: after the smoke; nothing to write here."
|
||||
exit 0
|
||||
fi
|
||||
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
||||
#
|
||||
# This step used to copy from the channel tag by NAME. Nothing
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
name: extension
|
||||
# Lint + unit tests. Deliberately NOT a publishing lane, which is why it is
|
||||
# not part of build.yml's gate. The sign-and-publish dance moved into build.yml's
|
||||
# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
|
||||
# because sign-extension runs as a build-web dependency in the SAME workflow,
|
||||
# eliminating the prior race between build.yml and a separate extension.yml.
|
||||
# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
|
||||
on:
|
||||
push:
|
||||
branches: [dev, main]
|
||||
paths:
|
||||
- 'extension/**'
|
||||
- '.forgejo/workflows/extension.yml'
|
||||
# test/version.spec.js asserts things ABOUT build.yml — that it does not
|
||||
# inline the packaged-file set, and that build.yml derives
|
||||
# the shipped version rather than reading it out of the repo. A
|
||||
# workflow-only edit can therefore break this suite, so it has to trigger
|
||||
# it. build.yml joined the list at milestone 271 step 5, when the spec
|
||||
# started asserting against it.
|
||||
- '.forgejo/workflows/build.yml'
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- 'extension/**'
|
||||
- '.forgejo/workflows/build.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: python-ci
|
||||
container:
|
||||
image: node:24-bookworm-slim
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Not --no-save: vitest and web-ext are both real devDependencies now,
|
||||
# and the suite needs vitest resolvable from node_modules.
|
||||
- name: Install dev dependencies
|
||||
run: cd extension && npm install --no-audit --no-fund
|
||||
- name: Lint
|
||||
run: cd extension && npm run lint
|
||||
# Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
|
||||
# package version-consistency checks. No browser, no network.
|
||||
- name: Unit tests
|
||||
run: cd extension && npm run test:unit
|
||||
|
||||
# Everything else about packaging is asserted against our own declaration
|
||||
# of what ships. This is the only check that asks web-ext what it ACTUALLY
|
||||
# put in the archive. Until now that was an unverified assumption about
|
||||
# glob semantics — and a fragile one: `test/**` reaches web-ext intact
|
||||
# only because callers `set -f` first, so losing that quoting would
|
||||
# silently start shipping dev files with no other signal.
|
||||
- name: Verify XPI contents
|
||||
run: |
|
||||
set -eu
|
||||
command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
|
||||
cd extension
|
||||
npm run build
|
||||
ZIP=$(ls web-ext-artifacts/*.zip | head -1)
|
||||
echo "=== packaged entries in $ZIP ==="
|
||||
unzip -Z1 "$ZIP" | sort
|
||||
echo "=== end ==="
|
||||
ENTRIES=$(unzip -Z1 "$ZIP")
|
||||
fail=0
|
||||
# Must NOT ship: repo infrastructure with no business in a user's browser.
|
||||
for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
|
||||
if echo "$ENTRIES" | grep -q "^$pat"; then
|
||||
echo "ERROR: '$pat' was packaged into the XPI but must not be"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
# Must ship: if an exclusion pattern ever over-matches, the extension
|
||||
# breaks at runtime rather than at build time, so assert presence too.
|
||||
for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$req$"; then
|
||||
echo "ERROR: '$req' is missing from the XPI"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
|
||||
if ! echo "$ENTRIES" | grep -q "^$dir"; then
|
||||
echo "ERROR: nothing from '$dir' was packaged"
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
[ "$fail" -eq 0 ] || exit 1
|
||||
echo "XPI contents verified."
|
||||
+3
-7
@@ -36,11 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
libwebp7 \
|
||||
libpng16-16 \
|
||||
ca-certificates \
|
||||
# opencv-python-headless (via requirements-ml.txt) links these even in its
|
||||
# headless build. Came from Dockerfile.ml when the images merged
|
||||
# (milestone 422 step 6).
|
||||
libgl1 \
|
||||
libglib2.0-0 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
@@ -61,7 +56,8 @@ RUN pip install -r requirements.txt
|
||||
#
|
||||
# torch 2.12.1+cpu wheel 192.3 MB
|
||||
# torchvision 0.27.1+cpu 1.8 MB
|
||||
# transformers / onnxruntime / opencv / sklearn and friends
|
||||
# transformers / onnxruntime / opencv / sklearn and friends (opencv and
|
||||
# onnxruntime since dropped, #1451 — nothing here imported them)
|
||||
# 62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB
|
||||
# largest newly-pushed layer 222.07 MB
|
||||
#
|
||||
@@ -85,7 +81,7 @@ RUN pip install -r requirements.txt
|
||||
# CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the
|
||||
# GPU agent is a separate service with its own image.
|
||||
RUN pip install --index-url https://download.pytorch.org/whl/cpu \
|
||||
"torch>=2.12,<3.0" "torchvision>=0.27,<0.28"
|
||||
"torch>=2.14" "torchvision>=0.29"
|
||||
RUN pip install -r requirements-ml.txt
|
||||
|
||||
# Where the model lands. Deliberately NOT a VOLUME instruction: that mints an
|
||||
|
||||
@@ -271,11 +271,12 @@ Four deployable pieces, built by `.forgejo/workflows/build.yml`:
|
||||
|
||||
## CI / Forgejo setup
|
||||
|
||||
Three workflows: `build.yml` (the five verification lanes — lint,
|
||||
extension-version check, backend unit tests, frontend build, integration — and
|
||||
then sign + publish), `extension.yml` (extension lint, vitest, XPI content
|
||||
verification), and `release.yml`, which runs only on a `v*` tag and publishes a
|
||||
changelog without building anything.
|
||||
Two workflows that matter here: `build.yml` (the six verification lanes — lint,
|
||||
extension-version check, backend unit tests, frontend build, extension lint +
|
||||
vitest + XPI content check, integration — and then sign + publish), and
|
||||
`release.yml`, which runs only on a `v*` tag and publishes a changelog without
|
||||
building anything. The extension lane was its own `extension.yml` until
|
||||
milestone 429, which let a red extension suite sign and ship the XPI anyway.
|
||||
|
||||
**The lanes and the publish are one workflow on purpose.** They were two
|
||||
(`ci.yml` and `build.yml`) until 2026-09-23, on the same push trigger, which
|
||||
|
||||
+23
-10
@@ -1,10 +1,21 @@
|
||||
# FabledCurator GPU agent — runs on the desktop with the GPU.
|
||||
# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs
|
||||
# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open
|
||||
# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
|
||||
# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are
|
||||
# built against (CUDA 13 has only nascent ONNX Runtime support).
|
||||
FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04
|
||||
#
|
||||
# The `base` flavour, not `cudnn-runtime`: CUDA and cuDNN arrive as the
|
||||
# `nvidia-*` pip packages torch and onnxruntime-gpu depend on, so the base only
|
||||
# has to hand the container the driver (it sets NVIDIA_VISIBLE_DEVICES /
|
||||
# NVIDIA_DRIVER_CAPABILITIES for the Container Toolkit). Until #1451 this was
|
||||
# `12.9.2-cudnn-runtime` under a `torch==2.6.0+cu124` — and requirements.txt then
|
||||
# REPLACED that torch with PyPI's CUDA-13 build (ultralytics pulls torchvision,
|
||||
# which pulls its matching torch), beside a CUDA-13 onnxruntime-gpu. The image
|
||||
# ran CUDA 13 on a CUDA-12 base, carrying ~3 GB of base libraries and a ~3 GB
|
||||
# torch nothing loaded: 10 GB compressed.
|
||||
#
|
||||
# 13.0 because that is the line both wheels are built for (torch's cu130 index,
|
||||
# onnxruntime-gpu's `nvidia-cuda-runtime~=13.0`). Needs an NVIDIA driver that
|
||||
# supports CUDA 13 (580+); fc_agent/accel.py logs at startup whether torch and
|
||||
# onnxruntime actually got the GPU, since both fall back to the CPU silently.
|
||||
# ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
|
||||
FROM nvidia/cuda:13.0.3-base-ubuntu24.04
|
||||
|
||||
# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
|
||||
# managed (PEP 668), so a global `pip install` errors without this. It's a
|
||||
@@ -16,10 +27,12 @@ RUN apt-get update \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN
|
||||
# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first
|
||||
# + separately so the GPU build of torch is deterministic and layer-cached.
|
||||
RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
|
||||
# torch AND torchvision from the cu130 index, together and first. Installing
|
||||
# torch alone is what let the next step swap it out: ultralytics needs
|
||||
# torchvision, PyPI's torchvision pins its own torch, and pip replaced ours to
|
||||
# match. With both present, requirements.txt finds them satisfied.
|
||||
RUN pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu130 \
|
||||
torch torchvision
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install --no-cache-dir -r requirements.txt
|
||||
COPY fc_agent ./fc_agent
|
||||
|
||||
+16
-1
@@ -15,7 +15,22 @@ sudo pacman -S nvidia-container-toolkit
|
||||
sudo nvidia-ctk runtime configure --runtime=docker
|
||||
sudo systemctl restart docker
|
||||
# verify:
|
||||
docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi
|
||||
docker run --rm --gpus all nvidia/cuda:13.0.3-base-ubuntu24.04 nvidia-smi
|
||||
# the header's CUDA version must be 13.0 or later (driver 580+)
|
||||
```
|
||||
|
||||
### After a driver update: regenerate the CDI spec
|
||||
If the agent's first log lines say `accel: torch is NOT on the GPU` or report
|
||||
`cudaGetDeviceCount: unknown error (999)` while `nvidia-smi` still works, the
|
||||
toolkit's saved device list (`/etc/cdi/nvidia.yaml`) is out of date. The
|
||||
`nvidia-uvm` device number changes between driver versions, and a spec
|
||||
generated before the update hands the container a device node that no longer
|
||||
exists (2026-09-24: host `511,0`, container `235,0`). Compare
|
||||
`ls -l /dev/nvidia-uvm` on the host with the same inside the container, then:
|
||||
```sh
|
||||
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
|
||||
# if your toolkit ships it, this keeps it current on every driver update:
|
||||
sudo systemctl enable --now nvidia-cdi-refresh.path
|
||||
```
|
||||
|
||||
## 1. Get a token
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Which accelerator each runtime actually got — reported once, at startup.
|
||||
|
||||
The agent has two GPU runtimes and both fall back to the CPU without raising:
|
||||
torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils
|
||||
detector + CCIP models) when its CUDA provider cannot load its libraries. A
|
||||
fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the
|
||||
image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base
|
||||
(#1451), and whether the ONNX half was on the GPU could not be answered from
|
||||
anything the agent had ever logged.
|
||||
|
||||
Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA
|
||||
provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image
|
||||
they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT
|
||||
1.21+) loads them from there, so the provider resolves them by soname.
|
||||
|
||||
Stdlib-only at import, so the unit suite can import it — torch and onnxruntime
|
||||
are imported inside the functions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ctypes
|
||||
import importlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("fc_agent.accel")
|
||||
|
||||
# Filled by report(); /status carries it so the page can show it too.
|
||||
LAST: dict = {}
|
||||
|
||||
|
||||
def torch_status(imp=importlib.import_module) -> dict:
|
||||
try:
|
||||
torch = imp("torch")
|
||||
except Exception as e:
|
||||
return {"device": "unavailable", "error": str(e)}
|
||||
out = {"version": torch.__version__, "cuda_build": torch.version.cuda}
|
||||
if torch.cuda.is_available():
|
||||
out["device"] = "cuda"
|
||||
out["gpu"] = torch.cuda.get_device_name(0)
|
||||
else:
|
||||
out["device"] = "cpu"
|
||||
return out
|
||||
|
||||
|
||||
def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict:
|
||||
try:
|
||||
ort = imp("onnxruntime")
|
||||
except Exception as e:
|
||||
return {"device": "unavailable", "error": str(e)}
|
||||
out = {"version": ort.__version__, "providers": list(ort.get_available_providers())}
|
||||
if "CUDAExecutionProvider" not in out["providers"]:
|
||||
out["device"] = "cpu"
|
||||
return out
|
||||
preload = getattr(ort, "preload_dlls", None)
|
||||
if preload is not None:
|
||||
try:
|
||||
preload()
|
||||
except Exception as e:
|
||||
out["preload_error"] = str(e)
|
||||
# "Available" only means the build HAS the provider. Loading its library is
|
||||
# what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are
|
||||
# missing, and the one a session would otherwise fail silently on.
|
||||
capi = Path(ort.__file__).parent / "capi"
|
||||
try:
|
||||
load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL)
|
||||
load(str(capi / "libonnxruntime_providers_cuda.so"))
|
||||
except OSError as e:
|
||||
out["device"] = "cpu"
|
||||
out["error"] = str(e)
|
||||
return out
|
||||
# Loading proves the libraries resolve, NOT that a GPU can be used: on
|
||||
# 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with
|
||||
# "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA
|
||||
# runtime for a device initialises the driver the provider would use.
|
||||
error = _cuda_device_error(load)
|
||||
out["device"] = "cpu" if error else "cuda"
|
||||
if error:
|
||||
out["error"] = error
|
||||
return out
|
||||
|
||||
|
||||
def _cuda_device_error(load=ctypes.CDLL) -> str | None:
|
||||
"""None when the CUDA runtime can reach a device, else why it cannot."""
|
||||
try:
|
||||
cudart = load("libcudart.so.13")
|
||||
except OSError as e:
|
||||
return str(e)
|
||||
count = ctypes.c_int(0)
|
||||
rc = cudart.cudaGetDeviceCount(ctypes.byref(count))
|
||||
if rc != 0:
|
||||
cudart.cudaGetErrorString.restype = ctypes.c_char_p
|
||||
return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})"
|
||||
return None if count.value > 0 else "no CUDA device visible"
|
||||
|
||||
|
||||
def summary() -> dict | None:
|
||||
"""The report as FabledCurator stores it: each runtime's device, and why
|
||||
when it is not the GPU. Sent on every lease and heartbeat, so the System
|
||||
view can call a running agent that fell back to the CPU "degraded" rather
|
||||
than "running" — the 2026-09-24 fallback went unseen for weeks because
|
||||
only this agent's own log said so. None before report() has run."""
|
||||
if not LAST:
|
||||
return None
|
||||
out = {}
|
||||
for name, s in LAST.items():
|
||||
entry = {"device": s.get("device")}
|
||||
if s.get("error"):
|
||||
entry["error"] = str(s["error"])[:200]
|
||||
out[name] = entry
|
||||
return out
|
||||
|
||||
|
||||
def report() -> dict:
|
||||
"""Check both runtimes, log the result, and keep it for /status."""
|
||||
LAST.clear()
|
||||
LAST.update(torch=torch_status(), onnx=onnx_status())
|
||||
for name, s in LAST.items():
|
||||
if s.get("device") == "cuda":
|
||||
log.info("accel: %s on GPU (%s)", name, s)
|
||||
else:
|
||||
log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s)
|
||||
return dict(LAST)
|
||||
+28
-3
@@ -11,7 +11,7 @@ import logging
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import HTMLResponse, JSONResponse
|
||||
|
||||
from . import logbuf
|
||||
from . import accel, logbuf
|
||||
from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version
|
||||
from .config import Config
|
||||
from .gpu import read_gpu
|
||||
@@ -47,6 +47,9 @@ async def _no_store(request, call_next):
|
||||
|
||||
@app.on_event("startup")
|
||||
def _maybe_autostart() -> None:
|
||||
# Before the worker: the report also preloads the CUDA libraries the ONNX
|
||||
# models need, and it says in the log which runtimes landed on the GPU.
|
||||
accel.report()
|
||||
# With AUTO_START set, a container restart (host reboot, or `restart:
|
||||
# unless-stopped` after a crash) resumes the worker on its own — the slots
|
||||
# then ride out a still-down curator via lease backoff. Lets the agent
|
||||
@@ -137,6 +140,7 @@ def status():
|
||||
s["version"] = FC_VERSION or None
|
||||
s["channel"] = FC_CHANNEL or None
|
||||
s["revision"] = FC_REVISION or None
|
||||
s["accel"] = accel.LAST or None
|
||||
return JSONResponse(s)
|
||||
|
||||
|
||||
@@ -188,7 +192,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
|
||||
.step:hover{border-color:var(--acc)}
|
||||
#conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
|
||||
color:var(--fg);border:1px solid var(--bd);border-radius:8px}
|
||||
color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
|
||||
/* The browser's own spin arrows, hidden: the − / + beside each field are the
|
||||
control, styled like the rest of the page (operator, 2026-09-24). */
|
||||
#conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
|
||||
#bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
|
||||
.unit{color:var(--mut);font-size:12px;font-weight:600}
|
||||
.hint{color:var(--mut);font-size:12px;margin-top:12px}
|
||||
.tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
|
||||
@@ -227,6 +235,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
<div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
|
||||
a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
|
||||
</div>
|
||||
<div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
|
||||
<div id=banner class=banner style=display:none>
|
||||
curator unreachable — holding work + retrying, resumes on its own (no restart needed)
|
||||
</div>
|
||||
@@ -244,7 +253,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
<button class=step onclick=setc(1)>+</button>
|
||||
</div>
|
||||
<div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
|
||||
<button class=step onclick=stepbw(-1)>−</button>
|
||||
<input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
|
||||
<button class=step onclick=stepbw(1)>+</button>
|
||||
<span class=unit>MB/s</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -312,6 +323,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
body:JSON.stringify({value:on})});refresh()
|
||||
}
|
||||
function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
|
||||
// Runtimes that did NOT get the GPU, from the startup report. Both fall back
|
||||
// to the CPU without raising, so this banner and the pill are the only place
|
||||
// on this page a slow, CPU-bound agent announces itself.
|
||||
function cpuOnly(s){
|
||||
const a=s.accel||{}
|
||||
return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
|
||||
}
|
||||
async function setbw(v){
|
||||
v=Math.max(0,parseFloat(v)||0); bw.value=v
|
||||
await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||
@@ -382,11 +401,17 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
||||
// unreachable curator; grey when stopped; red with no token.
|
||||
let dc='dot', lbl='stopped'
|
||||
if(!ok){ dc='dot red'; lbl='no token' }
|
||||
else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
|
||||
else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
|
||||
if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
|
||||
else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
|
||||
else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
|
||||
dot.className=dc; connlbl.textContent=lbl
|
||||
banner.style.display=(st==='running' && !s.queue)?'block':'none'
|
||||
const slow=cpuOnly(s)
|
||||
accelbanner.style.display=slow.length?'block':'none'
|
||||
accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
|
||||
+slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
|
||||
+'. After a driver update, regenerate the CDI spec (agent README).'):''
|
||||
queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from . import accel
|
||||
|
||||
|
||||
class FcClient:
|
||||
def __init__(self, base_url: str, token: str, agent_id: str):
|
||||
@@ -72,7 +74,10 @@ class FcClient:
|
||||
def lease(self, batch_size: int) -> list[dict]:
|
||||
r = self.s.post(
|
||||
f"{self.base}/api/gpu/jobs/lease",
|
||||
json={"agent_id": self.agent_id, "batch_size": batch_size},
|
||||
json={
|
||||
"agent_id": self.agent_id, "batch_size": batch_size,
|
||||
"accel": accel.summary(),
|
||||
},
|
||||
timeout=30,
|
||||
)
|
||||
r.raise_for_status()
|
||||
@@ -90,7 +95,9 @@ class FcClient:
|
||||
})
|
||||
|
||||
def heartbeat(self, job_ids: list[int]) -> None:
|
||||
self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
|
||||
self._post_quiet(
|
||||
"/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
|
||||
)
|
||||
|
||||
def fail(self, job_id: int, error: str) -> None:
|
||||
self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace).
|
||||
dghs-imgutils>=0.4
|
||||
# GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
|
||||
# server-side fallback run.
|
||||
onnxruntime-gpu
|
||||
# The crop EMBEDDER (concept bag). torch is installed separately in the
|
||||
# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic;
|
||||
# server-side fallback run. The extras declare the CUDA/cuDNN pip packages its
|
||||
# CUDA provider loads (fc_agent/accel.py preloads them) rather than relying on
|
||||
# torch happening to install the same ones.
|
||||
onnxruntime-gpu[cuda,cudnn]
|
||||
# The crop EMBEDDER (concept bag). torch + torchvision are installed separately
|
||||
# in the Dockerfile from the cu130 wheel index, so pip never swaps them out;
|
||||
# transformers loads whatever SigLIP-family model the server announces.
|
||||
transformers>=4.45
|
||||
# Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Discord native ingester ledgers — seen and dead-letter, per source.
|
||||
|
||||
Milestone 428, #4415. Discord moves off gallery-dl onto the native core, which
|
||||
keeps its memory of what a source has already fetched in these two tables
|
||||
instead of gallery-dl's archive. Same shape as the SubscribeStar pair.
|
||||
|
||||
Revision ID: 0111
|
||||
Revises: 0110
|
||||
Create Date: 2026-09-24
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "0111"
|
||||
down_revision = "0110"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table(
|
||||
"discord_seen_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("post_id", sa.String(length=64), nullable=True),
|
||||
sa.Column(
|
||||
"seen_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_discord_seen_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_seen_media")),
|
||||
sa.UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_discord_seen_media_source_id"), "discord_seen_media", ["source_id"],
|
||||
)
|
||||
op.create_table(
|
||||
"discord_failed_media",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||
sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column(
|
||||
"first_failed_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"last_failed_at", sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"), nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["source_id"], ["source.id"],
|
||||
name=op.f("fk_discord_failed_media_source_id_source"), ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_failed_media")),
|
||||
sa.UniqueConstraint(
|
||||
"source_id", "filehash", name="uq_discord_failed_media_source_id",
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_discord_failed_media_source_id"), "discord_failed_media", ["source_id"],
|
||||
)
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.drop_index(op.f("ix_discord_failed_media_source_id"), table_name="discord_failed_media")
|
||||
op.drop_table("discord_failed_media")
|
||||
op.drop_index(op.f("ix_discord_seen_media_source_id"), table_name="discord_seen_media")
|
||||
op.drop_table("discord_seen_media")
|
||||
@@ -19,6 +19,7 @@ from ..models import AppSetting
|
||||
from ..services.extension_service import (
|
||||
ExtensionService,
|
||||
InvalidUrlError,
|
||||
UnknownArtistError,
|
||||
UnknownPlatformError,
|
||||
)
|
||||
from ..services.source_service import KNOWN_PLATFORMS
|
||||
@@ -87,10 +88,16 @@ async def probe_source():
|
||||
url = (request.args.get("url") or "").strip()
|
||||
if not url:
|
||||
return _bad("invalid_body", detail="url query parameter is required")
|
||||
from .credentials import _get_crypto
|
||||
|
||||
async with get_session() as session:
|
||||
if not await _ext_key_required(session):
|
||||
return _bad("unauthorized", status=401)
|
||||
result = await ExtensionService(session).probe(url)
|
||||
# crypto lets a Discord probe name the server and channel with the
|
||||
# stored token; every other platform ignores it.
|
||||
result = await ExtensionService(session, _get_crypto()).probe(
|
||||
url, names=request.args.get("names") in ("1", "true"),
|
||||
)
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@@ -102,6 +109,18 @@ async def quick_add_source():
|
||||
url = body.get("url")
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
return _bad("invalid_body", detail="url is required")
|
||||
# Optional: connect the new source to an existing artist (artist_id) or to
|
||||
# the artist of that name (artist_name). A Discord channel names no
|
||||
# creator, so the extension's Add panel always sends one of them.
|
||||
artist_id = body.get("artist_id")
|
||||
if artist_id is not None and (isinstance(artist_id, bool) or not isinstance(artist_id, int)):
|
||||
return _bad("invalid_body", detail="artist_id must be an integer")
|
||||
artist_name = body.get("artist_name")
|
||||
if artist_name is not None and not isinstance(artist_name, str):
|
||||
return _bad("invalid_body", detail="artist_name must be a string")
|
||||
# Patreon is canon: adding a Patreon source to an existing artist can take
|
||||
# the creator's Patreon display name (name only; the slug never moves).
|
||||
use_platform_name = body.get("use_platform_name") is True
|
||||
|
||||
from .credentials import _get_crypto
|
||||
|
||||
@@ -111,7 +130,12 @@ async def quick_add_source():
|
||||
try:
|
||||
# crypto lets an add resolve the artist's display name via the
|
||||
# stored credential (else it falls back to the URL handle). #130.
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
|
||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(
|
||||
url, artist_id=artist_id, artist_name=artist_name,
|
||||
use_platform_name=use_platform_name,
|
||||
)
|
||||
except UnknownArtistError as exc:
|
||||
return _bad("not_found", detail=str(exc), status=404)
|
||||
except UnknownPlatformError as exc:
|
||||
return _bad(
|
||||
"unknown_platform",
|
||||
|
||||
+31
-2
@@ -245,6 +245,29 @@ async def errors_recover(image_id: int):
|
||||
|
||||
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
||||
|
||||
|
||||
def _accel_detail(body: dict) -> dict:
|
||||
"""The agent's own report of which runtime got the GPU, kept on its roster
|
||||
row so the System view can call a CPU-bound agent degraded (#4410).
|
||||
|
||||
Only a dict of {runtime: {device, error?}} is kept, and each value is
|
||||
reduced to those two short strings: this is written on every lease, by a
|
||||
client the server does not control. An agent that sends nothing (an
|
||||
older build) simply has no `accel`, which reads as not-yet-reported.
|
||||
"""
|
||||
raw = body.get("accel")
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
accel = {}
|
||||
for name, entry in list(raw.items())[:4]:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
clean = {"device": str(entry.get("device") or "")[:16]}
|
||||
if entry.get("error"):
|
||||
clean["error"] = str(entry["error"])[:200]
|
||||
accel[str(name)[:16]] = clean
|
||||
return {"accel": accel} if accel else {}
|
||||
|
||||
@gpu_bp.route("/jobs/lease", methods=["POST"])
|
||||
async def lease():
|
||||
body = await request.get_json(silent=True) or {}
|
||||
@@ -267,7 +290,10 @@ async def lease():
|
||||
key=f"agent:{agent_id}",
|
||||
kind="agent",
|
||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||
details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)},
|
||||
details={
|
||||
"agent_id": agent_id, "last_call": "lease", "leased": len(jobs),
|
||||
**_accel_detail(body),
|
||||
},
|
||||
)
|
||||
ml = await MLSettings.load(session)
|
||||
# image rows for url/mime in one shot
|
||||
@@ -347,7 +373,10 @@ async def heartbeat():
|
||||
key=f"agent:{agent_id}",
|
||||
kind="agent",
|
||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||
details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n},
|
||||
details={
|
||||
"agent_id": agent_id, "last_call": "heartbeat", "extended": n,
|
||||
**_accel_detail(body),
|
||||
},
|
||||
)
|
||||
await session.commit()
|
||||
return jsonify({"extended": n})
|
||||
|
||||
@@ -72,9 +72,14 @@ assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, (
|
||||
PROBE_TIMEOUT_SECONDS = 2.0
|
||||
|
||||
_OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
|
||||
# Checking in, but working at a fraction of its speed: a GPU agent whose
|
||||
# runtimes fell back to the CPU (#4410). Below stale — a part that may have
|
||||
# stopped is the more urgent question — and above unknown, because this one
|
||||
# IS known to be wrong.
|
||||
_DEGRADED = "degraded"
|
||||
|
||||
# Worst-first, so an overall verdict is just the max.
|
||||
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3}
|
||||
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4}
|
||||
|
||||
|
||||
def _age_state(age_seconds: float) -> str:
|
||||
@@ -100,6 +105,39 @@ def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
|
||||
return f"{name} has not checked in for {ago} — treat it as stopped"
|
||||
|
||||
|
||||
def _cpu_runtimes(details: dict) -> list[str]:
|
||||
"""The runtimes an agent reported as NOT on the GPU, with why.
|
||||
|
||||
Both torch and onnxruntime fall back to the CPU without raising, so an
|
||||
agent in that state leases, works and checks in exactly like a healthy
|
||||
one. On 2026-09-24 one had been doing so since a driver update left a
|
||||
stale CDI spec; the only sign was a line in the agent's own log.
|
||||
"""
|
||||
accel = details.get("accel")
|
||||
if not isinstance(accel, dict):
|
||||
return []
|
||||
out = []
|
||||
for name, entry in sorted(accel.items()):
|
||||
if not isinstance(entry, dict) or entry.get("device") == "cuda":
|
||||
continue
|
||||
why = entry.get("error") or entry.get("device") or "unknown"
|
||||
out.append(f"{name} ({why})")
|
||||
return out
|
||||
|
||||
|
||||
def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]:
|
||||
"""A roster row's state and its sentence, degraded included."""
|
||||
if state == _OK:
|
||||
cpu = _cpu_runtimes(details)
|
||||
if cpu:
|
||||
return _DEGRADED, (
|
||||
f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. "
|
||||
"After a driver update, regenerate the agent host's CDI spec "
|
||||
"(agent README)."
|
||||
)
|
||||
return state, _describe_learned(name, state, age, details)
|
||||
|
||||
|
||||
async def _probe_postgres(session) -> dict:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
@@ -175,13 +213,15 @@ async def system_health():
|
||||
).scalars().all()
|
||||
for row in rows:
|
||||
age = (now - row.last_seen_at).total_seconds()
|
||||
state = _age_state(age)
|
||||
state, detail = _learned_state(
|
||||
row.display_name, _age_state(age), age, row.details or {},
|
||||
)
|
||||
parts.append({
|
||||
"key": row.key,
|
||||
"kind": row.kind,
|
||||
"name": row.display_name,
|
||||
"state": state,
|
||||
"detail": _describe_learned(row.display_name, state, age, row.details or {}),
|
||||
"detail": detail,
|
||||
"last_seen_at": row.last_seen_at.isoformat(),
|
||||
"first_seen_at": row.first_seen_at.isoformat(),
|
||||
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Celery beat that remembers when each job last ran — from task_run, not a file.
|
||||
|
||||
Celery's default PersistentScheduler keeps its memory in a shelve file in the
|
||||
working directory. Nothing mounts that directory, so every container recreate
|
||||
forgets it, and a scheduler that remembers nothing seeds every entry with
|
||||
`last_run_at = now`: each job waits a FULL interval after startup. A daily job
|
||||
therefore needs 24 hours without a redeploy to fire. Since the one-container
|
||||
image (172e33d) every redeploy restarts beat, and on 2026-09-24 no daily or
|
||||
weekly job had run since the 21st (#4408).
|
||||
|
||||
task_run already records every task that starts (celery_signals), indexed on
|
||||
(task_name, started_at DESC), and prune_task_runs keeps the newest row of each
|
||||
task however old it is. So on startup each entry takes its last_run_at from
|
||||
there:
|
||||
- a job that is overdue runs at once;
|
||||
- a job that is not due waits only the remainder of its interval;
|
||||
- a job that has never run is due now.
|
||||
|
||||
Beat keeps last_run_at in memory from then on, as the default scheduler does;
|
||||
only the startup seed changes. If the database cannot be read, the entries keep
|
||||
Celery's own default rather than beat failing to start.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from celery.beat import Scheduler
|
||||
from sqlalchemy import func, select
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Seed for a job with no recorded run: far enough back that any interval or
|
||||
# crontab reads as due.
|
||||
NEVER = datetime(2000, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def last_runs(session, task_names: list[str]) -> dict[str, datetime]:
|
||||
"""The latest recorded start of each task, by task name."""
|
||||
from .models import TaskRun
|
||||
|
||||
if not task_names:
|
||||
return {}
|
||||
rows = session.execute(
|
||||
select(TaskRun.task_name, func.max(TaskRun.started_at))
|
||||
.where(TaskRun.task_name.in_(sorted(set(task_names))))
|
||||
.group_by(TaskRun.task_name)
|
||||
).all()
|
||||
return dict(rows)
|
||||
|
||||
|
||||
def seed(entries, last: dict[str, datetime]) -> None:
|
||||
"""Set each entry's last_run_at from `last`; a task with none is due now."""
|
||||
for entry in entries:
|
||||
entry.last_run_at = last.get(entry.task, NEVER)
|
||||
|
||||
|
||||
class TaskRunScheduler(Scheduler):
|
||||
"""An in-memory beat seeded from task_run history at startup."""
|
||||
|
||||
def setup_schedule(self):
|
||||
super().setup_schedule()
|
||||
try:
|
||||
from .tasks._sync_engine import sync_session_factory
|
||||
|
||||
with sync_session_factory()() as session:
|
||||
last = last_runs(session, [e.task for e in self.schedule.values()])
|
||||
except Exception:
|
||||
log.exception("beat: could not read task_run; every job waits a full interval")
|
||||
return
|
||||
seed(self.schedule.values(), last)
|
||||
due = sum(1 for e in self.schedule.values() if e.is_due()[0])
|
||||
log.info(
|
||||
"beat: seeded %d job(s) from task_run, %d due now",
|
||||
len(self.schedule), due,
|
||||
)
|
||||
@@ -62,6 +62,13 @@ def make_celery() -> Celery:
|
||||
# can never starve the quick self-healing sweeps (operator-flagged
|
||||
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||
# The one long job in maintenance.py: a whole-library phash
|
||||
# recompute (35 min hard limit; the library was cleared for
|
||||
# re-hashing by migration 0098). On the quick lane it held a
|
||||
# scheduler process for its whole run, and the minute ticks queued
|
||||
# up behind it (2026-09-24: 7 waiting, "all workers busy for 18
|
||||
# minutes"). An exact name wins over the glob above.
|
||||
"backend.app.tasks.maintenance.backfill_phash": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||
@@ -353,6 +360,9 @@ def make_celery() -> Celery:
|
||||
},
|
||||
},
|
||||
timezone="UTC",
|
||||
# Beat's memory of when each job last ran comes from task_run, not a
|
||||
# shelve file nothing persists — see beat_scheduler (#4408).
|
||||
beat_scheduler="backend.app.beat_scheduler:TaskRunScheduler",
|
||||
)
|
||||
# FC-3i: register task_run signal handlers (side-effect import).
|
||||
from . import celery_signals # noqa: F401
|
||||
|
||||
@@ -8,6 +8,8 @@ from .backup_run import BackupRun
|
||||
from .base import Base
|
||||
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
||||
from .credential import Credential
|
||||
from .discord_failed_media import DiscordFailedMedia
|
||||
from .discord_seen_media import DiscordSeenMedia
|
||||
from .download_event import DownloadEvent
|
||||
from .external_link import ExternalLink
|
||||
from .gpu_job import GpuJob
|
||||
@@ -56,6 +58,8 @@ __all__ = [
|
||||
"BackupRun",
|
||||
"Source",
|
||||
"Credential",
|
||||
"DiscordFailedMedia",
|
||||
"DiscordSeenMedia",
|
||||
"PatreonFailedMedia",
|
||||
"PatreonSeenMedia",
|
||||
"SubscribeStarFailedMedia",
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""DiscordFailedMedia — per-source dead-letter ledger of Discord files that
|
||||
keep failing to download or validate.
|
||||
|
||||
Mirror of SubscribeStarFailedMedia. After `attempts` reaches the dead-letter
|
||||
threshold a routine walk skips the file (recovery still retries it); a later
|
||||
clean download clears the row. `filehash` is the seen-ledger's key.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class DiscordFailedMedia(Base):
|
||||
__tablename__ = "discord_failed_media"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "filehash", name="uq_discord_failed_media_source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
first_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
last_failed_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -0,0 +1,34 @@
|
||||
"""DiscordSeenMedia — per-source ledger of Discord files already downloaded.
|
||||
|
||||
Mirror of SubscribeStarSeenMedia for the native Discord ingester (milestone
|
||||
428). `filehash` holds the ingester's per-file key, `<message_id>:<media_id>`:
|
||||
the attachment id, or for an embed a hash of its URL path. Not the file's
|
||||
position in the message — an edit that removes a file renumbers the rest
|
||||
(see `discord_client.MediaItem`). The message record's own gate is the
|
||||
synthetic `message:<id>` key in the same column.
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
from sqlalchemy.types import DateTime
|
||||
|
||||
from .base import Base
|
||||
|
||||
|
||||
class DiscordSeenMedia(Base):
|
||||
__tablename__ = "discord_seen_media"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
source_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||
)
|
||||
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
seen_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||
)
|
||||
@@ -316,17 +316,32 @@ class ArtistService:
|
||||
cleaned = (prefix or "").strip()
|
||||
if not cleaned:
|
||||
return []
|
||||
like = f"%{cleaned.lower()}%"
|
||||
prefix_like = f"{cleaned.lower()}%"
|
||||
# Rank: exact (0) < prefix (1) < substring (2).
|
||||
low = cleaned.lower()
|
||||
like = f"%{low}%"
|
||||
prefix_like = f"{low}%"
|
||||
# Spacing- and punctuation-insensitive too, so "Tamada Heijun" finds
|
||||
# "TamadaHeijun" and "sabu_art" finds "Sabu Art" — the same creator is
|
||||
# spelled differently on every platform, and the browser extension's
|
||||
# Add panel matches a Discord server name against these (milestone 429).
|
||||
# [[:alnum:]] keeps non-Latin letters; a query with none (all
|
||||
# punctuation) skips this arm rather than matching every artist.
|
||||
squashed = "".join(ch for ch in low if ch.isalnum())
|
||||
name_squashed = func.regexp_replace(func.lower(Artist.name), "[^[:alnum:]]", "", "g")
|
||||
matches = [func.lower(Artist.name).like(like)]
|
||||
if squashed:
|
||||
matches.append(name_squashed.like(f"%{squashed}%"))
|
||||
# Rank: exact (0) < exact ignoring spacing (1) < prefix (2) <
|
||||
# substring (3) < substring ignoring spacing (4).
|
||||
rank = case(
|
||||
(func.lower(Artist.name) == cleaned.lower(), 0),
|
||||
(func.lower(Artist.name).like(prefix_like), 1),
|
||||
else_=2,
|
||||
(func.lower(Artist.name) == low, 0),
|
||||
(name_squashed == squashed, 1),
|
||||
(func.lower(Artist.name).like(prefix_like), 2),
|
||||
(func.lower(Artist.name).like(like), 3),
|
||||
else_=4,
|
||||
).label("rank")
|
||||
rows = (await self.session.execute(
|
||||
select(Artist, rank)
|
||||
.where(func.lower(Artist.name).like(like))
|
||||
.where(or_(*matches))
|
||||
.order_by(rank, Artist.name.asc())
|
||||
.limit(limit)
|
||||
)).all()
|
||||
|
||||
@@ -0,0 +1,562 @@
|
||||
"""Native Discord read client — the Discord counterpart to subscribestar_client.
|
||||
|
||||
Mirrors gallery-dl 1.32.13's `extractor/discord.py` (rule 130: gallery-dl is the
|
||||
known-working base), adapted to the native core's client contract
|
||||
(`ingest_core` module docstring): `iter_posts` / `extract_media`, plus the
|
||||
post-first `post_record_key` and the `post_meta` date the revisit window reads.
|
||||
|
||||
What is mirrored exactly, because drift in any of it changes what we fetch or
|
||||
where it lands on disk:
|
||||
|
||||
- API v10, `Authorization: <user token>` (a USER token, not a bot token).
|
||||
- gallery-dl's request profile: its date-derived Firefox User-Agent,
|
||||
`Accept: */*`, `Accept-Language`, `Referer: https://discord.com/`.
|
||||
- `GET /channels/{id}/messages?limit=100&before=<last id>`, newest first,
|
||||
stopping on a short page. Message types {0, 19, 21} only.
|
||||
- The walk: a text/news channel's own messages then its threads, a forum's
|
||||
threads, a category's children, a server's text/news/forum channels.
|
||||
- Files: attachments, then embeds of type image/gifv/video (FC configures
|
||||
`embeds: all`, which for files is the same three plus rich/link embeds that
|
||||
carry an image), then forwarded `message_snapshots`, numbered from 1 across
|
||||
the lot — the `num` in `{date}_{message_id}_{num}_{filename}`.
|
||||
- Text: `content`, rich-embed author/title/description/fields/footer, poll.
|
||||
|
||||
Two deliberate departures, both about the walk order, neither about content:
|
||||
|
||||
- Threads are walked newest-CREATED first (by id), not by last-message time.
|
||||
A backfill resumes from a checkpointed channel; last-message order shifts
|
||||
between chunks whenever someone posts, which can move an unwalked thread
|
||||
above the resume point and skip it. Creation order only ever grows at the
|
||||
front, where the next tick finds it.
|
||||
- A 403 on a thread or a nested channel skips that feed instead of failing
|
||||
the walk. gallery-dl skips only nested channels; one private thread the
|
||||
token cannot read would otherwise stop every channel after it.
|
||||
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from urllib.parse import unquote
|
||||
|
||||
import requests
|
||||
|
||||
from .native_ingest_common import (
|
||||
NativeAuthError,
|
||||
NativeDriftError,
|
||||
NativeIngestError,
|
||||
retry_after_seconds,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
API_ROOT = "https://discord.com/api/v10"
|
||||
_ROOT = "https://discord.com"
|
||||
|
||||
_TIMEOUT_SECONDS = 60.0
|
||||
_MESSAGES_BATCH = 100
|
||||
_THREADS_BATCH = 25
|
||||
# gallery-dl retries a 429 up to its default 4 retries, waiting
|
||||
# `request_interval_429` (60s) between them. Discord's Retry-After is exact, so
|
||||
# it is honoured when present; 60s is the fallback and the cap.
|
||||
_MAX_429_RETRIES = 4
|
||||
_429_WAIT_SECONDS = 60.0
|
||||
|
||||
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||||
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||||
MESSAGE_TYPES = frozenset({0, 19, 21})
|
||||
# https://discord.com/developers/docs/resources/channel#channel-object-channel-types
|
||||
_TEXT = frozenset({0, 5}) # text, announcement: messages + threads
|
||||
_DIRECT = frozenset({1, 3, 10, 11, 12}) # DMs and threads: messages only
|
||||
_FORUM = frozenset({15, 16}) # forum, media: threads only
|
||||
_CATEGORY = 4
|
||||
_SERVER_WALK = _TEXT | _FORUM
|
||||
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
|
||||
|
||||
_URL_RE = re.compile(
|
||||
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
|
||||
r"(?P<server>@me|\d+)(?:/(?:\d+/threads/)?(?P<channel>\d+))?(?P<rest>/.*)?/?$"
|
||||
)
|
||||
|
||||
|
||||
class DiscordAPIError(NativeIngestError):
|
||||
"""Base for native Discord client failures."""
|
||||
|
||||
|
||||
class DiscordAuthError(DiscordAPIError, NativeAuthError):
|
||||
"""401 (the token is invalid or expired) or a 403 on the channel the
|
||||
source names. The fix is a new token, not a new client."""
|
||||
|
||||
|
||||
class DiscordDriftError(DiscordAPIError, NativeDriftError):
|
||||
"""A response did not have the shape the walk depends on."""
|
||||
|
||||
|
||||
def firefox_user_agent(today: date | None = None) -> str:
|
||||
"""gallery-dl's default User-Agent: a Firefox whose version advances every
|
||||
four weeks (`util._ff_ver`, "147 on 2026-01-13"). Computed the same way so
|
||||
the profile keeps matching the gallery-dl this replaced."""
|
||||
ver = ((today or date.today()).toordinal() - 735_513) // 28
|
||||
return (
|
||||
f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:{ver}.0) "
|
||||
f"Gecko/20100101 Firefox/{ver}.0"
|
||||
)
|
||||
|
||||
|
||||
def nameext_from_url(url: str) -> tuple[str, str]:
|
||||
"""gallery-dl's `text.nameext_from_url`: the URL's last path segment,
|
||||
unquoted, split at the last dot when the extension is at most 16 chars
|
||||
(lowercased); otherwise the whole name and no extension."""
|
||||
filename = unquote(url.partition("?")[0].rpartition("/")[2])
|
||||
name, _, ext = filename.rpartition(".")
|
||||
if name and len(ext) <= 16:
|
||||
return name, ext.lower()
|
||||
return filename, ""
|
||||
|
||||
|
||||
def parse_source_url(url: str) -> tuple[str | None, str | None]:
|
||||
"""`(server_id, channel_id)` from a Discord channel/server URL. `server_id`
|
||||
is None for a DM (`@me`); `channel_id` is None for a whole server. Raises
|
||||
DiscordAPIError for anything else, including a link to a single message —
|
||||
a message is not something a source can subscribe to."""
|
||||
m = _URL_RE.match((url or "").strip())
|
||||
if not m or (m.group("rest") or "").strip("/"):
|
||||
raise DiscordAPIError(
|
||||
f"Not a Discord channel or server link: {url!r} "
|
||||
"(expected https://discord.com/channels/<server>[/<channel>])"
|
||||
)
|
||||
server = m.group("server")
|
||||
channel = m.group("channel")
|
||||
if server == "@me":
|
||||
if not channel:
|
||||
raise DiscordAPIError(f"A DM link needs a channel id: {url!r}")
|
||||
return None, channel
|
||||
return server, channel
|
||||
|
||||
|
||||
def message_text(message: dict) -> str:
|
||||
"""gallery-dl's `extract_message_text`: the body plus the text of rich
|
||||
embeds and polls, newline-joined, empties dropped."""
|
||||
parts = [message.get("content") or ""]
|
||||
for embed in message.get("embeds") or []:
|
||||
if embed.get("type") != "rich":
|
||||
continue
|
||||
parts.append((embed.get("author") or {}).get("name") or "")
|
||||
parts.append(embed.get("title") or "")
|
||||
parts.append(embed.get("description") or "")
|
||||
for fld in embed.get("fields") or []:
|
||||
parts.append(fld.get("name") or "")
|
||||
parts.append(fld.get("value") or "")
|
||||
parts.append((embed.get("footer") or {}).get("text") or "")
|
||||
poll = message.get("poll")
|
||||
if poll:
|
||||
parts.append(((poll.get("question") or {}).get("text")) or "")
|
||||
for answer in poll.get("answers") or []:
|
||||
parts.append(((answer.get("poll_media") or {}).get("text")) or "")
|
||||
return "\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaItem:
|
||||
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||||
split of the URL; `num` is its 1-based position across the message's files,
|
||||
which is what names it on disk.
|
||||
|
||||
`media_id` is what the seen-ledger keys on, and it is deliberately NOT
|
||||
`num`: an edit that removes a file renumbers the ones after it, and a
|
||||
positional key would then call a different file seen. It is the
|
||||
attachment's id, or for an embed (which has none) a hash of its URL path —
|
||||
the query string is a signature that changes on every fetch. `filehash` is
|
||||
always None; nothing in a signed CDN URL is a content hash."""
|
||||
|
||||
url: str
|
||||
filename: str
|
||||
extension: str
|
||||
kind: str
|
||||
post_id: str
|
||||
num: int
|
||||
media_id: str
|
||||
filehash: str | None = None
|
||||
|
||||
|
||||
class DiscordClient:
|
||||
"""Synchronous Discord API v10 read client for one user token."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
token: str | None,
|
||||
*,
|
||||
request_sleep: float = 0.0,
|
||||
max_retries: int = _MAX_429_RETRIES,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
self._session = session or requests.Session()
|
||||
self._session.headers.update({
|
||||
"User-Agent": firefox_user_agent(),
|
||||
"Accept": "*/*",
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Referer": _ROOT + "/",
|
||||
})
|
||||
if token:
|
||||
self._session.headers["Authorization"] = token
|
||||
self._token = token
|
||||
self._request_sleep = request_sleep or 0.0
|
||||
self._max_retries = max_retries
|
||||
self._server: dict = {}
|
||||
self._channels: dict[str, dict] = {}
|
||||
self._skip_feed = False
|
||||
|
||||
# -- request -----------------------------------------------------------
|
||||
|
||||
def _get(self, endpoint: str, params: dict | None = None):
|
||||
if not self._token:
|
||||
raise DiscordAuthError("No Discord token is configured for this source")
|
||||
if self._request_sleep > 0:
|
||||
time.sleep(self._request_sleep)
|
||||
url = API_ROOT + endpoint
|
||||
attempt = 0
|
||||
while True:
|
||||
try:
|
||||
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
||||
except requests.RequestException as exc:
|
||||
raise DiscordAPIError(f"Discord request failed ({endpoint}): {exc}") from exc
|
||||
if resp.status_code == 429 and attempt < self._max_retries:
|
||||
attempt += 1
|
||||
delay = retry_after_seconds(
|
||||
resp, attempt, base=_429_WAIT_SECONDS, cap=_429_WAIT_SECONDS,
|
||||
)
|
||||
log.warning(
|
||||
"Discord 429 (%s) — waiting %.1fs (retry %d/%d)",
|
||||
endpoint, delay, attempt, self._max_retries,
|
||||
)
|
||||
time.sleep(delay)
|
||||
continue
|
||||
break
|
||||
if resp.status_code == 401:
|
||||
raise DiscordAuthError(
|
||||
"Discord rejected the token (HTTP 401) — it is invalid or has "
|
||||
"expired; copy a fresh one from the browser",
|
||||
status_code=401,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise DiscordAPIError(
|
||||
f"Discord returned HTTP {resp.status_code} ({endpoint})",
|
||||
status_code=resp.status_code,
|
||||
retry_after=_retry_after(resp),
|
||||
)
|
||||
try:
|
||||
return resp.json()
|
||||
except ValueError as exc:
|
||||
raise DiscordDriftError(
|
||||
f"Discord returned non-JSON for {endpoint} ({len(resp.content)} bytes)"
|
||||
) from exc
|
||||
|
||||
# -- metadata (gallery-dl parse_server / parse_channel) -----------------
|
||||
|
||||
def _load_server(self, server_id: str) -> None:
|
||||
server = self._get(f"/guilds/{server_id}")
|
||||
if not isinstance(server, dict) or "id" not in server:
|
||||
raise DiscordDriftError(f"Discord server {server_id} came back without an id")
|
||||
self._server = {
|
||||
"server": server.get("name") or "",
|
||||
"server_id": str(server["id"]),
|
||||
"owner_id": server.get("owner_id"),
|
||||
}
|
||||
channels = self._get(f"/guilds/{server_id}/channels")
|
||||
if not isinstance(channels, list):
|
||||
raise DiscordDriftError(f"Discord server {server_id} channel list is not a list")
|
||||
# Categories first, so every child can name its parent.
|
||||
for channel in sorted(channels, key=lambda ch: ch.get("type") != _CATEGORY):
|
||||
self._parse_channel(channel)
|
||||
|
||||
def _parse_channel(self, channel: dict) -> dict:
|
||||
parent_id = channel.get("parent_id")
|
||||
meta = {
|
||||
"channel": channel.get("name") or "",
|
||||
"channel_id": str(channel.get("id")),
|
||||
"channel_type": channel.get("type"),
|
||||
"channel_topic": channel.get("topic") or "",
|
||||
"parent_id": parent_id,
|
||||
"is_thread": "thread_metadata" in channel,
|
||||
}
|
||||
parent = self._channels.get(parent_id) if parent_id else None
|
||||
if parent:
|
||||
meta["parent"] = parent["channel"]
|
||||
meta["parent_type"] = parent["channel_type"]
|
||||
if meta["channel_type"] in {1, 3}:
|
||||
recipients = channel.get("recipients") or []
|
||||
meta["channel"] = "DMs"
|
||||
meta["recipients"] = [u.get("username") for u in recipients]
|
||||
meta["recipients_id"] = [u.get("id") for u in recipients]
|
||||
self._channels[meta["channel_id"]] = meta
|
||||
return meta
|
||||
|
||||
def _channel_meta(self, channel_id: str) -> dict:
|
||||
if channel_id not in self._channels:
|
||||
self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||||
return self._channels[channel_id]
|
||||
|
||||
def _threads(self, channel_id: str) -> list[dict]:
|
||||
"""Every thread of a channel or forum, newest-created first (see the
|
||||
module docstring for why not last-message order)."""
|
||||
threads: list[dict] = []
|
||||
offset = 0
|
||||
while True:
|
||||
data = self._get(f"/channels/{channel_id}/threads/search", {
|
||||
"sort_by": "last_message_time",
|
||||
"sort_order": "desc",
|
||||
"limit": _THREADS_BATCH,
|
||||
"offset": offset,
|
||||
})
|
||||
batch = (data.get("threads") or []) if isinstance(data, dict) else []
|
||||
threads.extend(batch)
|
||||
if len(batch) < _THREADS_BATCH:
|
||||
break
|
||||
offset += len(batch)
|
||||
threads.sort(key=lambda t: int(t.get("id") or 0), reverse=True)
|
||||
return threads
|
||||
|
||||
# -- the walk ------------------------------------------------------------
|
||||
|
||||
def _feeds(self, channel_id: str, *, safe: bool) -> Iterator[tuple[str, bool]]:
|
||||
"""`(channel_id, safe)` for every message feed under `channel_id`, in
|
||||
gallery-dl's order. `safe` feeds are skipped on a 403."""
|
||||
try:
|
||||
ctype = self._channel_meta(channel_id)["channel_type"]
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code != 403:
|
||||
raise
|
||||
if not safe:
|
||||
raise DiscordAuthError(
|
||||
f"The Discord token cannot see channel {channel_id} (HTTP 403)",
|
||||
status_code=403,
|
||||
) from exc
|
||||
log.info("Discord: no access to channel %s — skipped", channel_id)
|
||||
return
|
||||
if ctype in _TEXT or ctype in _DIRECT:
|
||||
yield channel_id, safe
|
||||
if ctype in _TEXT or ctype in _FORUM:
|
||||
try:
|
||||
threads = self._threads(channel_id)
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code != 403:
|
||||
raise
|
||||
log.info("Discord: cannot list threads of %s — skipped", channel_id)
|
||||
threads = []
|
||||
for thread in threads:
|
||||
yield self._parse_channel(thread)["channel_id"], True
|
||||
elif ctype == _CATEGORY:
|
||||
for child in list(self._channels.values()):
|
||||
if child.get("parent_id") == channel_id:
|
||||
yield from self._feeds(child["channel_id"], safe=True)
|
||||
elif ctype not in _DIRECT and not safe:
|
||||
raise DiscordAPIError(
|
||||
f"Discord channel {channel_id} is of type {ctype}, which has no messages"
|
||||
)
|
||||
|
||||
def _source_feeds(self, url: str) -> Iterator[tuple[str, bool]]:
|
||||
server_id, channel_id = parse_source_url(url)
|
||||
self._server, self._channels = {}, {}
|
||||
if server_id is not None:
|
||||
self._load_server(server_id)
|
||||
if channel_id is not None:
|
||||
yield from self._feeds(channel_id, safe=False)
|
||||
return
|
||||
for meta in list(self._channels.values()):
|
||||
if meta["channel_type"] in _SERVER_WALK:
|
||||
yield from self._feeds(meta["channel_id"], safe=True)
|
||||
|
||||
def skip_feed(self) -> None:
|
||||
"""Optional core seam (#4413): end the current channel and go on to the
|
||||
next one. A tick's early-out means THIS channel has nothing new, not
|
||||
that the server has nothing new."""
|
||||
self._skip_feed = True
|
||||
|
||||
def iter_posts(
|
||||
self, campaign_id: str, cursor: str | None = None
|
||||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||
"""Yield `(message, channel_meta, page_cursor)` for every content
|
||||
message the source reaches, channel by channel, each newest first.
|
||||
|
||||
`campaign_id` is the source URL. The cursor is `<channel_id>:<before>`
|
||||
— the channel and the `before` id that fetched the page (empty for a
|
||||
channel's first page) — so a backfill resumes inside the right channel
|
||||
and re-fetches the page it was cut in. A cursor naming a channel the
|
||||
walk no longer reaches (a deleted thread) restarts from the top rather
|
||||
than walking nothing.
|
||||
"""
|
||||
resume_channel, _, resume_before = (cursor or "").partition(":")
|
||||
resuming = bool(resume_channel)
|
||||
feeds = list(self._source_feeds(campaign_id)) if resuming else None
|
||||
if feeds is not None and resume_channel not in {cid for cid, _ in feeds}:
|
||||
log.warning(
|
||||
"Discord: resume channel %s is no longer in %s — restarting",
|
||||
resume_channel, campaign_id,
|
||||
)
|
||||
resuming = False
|
||||
for channel_id, safe in feeds if feeds is not None else self._source_feeds(campaign_id):
|
||||
before = None
|
||||
if resuming:
|
||||
if channel_id != resume_channel:
|
||||
continue
|
||||
resuming = False
|
||||
before = resume_before or None
|
||||
yield from self._iter_channel(channel_id, before, safe=safe)
|
||||
|
||||
def _iter_channel(
|
||||
self, channel_id: str, before: str | None, *, safe: bool
|
||||
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||
self._skip_feed = False
|
||||
meta = {**self._server, **self._channels.get(channel_id, {})}
|
||||
while True:
|
||||
page_cursor = f"{channel_id}:{before or ''}"
|
||||
try:
|
||||
messages = self._get(
|
||||
f"/channels/{channel_id}/messages",
|
||||
{"limit": _MESSAGES_BATCH, "before": before},
|
||||
)
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code != 403:
|
||||
raise
|
||||
if not safe:
|
||||
raise DiscordAuthError(
|
||||
f"The Discord token cannot read channel {channel_id} (HTTP 403)",
|
||||
status_code=403,
|
||||
) from exc
|
||||
log.info("Discord: no access to messages of %s — skipped", channel_id)
|
||||
return
|
||||
if not isinstance(messages, list):
|
||||
raise DiscordDriftError(
|
||||
f"Discord messages of {channel_id} came back as "
|
||||
f"{type(messages).__name__}, not a list"
|
||||
)
|
||||
for message in messages:
|
||||
if message.get("type") not in MESSAGE_TYPES:
|
||||
continue
|
||||
message["_meta"] = meta
|
||||
yield message, meta, page_cursor
|
||||
if self._skip_feed:
|
||||
return
|
||||
if len(messages) < _MESSAGES_BATCH:
|
||||
return
|
||||
before = str(messages[-1]["id"])
|
||||
|
||||
# -- per-message -------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
|
||||
"""gallery-dl's file list for one message: attachments, then the first
|
||||
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
|
||||
the same for every forwarded snapshot; numbered from 1 across them."""
|
||||
mid = str(post.get("id") or "")
|
||||
snapshots = [post] + [
|
||||
(s or {}).get("message") or {}
|
||||
for s in post.get("message_snapshots") or []
|
||||
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||
]
|
||||
found: list[tuple[str, str, str | None]] = []
|
||||
for snap in snapshots:
|
||||
for att in snap.get("attachments") or []:
|
||||
if att.get("url"):
|
||||
aid = att.get("id")
|
||||
found.append((att["url"], "attachment", str(aid) if aid else None))
|
||||
for embed in snap.get("embeds") or []:
|
||||
if embed.get("type") not in _EMBED_TYPES:
|
||||
continue
|
||||
for fld in ("video", "image", "thumbnail"):
|
||||
url = (embed.get(fld) or {}).get("proxy_url")
|
||||
if url:
|
||||
found.append((url, "embed", None))
|
||||
break
|
||||
items = []
|
||||
for num, (url, kind, fid) in enumerate(found, start=1):
|
||||
name, ext = nameext_from_url(url)
|
||||
if fid is None:
|
||||
path = url.partition("?")[0].encode()
|
||||
fid = "u" + hashlib.sha1(path, usedforsecurity=False).hexdigest()[:32]
|
||||
items.append(MediaItem(
|
||||
url=url, filename=name, extension=ext, kind=kind, post_id=mid,
|
||||
num=num, media_id=fid,
|
||||
))
|
||||
return items
|
||||
|
||||
@staticmethod
|
||||
def post_meta(post: dict) -> dict:
|
||||
"""No title (Discord has none); `date` is the message timestamp, ISO
|
||||
with an offset — what the core's revisit window reads."""
|
||||
return {"title": None, "date": post.get("timestamp")}
|
||||
|
||||
@classmethod
|
||||
def post_record_key(cls, post: dict) -> tuple[str, str] | None:
|
||||
"""`(message:<id>, <id>)` — gates the message record through the seen
|
||||
ledger, like `post:<id>` on the other platforms.
|
||||
|
||||
None for a message with no files. gallery-dl wrote a sidecar only
|
||||
beside a file, so a text-only chat line never became a post, and the
|
||||
drop grouping (discord_grouping) is built on that: a channel's chatter
|
||||
recorded as posts would bury the drops it exists to surface."""
|
||||
mid = post.get("id")
|
||||
mid = str(mid) if mid is not None else ""
|
||||
if not mid or not cls.extract_media(post):
|
||||
return None
|
||||
return (f"message:{mid}", mid)
|
||||
|
||||
# -- verify ------------------------------------------------------------
|
||||
|
||||
def describe(self, server_id: str | None, channel_id: str | None) -> dict:
|
||||
"""The display names behind a server/channel pair, for the browser
|
||||
extension's Add panel. Best-effort per name: one that can't be read
|
||||
comes back None, and the other is still returned."""
|
||||
out: dict = {"server": None, "channel": None, "parent": None}
|
||||
if server_id:
|
||||
try:
|
||||
out["server"] = (self._get(f"/guilds/{server_id}") or {}).get("name") or None
|
||||
except DiscordAPIError:
|
||||
pass
|
||||
if channel_id:
|
||||
try:
|
||||
meta = self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||||
out["channel"] = meta.get("channel") or None
|
||||
except (DiscordAPIError, AttributeError):
|
||||
pass
|
||||
return out
|
||||
|
||||
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||||
"""Is the token valid, and can it see what the source names?"""
|
||||
try:
|
||||
server_id, channel_id = parse_source_url(url)
|
||||
except DiscordAPIError as exc:
|
||||
return None, str(exc)
|
||||
try:
|
||||
me = self._get("/users/@me")
|
||||
if channel_id is not None:
|
||||
self._get(f"/channels/{channel_id}")
|
||||
elif server_id is not None:
|
||||
self._get(f"/guilds/{server_id}")
|
||||
except DiscordAuthError as exc:
|
||||
return False, f"Discord rejected the token — {exc}"
|
||||
except DiscordAPIError as exc:
|
||||
if exc.status_code in (403, 404):
|
||||
return False, (
|
||||
"The token is valid, but its account cannot see "
|
||||
f"{'this channel' if channel_id else 'this server'} "
|
||||
f"(HTTP {exc.status_code})"
|
||||
)
|
||||
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||||
who = (me or {}).get("username") if isinstance(me, dict) else None
|
||||
return True, f"Token valid{f' ({who})' if who else ''} — the source is readable."
|
||||
|
||||
|
||||
def _retry_after(resp: requests.Response) -> float | None:
|
||||
hdr = resp.headers.get("Retry-After")
|
||||
try:
|
||||
return float(hdr) if hdr else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,212 @@
|
||||
"""Native Discord media downloader — the Discord counterpart to
|
||||
subscribestar_downloader.
|
||||
|
||||
Writes files exactly where gallery-dl wrote them, so a cutover finds every
|
||||
existing file on disk (`skipped_disk`) instead of fetching it again:
|
||||
|
||||
<images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>
|
||||
|
||||
That is what FC's gallery-dl config produced (directory `{channel}`, filename
|
||||
`{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the
|
||||
per-source base directory `<images>/<artist>/<platform>`), retired from that
|
||||
config once Discord moved here; tests/test_discord_naming.py pins the match
|
||||
against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it
|
||||
on Linux — `/` becomes `_` and control characters are removed, nothing else
|
||||
(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
|
||||
whose Windows set would turn a `:` in a channel or file name into `_` and miss
|
||||
the file gallery-dl wrote.
|
||||
|
||||
Post-first (rule 120): each file gets a minimal sidecar named like it minus the
|
||||
extension (what `find_sidecar` pairs first), and the message itself gets one
|
||||
record, `<YYYYMMDD>_<message_id>_post.json`, carrying gallery-dl's metadata keys
|
||||
— `message_id` for the post id, `server_id`/`channel_id` for the permalink,
|
||||
`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read
|
||||
the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both
|
||||
outrank `message_id` in the post-id chain (`platforms.base`).
|
||||
|
||||
PURE: no DB; the seen-skip is an injected predicate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from .discord_client import firefox_user_agent, message_text
|
||||
from .native_ingest_common import (
|
||||
BaseNativeDownloader,
|
||||
MediaOutcome,
|
||||
PostRecordOutcome,
|
||||
make_session,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
PLATFORM = "discord"
|
||||
_CONTROL = re.compile("[\x00-\x1f\x7f]")
|
||||
# gallery-dl falls back to the response's type for a URL with no extension;
|
||||
# we never see the response before naming, and such URLs do not occur for
|
||||
# Discord attachments or embed proxies in practice.
|
||||
_NO_EXTENSION = "bin"
|
||||
|
||||
|
||||
def gdl_clean(segment: str) -> str:
|
||||
"""One path segment as gallery-dl writes it on Linux."""
|
||||
return _CONTROL.sub("", segment.replace("/", "_"))
|
||||
|
||||
|
||||
def message_date(post: dict) -> datetime | None:
|
||||
raw = post.get("timestamp")
|
||||
if not isinstance(raw, str) or not raw:
|
||||
return None
|
||||
try:
|
||||
dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC)
|
||||
|
||||
|
||||
def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path:
|
||||
"""gallery-dl's `{channel}` directory; an empty name adds no segment."""
|
||||
base = Path(images_root) / artist_slug / PLATFORM
|
||||
channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip())
|
||||
return base / channel if channel else base
|
||||
|
||||
|
||||
def media_stem(post: dict, media) -> str:
|
||||
"""`<YYYYMMDD>_<message_id>_<NN>_<name>` — the file's name minus `.<ext>`."""
|
||||
when = message_date(post)
|
||||
day = f"{when:%Y%m%d}" if when else "None"
|
||||
return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}")
|
||||
|
||||
|
||||
class DiscordDownloader(BaseNativeDownloader):
|
||||
"""Download a message's files to gallery-dl's layout. The CDN gets
|
||||
gallery-dl's browser profile and no token — gallery-dl sends the token only
|
||||
to the API, and the CDN URLs are pre-signed."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None = None,
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
session: requests.Session | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
images_root, None, platform=PLATFORM,
|
||||
validate=validate, rate_limit=rate_limit,
|
||||
session=session if session is not None else make_session(None, extra_headers={
|
||||
"User-Agent": firefox_user_agent(),
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"Referer": "https://discord.com/",
|
||||
}),
|
||||
)
|
||||
|
||||
def download_post(
|
||||
self,
|
||||
post: dict,
|
||||
media_items: list,
|
||||
artist_slug: str,
|
||||
*,
|
||||
is_seen: Callable[[object], bool] = lambda m: False,
|
||||
should_stop: Callable[[], bool] = lambda: False,
|
||||
recapture: bool = False,
|
||||
) -> list[MediaOutcome]:
|
||||
"""Every file of one message; per-file outcomes, one failure isolated."""
|
||||
folder = channel_dir(self.images_root, artist_slug, post)
|
||||
outcomes: list[MediaOutcome] = []
|
||||
for media in media_items:
|
||||
if should_stop():
|
||||
break
|
||||
try:
|
||||
outcomes.append(self._download_one(
|
||||
post, media, folder, artist_slug, is_seen, recapture=recapture,
|
||||
))
|
||||
except Exception as exc: # resilient: isolate one item's failure
|
||||
log.warning(
|
||||
"Discord media failed (message %s, file %d): %s",
|
||||
post.get("id"), media.num, exc,
|
||||
)
|
||||
outcomes.append(
|
||||
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||
)
|
||||
return outcomes
|
||||
|
||||
def _download_one(
|
||||
self,
|
||||
post: dict,
|
||||
media,
|
||||
folder: Path,
|
||||
artist_slug: str,
|
||||
is_seen: Callable[[object], bool],
|
||||
*,
|
||||
recapture: bool = False,
|
||||
) -> MediaOutcome:
|
||||
seen = is_seen(media)
|
||||
if seen and not recapture:
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
stem = media_stem(post, media)
|
||||
path = folder / f"{stem}.{media.extension or _NO_EXTENSION}"
|
||||
if path.exists(): # tier-2: gallery-dl (or an earlier walk) wrote it
|
||||
return MediaOutcome(media=media, status="skipped_disk", path=path, error=None)
|
||||
if seen: # recapture never re-fetches a seen file that is gone
|
||||
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
if self._rate_limit > 0:
|
||||
time.sleep(self._rate_limit)
|
||||
out = self._fetch_get(media.url, path)
|
||||
reason, quarantined = self._validate_path(out, artist_slug, media.url)
|
||||
if reason is not None:
|
||||
return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
|
||||
sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")}
|
||||
sidecar["source_url"] = media.url
|
||||
(folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2))
|
||||
return MediaOutcome(media=media, status="downloaded", path=out, error=None)
|
||||
|
||||
def write_post_record(
|
||||
self, post: dict, artist_slug: str, *, revisit: bool = False,
|
||||
) -> PostRecordOutcome:
|
||||
"""The message record — the one writer of a Discord post's body, date
|
||||
and permalink ids. `revisit` re-reads a message already captured (an
|
||||
edit); an empty re-read writes nothing, so it never blanks a body."""
|
||||
mid = str(post.get("id") or "")
|
||||
body = message_text(post)
|
||||
if not mid or (revisit and not body.strip()):
|
||||
return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0)
|
||||
meta = post.get("_meta") or {}
|
||||
author = post.get("author") or {}
|
||||
record = {
|
||||
"category": PLATFORM,
|
||||
"message_id": mid,
|
||||
"server": meta.get("server"),
|
||||
"server_id": meta.get("server_id"),
|
||||
"channel": meta.get("channel"),
|
||||
"channel_id": meta.get("channel_id") or post.get("channel_id"),
|
||||
"parent": meta.get("parent"),
|
||||
"is_thread": meta.get("is_thread"),
|
||||
"author": author.get("username"),
|
||||
"author_id": author.get("id"),
|
||||
"message": body,
|
||||
"date": post.get("timestamp"),
|
||||
}
|
||||
folder = channel_dir(self.images_root, artist_slug, post)
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
when = message_date(post)
|
||||
day = f"{when:%Y%m%d}" if when else "None"
|
||||
path = folder / f"{day}_{mid}_post.json"
|
||||
path.write_text(json.dumps(
|
||||
{k: v for k, v in record.items() if v is not None}, indent=2,
|
||||
))
|
||||
return PostRecordOutcome(
|
||||
path=path, post_type=None, title=None, body_chars=len(body),
|
||||
)
|
||||
@@ -63,7 +63,7 @@ from collections import Counter
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
from sqlalchemy import Select, delete, func, select, update
|
||||
from sqlalchemy import Select, delete, func, select, union, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
@@ -126,6 +126,31 @@ def cosine_distance(a, b) -> float:
|
||||
return 1.0 - (dot / (na * nb))
|
||||
|
||||
|
||||
def _message_images(posts):
|
||||
"""(post_id, image_id) for every image a message carries — owned AND re-posted.
|
||||
|
||||
A message owns an image through `primary_post_id`, but only the FIRST
|
||||
message imported with a given file does. The same file posted again is a
|
||||
provenance link, and the backfill runs newest-first, so it is usually the
|
||||
ORIGINAL message that ends up owning nothing. Reading ownership alone left
|
||||
101 of Yellowroom's messages (2018–2020 mostly) ungroupable: every image
|
||||
they carried also sat in another message.
|
||||
|
||||
`posts` is a list of ids or a select of them; filtering both branches by it
|
||||
keeps the union to the messages in hand rather than the whole library.
|
||||
Callers pass MESSAGE posts only — a drop's own provenance rows would read
|
||||
as images it carries.
|
||||
"""
|
||||
owned = select(
|
||||
ImageRecord.primary_post_id.label("post_id"), ImageRecord.id.label("image_id"),
|
||||
).where(ImageRecord.primary_post_id.in_(posts))
|
||||
reposted = select(
|
||||
ImageProvenance.post_id.label("post_id"),
|
||||
ImageProvenance.image_record_id.label("image_id"),
|
||||
).where(ImageProvenance.post_id.in_(posts))
|
||||
return union(owned, reposted).subquery()
|
||||
|
||||
|
||||
def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
||||
"""Ungrouped Discord message-posts, one representative image each, OLDEST
|
||||
FIRST — which is the order `build_groups` requires.
|
||||
@@ -143,13 +168,15 @@ def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
||||
take the OLDEST candidates instead of the lowest-numbered ones.
|
||||
"""
|
||||
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||
carried = _message_images(select(Post.id).where(Post.source_id == source_id))
|
||||
inner = (
|
||||
select(
|
||||
Post.id.label("post_id"),
|
||||
sort_key.label("occurred_at"),
|
||||
ImageRecord.siglip_embedding.label("embedding"),
|
||||
)
|
||||
.join(ImageRecord, ImageRecord.primary_post_id == Post.id)
|
||||
.join(carried, carried.c.post_id == Post.id)
|
||||
.join(ImageRecord, ImageRecord.id == carried.c.image_id)
|
||||
.where(
|
||||
Post.source_id == source_id,
|
||||
# Never absorb a post FC wrote, and never re-absorb one already
|
||||
@@ -295,9 +322,10 @@ async def _link_member_images(
|
||||
"""
|
||||
if not member_ids:
|
||||
return 0
|
||||
image_rows = (await session.execute(
|
||||
select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids))
|
||||
)).scalars().all()
|
||||
carried = _message_images(member_ids)
|
||||
image_rows = sorted(set((await session.execute(
|
||||
select(carried.c.image_id)
|
||||
)).scalars().all()))
|
||||
if not image_rows:
|
||||
return 0
|
||||
await session.execute(
|
||||
@@ -421,9 +449,11 @@ async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None
|
||||
free to disagree; this way there is one.
|
||||
"""
|
||||
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||
carried = _message_images(select(Post.id).where(Post.absorbed_by_post_id == post_id))
|
||||
return (await session.execute(
|
||||
select(ImageRecord.siglip_embedding)
|
||||
.join(Post, ImageRecord.primary_post_id == Post.id)
|
||||
.join(carried, carried.c.image_id == ImageRecord.id)
|
||||
.join(Post, Post.id == carried.c.post_id)
|
||||
.where(
|
||||
Post.absorbed_by_post_id == post_id,
|
||||
ImageRecord.siglip_embedding.is_not(None),
|
||||
@@ -725,12 +755,15 @@ async def _load_drops(session: AsyncSession, source: Source) -> list[_Drop]:
|
||||
names: dict[int, set[str]] = {pid: set() for pid in by_id}
|
||||
images: dict[int, list[int]] = {pid: [] for pid in by_id}
|
||||
if owner_of:
|
||||
for iid, primary, path in (await session.execute(
|
||||
select(ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path)
|
||||
.where(ImageRecord.primary_post_id.in_(list(owner_of)))
|
||||
carried = _message_images(list(owner_of))
|
||||
for iid, message, path in (await session.execute(
|
||||
select(ImageRecord.id, carried.c.post_id, ImageRecord.path)
|
||||
.join(carried, carried.c.image_id == ImageRecord.id)
|
||||
.order_by(ImageRecord.id)
|
||||
)).all():
|
||||
drop = owner_of[primary]
|
||||
drop = owner_of[message]
|
||||
if iid in images[drop]:
|
||||
continue # one file carried by two of the drop's messages
|
||||
images[drop].append(iid)
|
||||
if (name := leading_name(path)) is not None:
|
||||
names[drop].add(name)
|
||||
@@ -844,7 +877,12 @@ async def merge_trickles(
|
||||
if drop.first_at - earlier.last_at > gap:
|
||||
continue
|
||||
shared = mine & gated(earlier.names)
|
||||
if shared:
|
||||
if set(drop.images) & set(earlier.images):
|
||||
# The creator posted the very same file again — the strongest
|
||||
# evidence there is, and one nearest-neighbour cannot see: it
|
||||
# skips the image itself, which is the one they share.
|
||||
targets[earlier.post.id] = (earlier, "same_image")
|
||||
elif shared:
|
||||
targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
|
||||
elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
|
||||
targets[earlier.post.id] = (earlier, "nearest")
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Native Discord ingester — the Discord ADAPTER over `ingest_core.Ingester`.
|
||||
|
||||
Thin counterpart to subscribestar_ingester (milestone 428). The walk's modes,
|
||||
both ledgers, cursor checkpointing and the post-first capture live in the core;
|
||||
this wires in the Discord client, downloader, ledger models and key.
|
||||
|
||||
Two things differ from the cookie platforms:
|
||||
|
||||
- Discord authenticates with a user TOKEN, so `auth_token` is the credential
|
||||
here rather than an argument accepted and ignored.
|
||||
- The body canary is off. It fails a walk whose first 30+ captured posts all
|
||||
came back without text, on the theory that a creator nearly always writes
|
||||
something; a Discord drop is routinely files and nothing else, so on
|
||||
Discord that is an ordinary backfill, not a broken parser.
|
||||
|
||||
`campaign_id` is the source URL (a server, channel, thread or category link).
|
||||
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
|
||||
from ..models import DiscordFailedMedia, DiscordSeenMedia
|
||||
from .discord_client import DiscordAPIError, DiscordClient, MediaItem
|
||||
from .discord_downloader import DiscordDownloader
|
||||
from .ingest_core import Ingester
|
||||
|
||||
_LEDGER_KEY_MAX = 128
|
||||
|
||||
|
||||
def _ledger_key(media: MediaItem) -> str:
|
||||
"""`<message_id>:<media_id>` — stable across edits (see MediaItem)."""
|
||||
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||
|
||||
|
||||
class DiscordIngester(Ingester):
|
||||
"""Walk a Discord source's channels, download unseen files, return a
|
||||
`DownloadResult`. `client` / `downloader` are injectable for tests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
images_root: Path,
|
||||
cookies_path: str | None,
|
||||
session_factory: Callable[[], object],
|
||||
*,
|
||||
validate: bool = True,
|
||||
rate_limit: float = 0.0,
|
||||
request_sleep: float = 0.0,
|
||||
auth_token: str | None = None,
|
||||
client: DiscordClient | None = None,
|
||||
downloader: DiscordDownloader | None = None,
|
||||
):
|
||||
del cookies_path # Discord authenticates by token (uniform signature)
|
||||
self.images_root = Path(images_root)
|
||||
super().__init__(
|
||||
client=client if client is not None else DiscordClient(
|
||||
auth_token, request_sleep=request_sleep,
|
||||
),
|
||||
downloader=downloader if downloader is not None else DiscordDownloader(
|
||||
self.images_root, validate=validate, rate_limit=rate_limit,
|
||||
),
|
||||
session_factory=session_factory,
|
||||
seen_model=DiscordSeenMedia,
|
||||
failed_model=DiscordFailedMedia,
|
||||
seen_constraint="uq_discord_seen_media_source_id",
|
||||
failed_constraint="uq_discord_failed_media_source_id",
|
||||
ledger_key=_ledger_key,
|
||||
platform="discord",
|
||||
error_base=DiscordAPIError,
|
||||
drift_label="Discord API",
|
||||
body_canary=False,
|
||||
)
|
||||
|
||||
|
||||
async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
|
||||
"""The uniform `(ok, message)` probe: is the token valid, and can its
|
||||
account see the channel or server the source names?"""
|
||||
if not auth_token:
|
||||
return False, "No Discord token is saved — add one under Credentials."
|
||||
client = DiscordClient(auth_token)
|
||||
loop = asyncio.get_running_loop()
|
||||
return await loop.run_in_executor(None, client.verify_auth, url)
|
||||
@@ -23,6 +23,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from .discord_ingester import DiscordIngester
|
||||
from .gallery_dl import DownloadResult, ErrorType
|
||||
from .ingest_core import DEFAULT_REVISIT_DAYS
|
||||
from .patreon_ingester import PatreonIngester
|
||||
@@ -31,9 +32,13 @@ from .platforms import known_platform_keys
|
||||
from .subscribestar_ingester import SubscribeStarIngester
|
||||
|
||||
# Platforms whose download + verify go through the native ingester rather than
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
|
||||
# they migrate too.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry) until it
|
||||
# migrates too. Discord joined in milestone 428.
|
||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||
|
||||
# Native platforms whose feed id IS the source URL, so there is nothing to
|
||||
# resolve: SubscribeStar's creator page, Discord's server/channel link.
|
||||
_URL_IS_FEED = frozenset({"subscribestar", "discord"})
|
||||
|
||||
|
||||
def _unsupported_platform_message(platform: str) -> str | None:
|
||||
@@ -67,6 +72,7 @@ def _native_ingester_cls(platform: str):
|
||||
return {
|
||||
"patreon": PatreonIngester,
|
||||
"subscribestar": SubscribeStarIngester,
|
||||
"discord": DiscordIngester,
|
||||
}[platform]
|
||||
|
||||
|
||||
@@ -126,10 +132,10 @@ async def _resolve_native_campaign_id(
|
||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
||||
so phase 3 caches it)."""
|
||||
if platform == "subscribestar":
|
||||
and Discord's feed id IS the source URL (no lookup → resolved None). Patreon
|
||||
resolves the campaign id from the vanity URL (resolved non-None when a lookup
|
||||
actually ran, so phase 3 caches it)."""
|
||||
if platform in _URL_IS_FEED:
|
||||
return url, None
|
||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||
|
||||
@@ -252,6 +258,10 @@ async def verify_source_credential(
|
||||
from .subscribestar_ingester import verify_subscribestar_credential
|
||||
|
||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||
if platform == "discord":
|
||||
from .discord_ingester import verify_discord_credential
|
||||
|
||||
return await verify_discord_credential(url, auth_token)
|
||||
from .patreon_ingester import verify_patreon_credential
|
||||
|
||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||
|
||||
@@ -423,6 +423,13 @@ class DownloadService:
|
||||
|
||||
await loop.run_in_executor(None, _upsert)
|
||||
|
||||
# Only now is it safe to call this walk's media seen: every file above
|
||||
# has been through the importer. Had the run died before here they stay
|
||||
# unmarked, and the next walk imports them from disk (ingest_core).
|
||||
mark_seen = getattr(dl_result, "mark_seen_after_import", None)
|
||||
if mark_seen is not None:
|
||||
await loop.run_in_executor(None, mark_seen)
|
||||
|
||||
# #830 recapture: backfill source_filehash on EXISTING on-disk images so
|
||||
# their post-body inline <img src=CDN> remaps to the local copy. A
|
||||
# SEPARATE non-deleting channel (NOT the import list — that would unlink
|
||||
|
||||
@@ -21,6 +21,9 @@ from .source_service import BACKFILL_MAX_CHUNKS
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# The probe runs while the chip is drawing; names that take longer are skipped.
|
||||
_NAME_LOOKUP_SECONDS = 6.0
|
||||
|
||||
|
||||
class UnknownPlatformError(Exception):
|
||||
"""URL didn't match any platform pattern."""
|
||||
@@ -30,6 +33,10 @@ class InvalidUrlError(Exception):
|
||||
"""URL was empty or missing a scheme."""
|
||||
|
||||
|
||||
class UnknownArtistError(Exception):
|
||||
"""quick-add named an `artist_id` that does not exist."""
|
||||
|
||||
|
||||
# Mirrored byte-for-byte from extension/lib/platforms.js
|
||||
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
||||
# reviewers catch drift.
|
||||
@@ -55,8 +62,39 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
# A Discord URL names a server or a channel, never a creator, so the slug is
|
||||
# `<server>` or `<server>/<channel>` and the artist is chosen, not derived.
|
||||
# A trailing message id (a jump link) still names its channel. DMs (`@me`)
|
||||
# are not sources; thread links (`/threads/`) are left to the manual form.
|
||||
("discord", re.compile(
|
||||
r"^https?://(?:www\.|ptb\.|canary\.)?discord\.com/channels/"
|
||||
r"(?P<slug>\d+(?:/\d+)?)(?:/\d+)?/?(?:[?#].*)?$",
|
||||
re.IGNORECASE,
|
||||
)),
|
||||
]
|
||||
|
||||
DISCORD = "discord"
|
||||
|
||||
|
||||
def canonical_source_url(platform: str, url: str, slug: str) -> str:
|
||||
"""The URL a new source is stored under. Discord's is rebuilt from the ids
|
||||
— the form the manual Add form and the ingester use — so a jump link, a
|
||||
ptb/canary host or a trailing slash never makes a second source for the
|
||||
same channel. Every other platform keeps the URL as given."""
|
||||
if platform == DISCORD:
|
||||
return f"https://discord.com/channels/{slug}"
|
||||
return url
|
||||
|
||||
|
||||
def _discord_ids(url: str) -> tuple[str | None, str | None] | None:
|
||||
"""`(server_id, channel_id)` of a stored Discord source URL, None if it
|
||||
does not parse (a DM or thread link, or an old malformed row)."""
|
||||
from .discord_client import DiscordAPIError, parse_source_url
|
||||
try:
|
||||
return parse_source_url(url)
|
||||
except DiscordAPIError:
|
||||
return None
|
||||
|
||||
|
||||
class ExtensionService:
|
||||
def __init__(self, session: AsyncSession, crypto=None) -> None:
|
||||
@@ -65,29 +103,86 @@ class ExtensionService:
|
||||
# add-time. None → skip resolution, fall back to the handle.
|
||||
self._crypto = crypto
|
||||
|
||||
async def quick_add_source(self, url: str) -> dict:
|
||||
async def quick_add_source(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
artist_id: int | None = None,
|
||||
artist_name: str | None = None,
|
||||
use_platform_name: bool = False,
|
||||
) -> dict:
|
||||
"""Add `url` as a source. `artist_id` connects it to an existing
|
||||
artist, `artist_name` to that artist (created if new); with neither,
|
||||
the artist is resolved from the platform as before.
|
||||
|
||||
`use_platform_name` applies the operator's convention that the Patreon
|
||||
name is canon: a Patreon source added to an existing artist renames
|
||||
that artist to the creator's Patreon display name. Name only — the
|
||||
slug, and every path keyed off it, never moves (#130). Ignored on every
|
||||
other platform, and when the name can't be read."""
|
||||
platform, raw_slug = self._derive(url)
|
||||
url = canonical_source_url(platform, url, raw_slug)
|
||||
renamed_from = None
|
||||
# Identity by SOURCE handle (#130): an existing (platform, url) source
|
||||
# keeps its artist on re-add — even if that artist was since renamed (its
|
||||
# frozen slug no longer matches the current name). Only a genuinely new
|
||||
# source resolves/creates an artist.
|
||||
existing = (await self.session.execute(
|
||||
select(Source).where(Source.platform == platform, Source.url == url)
|
||||
)).scalar_one_or_none()
|
||||
# frozen slug no longer matches the current name), and even when the
|
||||
# add named a different artist. Only a genuinely new source
|
||||
# resolves/creates an artist.
|
||||
existing = await self._existing_source(platform, url)
|
||||
if existing is not None:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == existing.artist_id)
|
||||
)).scalar_one()
|
||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||
|
||||
# New source → name the artist properly by resolving the real display
|
||||
# name from the platform (falls back to the URL handle).
|
||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||
artist, created_artist = await self._find_or_create_artist(name)
|
||||
if artist_id is not None:
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.id == artist_id)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
raise UnknownArtistError(f"no artist with id {artist_id}")
|
||||
created_artist = False
|
||||
if use_platform_name and platform == "patreon":
|
||||
renamed_from = await self._adopt_patreon_name(artist, raw_slug, url)
|
||||
else:
|
||||
name = (artist_name or "").strip()
|
||||
if not name:
|
||||
# Name the artist properly by resolving the real display name
|
||||
# from the platform (falls back to the URL handle).
|
||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||
artist, created_artist = await self._find_or_create_artist(name)
|
||||
source, created_source = await self._find_or_create_source(
|
||||
artist_id=artist.id, platform=platform, url=url,
|
||||
)
|
||||
return self._shape(source, artist, created_source, created_artist)
|
||||
shaped = self._shape(source, artist, created_source, created_artist)
|
||||
if renamed_from is not None:
|
||||
shaped["renamed_from"] = renamed_from
|
||||
return shaped
|
||||
|
||||
async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
|
||||
"""Rename `artist` to the Patreon display name; the old name when it
|
||||
changed, else None. Unreadable name → no rename, never the handle."""
|
||||
name = await self._platform_display_name("patreon", raw_slug, url)
|
||||
if not name or name == artist.name:
|
||||
return None
|
||||
old = artist.name
|
||||
artist.name = name
|
||||
await self.session.commit()
|
||||
return old
|
||||
|
||||
async def _existing_source(self, platform: str, url: str) -> Source | None:
|
||||
"""The source this URL already is, whichever artist owns it. Discord
|
||||
compares ids, not strings, so a row stored before canonicalisation (a
|
||||
ptb host, a trailing slash) is still found."""
|
||||
if platform != DISCORD:
|
||||
return (await self.session.execute(
|
||||
select(Source).where(Source.platform == platform, Source.url == url)
|
||||
)).scalars().first()
|
||||
want = _discord_ids(url)
|
||||
rows = (await self.session.execute(
|
||||
select(Source).where(Source.platform == DISCORD).order_by(Source.id)
|
||||
)).scalars().all()
|
||||
return next((s for s in rows if _discord_ids(s.url) == want), None)
|
||||
|
||||
@staticmethod
|
||||
def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
|
||||
@@ -117,8 +212,23 @@ class ExtensionService:
|
||||
platforms (and any failure — no credential, network error) fall back to
|
||||
the URL handle, which is already readable.
|
||||
The resolvers are sync, so they run in an executor."""
|
||||
if platform == DISCORD:
|
||||
# The server's name: what the operator knows the community as.
|
||||
server_id = raw_slug.split("/", 1)[0]
|
||||
names = await self._discord_names(server_id, None)
|
||||
return names.get("server") or f"Discord {server_id}"
|
||||
return await self._platform_display_name(platform, raw_slug, url) or raw_slug
|
||||
|
||||
async def _platform_display_name(
|
||||
self, platform: str, raw_slug: str, url: str
|
||||
) -> str | None:
|
||||
"""The creator's display name as Patreon or SubscribeStar shows it, read
|
||||
with the stored cookies; None when it can't be read (no credential, a
|
||||
network error, a slow answer, any other platform). None, not the handle,
|
||||
so a caller can tell a real name from a fallback — a rename to the
|
||||
Patreon name must never rename to a URL handle instead."""
|
||||
if self._crypto is None or platform not in ("patreon", "subscribestar"):
|
||||
return raw_slug
|
||||
return None
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
@@ -128,7 +238,7 @@ class ExtensionService:
|
||||
if platform == "patreon":
|
||||
cookies = await cred.get_cookies_path("patreon")
|
||||
from .patreon_resolver import resolve_display_name
|
||||
name = await loop.run_in_executor(
|
||||
call = loop.run_in_executor(
|
||||
None, resolve_display_name, raw_slug,
|
||||
str(cookies) if cookies else None,
|
||||
)
|
||||
@@ -136,15 +246,14 @@ class ExtensionService:
|
||||
cookies = await cred.get_cookies_path("subscribestar")
|
||||
from .subscribestar_client import SubscribeStarClient
|
||||
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||
name = await loop.run_in_executor(
|
||||
None, client.resolve_display_name, url
|
||||
)
|
||||
call = loop.run_in_executor(None, client.resolve_display_name, url)
|
||||
name = await asyncio.wait_for(call, timeout=_NAME_LOOKUP_SECONDS)
|
||||
except Exception as exc: # resolution is best-effort — never block the add
|
||||
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||
return raw_slug
|
||||
return name or raw_slug
|
||||
return None
|
||||
return (name or "").strip() or None
|
||||
|
||||
async def probe(self, url: str) -> dict:
|
||||
async def probe(self, url: str, *, names: bool = False) -> dict:
|
||||
"""Read-only resolution of a creator-page URL against the FC DB.
|
||||
Returns one of:
|
||||
- {state: 'unknown_platform'} — URL didn't match any
|
||||
@@ -160,21 +269,30 @@ class ExtensionService:
|
||||
— exact (artist, platform,
|
||||
url) Source already exists
|
||||
|
||||
Side-effect-free: two SELECTs at most.
|
||||
`names` (the Add panel asks, the chip does not) adds `display_name`:
|
||||
the creator's name as Patreon/SubscribeStar shows it, or None. It costs
|
||||
a request to the platform, so a plain page view never pays it.
|
||||
|
||||
Side-effect-free: two SELECTs at most, plus that one lookup.
|
||||
"""
|
||||
try:
|
||||
platform, raw_slug = self._derive(url)
|
||||
except (UnknownPlatformError, InvalidUrlError):
|
||||
return {"state": "unknown_platform"}
|
||||
if platform == DISCORD:
|
||||
return await self._probe_discord(raw_slug)
|
||||
|
||||
slug = slugify(raw_slug)
|
||||
result: dict = {"platform": platform, "slug": slug}
|
||||
if names:
|
||||
result["display_name"] = await self._platform_display_name(
|
||||
platform, raw_slug, url,
|
||||
)
|
||||
artist = (await self.session.execute(
|
||||
select(Artist).where(Artist.slug == slug)
|
||||
)).scalar_one_or_none()
|
||||
if artist is None:
|
||||
return {"state": "new", "platform": platform, "slug": slug}
|
||||
|
||||
artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
return {"state": "new", **result}
|
||||
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(
|
||||
@@ -184,26 +302,114 @@ class ExtensionService:
|
||||
)
|
||||
)).scalar_one_or_none()
|
||||
if source is None:
|
||||
return {
|
||||
"state": "artist_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
}
|
||||
return {"state": "artist_match", **result, "artist": self._artist_payload(artist)}
|
||||
|
||||
return {
|
||||
"state": "source_match",
|
||||
"platform": platform,
|
||||
"slug": slug,
|
||||
"artist": artist_payload,
|
||||
"source": {
|
||||
"id": source.id,
|
||||
"artist_id": source.artist_id,
|
||||
"platform": source.platform,
|
||||
"url": source.url,
|
||||
"enabled": source.enabled,
|
||||
**result,
|
||||
"artist": self._artist_payload(artist),
|
||||
"source": self._source_payload(source),
|
||||
}
|
||||
|
||||
async def _probe_discord(self, raw_slug: str) -> dict:
|
||||
"""probe for a Discord server or channel. The states mean what they
|
||||
mean elsewhere, but the artist is never read off the URL:
|
||||
|
||||
- source_match: this channel is a source — or the whole server is
|
||||
(`covered_by_server`), which already walks every channel;
|
||||
- artist_match: another source on this server belongs to an artist,
|
||||
the one this channel most likely belongs to too (a suggestion the
|
||||
Add panel preselects, not a decision);
|
||||
- new: nothing on this server yet.
|
||||
|
||||
`discord` carries the ids, both canonical URLs and the display names,
|
||||
read with the stored token; a name that can't be read is None."""
|
||||
server_id, _, channel_id = raw_slug.partition("/")
|
||||
channel_id = channel_id or None
|
||||
rows = (await self.session.execute(
|
||||
select(Source, Artist)
|
||||
.join(Artist, Artist.id == Source.artist_id)
|
||||
.where(Source.platform == DISCORD)
|
||||
.order_by(Source.id)
|
||||
)).all()
|
||||
exact = server_whole = on_server = None
|
||||
for source, artist in rows:
|
||||
ids = _discord_ids(source.url)
|
||||
if ids is None or ids[0] != server_id:
|
||||
continue
|
||||
if ids[1] == channel_id and exact is None:
|
||||
exact = (source, artist)
|
||||
elif ids[1] is None and server_whole is None:
|
||||
server_whole = (source, artist)
|
||||
if on_server is None:
|
||||
on_server = (source, artist)
|
||||
|
||||
names = await self._discord_names(server_id, channel_id)
|
||||
base = f"https://discord.com/channels/{server_id}"
|
||||
result: dict = {
|
||||
"platform": DISCORD,
|
||||
"slug": raw_slug,
|
||||
"discord": {
|
||||
"server_id": server_id,
|
||||
"channel_id": channel_id,
|
||||
"server_name": names.get("server"),
|
||||
"channel_name": names.get("channel"),
|
||||
"server_url": base,
|
||||
"channel_url": f"{base}/{channel_id}" if channel_id else None,
|
||||
},
|
||||
}
|
||||
hit = exact or server_whole
|
||||
if hit is not None:
|
||||
source, artist = hit
|
||||
result.update(
|
||||
state="source_match",
|
||||
artist=self._artist_payload(artist),
|
||||
source=self._source_payload(source),
|
||||
covered_by_server=exact is None,
|
||||
)
|
||||
elif on_server is not None:
|
||||
result.update(state="artist_match", artist=self._artist_payload(on_server[1]))
|
||||
else:
|
||||
result["state"] = "new"
|
||||
return result
|
||||
|
||||
async def _discord_names(self, server_id: str | None, channel_id: str | None) -> dict:
|
||||
"""Server/channel display names via the stored Discord token. Never
|
||||
raises and never waits out a rate limit: it runs while the operator
|
||||
looks at a page, so a slow or missing answer just means no names."""
|
||||
if self._crypto is None:
|
||||
return {}
|
||||
import asyncio
|
||||
|
||||
from .credential_service import CredentialService
|
||||
from .discord_client import DiscordClient
|
||||
try:
|
||||
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||
if not token:
|
||||
return {}
|
||||
client = DiscordClient(token, max_retries=0)
|
||||
loop = asyncio.get_running_loop()
|
||||
return await asyncio.wait_for(
|
||||
loop.run_in_executor(None, client.describe, server_id, channel_id),
|
||||
timeout=_NAME_LOOKUP_SECONDS,
|
||||
)
|
||||
except Exception as exc: # names are decoration — never fail the call
|
||||
log.info("Discord name lookup failed: %s", exc)
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def _artist_payload(artist) -> dict:
|
||||
return {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||
|
||||
@staticmethod
|
||||
def _source_payload(source) -> dict:
|
||||
return {
|
||||
"id": source.id,
|
||||
"artist_id": source.artist_id,
|
||||
"platform": source.platform,
|
||||
"url": source.url,
|
||||
"enabled": source.enabled,
|
||||
}
|
||||
|
||||
def _derive(self, url: str) -> tuple[str, str]:
|
||||
if not isinstance(url, str) or not url.strip():
|
||||
|
||||
@@ -17,6 +17,7 @@ import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from enum import StrEnum
|
||||
@@ -94,48 +95,6 @@ BACKFILL_CHUNK_SECONDS = 600
|
||||
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
||||
|
||||
|
||||
# --- Discord naming ---------------------------------------------------------
|
||||
#
|
||||
# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
|
||||
# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
|
||||
# for an attachment: `channel` is a plain STRING (the channel's name), the
|
||||
# message is `message_id`, the attachment's position in it is `num`, and there
|
||||
# is NO `id` key at all.
|
||||
#
|
||||
# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
|
||||
# "None", so every Discord download since the platform was added landed in a
|
||||
# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
|
||||
# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
|
||||
# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
|
||||
# got a Post or a post date, and (b) collides: every `image.png` in a channel
|
||||
# overwrote the same `image.json`, so the one sidecar that survived described
|
||||
# whichever message happened to be written last.
|
||||
#
|
||||
# The fix names the sidecar EXACTLY like the media minus its extension, so
|
||||
# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
|
||||
# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
|
||||
# these patterns against a sanitized copy of the real sidecar, so a key that
|
||||
# does not exist fails CI instead of silently becoming "None".
|
||||
DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
|
||||
DISCORD_DIRECTORY = ["{channel}"]
|
||||
|
||||
|
||||
def sidecar_name_for(media_pattern: str) -> str | None:
|
||||
"""The metadata filename pattern that names a sidecar exactly like its media.
|
||||
|
||||
Returns None for a pattern that does not end in `.{extension}`, since then
|
||||
there is no media stem to mirror and the caller must fall back.
|
||||
"""
|
||||
suffix = ".{extension}"
|
||||
if not media_pattern.endswith(suffix):
|
||||
return None
|
||||
return media_pattern[: -len(suffix)] + ".json"
|
||||
|
||||
|
||||
def metadata_postprocessor(filename: str) -> dict:
|
||||
return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
|
||||
|
||||
|
||||
def archive_path(images_root: Path) -> Path:
|
||||
"""gallery-dl's download archive: the record of what it has already fetched.
|
||||
|
||||
@@ -230,6 +189,13 @@ class DownloadResult:
|
||||
# the platform cooldown matches the hint instead of a flat default. None when
|
||||
# unknown (no header, or not a rate-limit failure).
|
||||
retry_after_seconds: float | None = None
|
||||
# Native ingester only: marks this walk's fetched media seen in its ledger.
|
||||
# Phase 3 calls it AFTER the import loop, never before — a file marked seen
|
||||
# but not yet imported is invisible to every later walk, so a run killed in
|
||||
# between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13
|
||||
# images to a stranded run, 2026-09-24). Unmarked, the next walk finds the
|
||||
# file on disk with no ImageRecord and imports it. None on gallery-dl.
|
||||
mark_seen_after_import: Callable[[], None] | None = None
|
||||
|
||||
|
||||
def extract_errors_warnings(stderr: str) -> str:
|
||||
@@ -417,27 +383,16 @@ class GalleryDLService:
|
||||
# (services/patreon_ingester.py), not gallery-dl.
|
||||
PLATFORM_DEFAULTS = {
|
||||
# subscribestar removed — native-ingester platform now (#71); pixiv
|
||||
# removed likewise (#129); deviantart removed at #3069 as a dropped
|
||||
# platform, not a migrated one. The remaining entries are the
|
||||
# gallery-dl platforms not yet migrated.
|
||||
# removed likewise (#129); discord likewise (milestone 428, whose
|
||||
# downloader keeps this config's on-disk naming); deviantart removed at
|
||||
# #3069 as a dropped platform, not a migrated one. HentaiFoundry is the
|
||||
# one platform left here, by the operator's choice not to migrate it.
|
||||
"hentaifoundry": {
|
||||
"content_types": ["all"],
|
||||
"directory": [],
|
||||
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
||||
"include": "all",
|
||||
},
|
||||
"discord": {
|
||||
"content_types": ["all"],
|
||||
"directory": DISCORD_DIRECTORY,
|
||||
"filename": DISCORD_FILENAME,
|
||||
# Overrides the global `{filename}.json` sidecar for this extractor
|
||||
# only — see the Discord naming note above.
|
||||
"postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
|
||||
"embeds": "all",
|
||||
"stickers": True,
|
||||
"reactions": False,
|
||||
"threads": True,
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(
|
||||
@@ -552,17 +507,6 @@ class GalleryDLService:
|
||||
if source_config.filename_pattern:
|
||||
platform_section["filename"] = source_config.filename_pattern
|
||||
|
||||
# A platform that names its sidecar after its media must keep doing so
|
||||
# under a per-source filename override, or the pairing breaks exactly the
|
||||
# way Discord's did. No metadata wanted means no platform postprocessor
|
||||
# either — the global list was already dropped above.
|
||||
if "postprocessors" in platform_section:
|
||||
mirrored = sidecar_name_for(platform_section.get("filename") or "")
|
||||
if not source_config.save_metadata or mirrored is None:
|
||||
platform_section.pop("postprocessors")
|
||||
else:
|
||||
platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
|
||||
|
||||
platform_section["metadata"] = source_config.save_metadata
|
||||
|
||||
return config
|
||||
@@ -810,9 +754,6 @@ class GalleryDLService:
|
||||
|
||||
if cookies_path:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})
|
||||
config["extractor"]["discord"]["token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
@@ -996,8 +937,6 @@ class GalleryDLService:
|
||||
config = self._build_config_for_source(platform, source_config, artist_slug)
|
||||
if cookies_path:
|
||||
config["extractor"]["cookies"] = cookies_path
|
||||
if auth_token and platform == "discord":
|
||||
config["extractor"].setdefault("discord", {})["token"] = auth_token
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||
|
||||
@@ -36,6 +36,7 @@ from datetime import UTC, datetime, timedelta
|
||||
from sqlalchemy import delete, func, select, text
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from ..models import ImageRecord
|
||||
from .gallery_dl import (
|
||||
DownloadResult,
|
||||
ErrorType,
|
||||
@@ -105,6 +106,22 @@ _LIVE_PROGRESS_INTERVAL = 5.0
|
||||
# recapture (the operator's schema-test flow) reaches the sample.
|
||||
_CANARY_MIN_SAMPLE = 30
|
||||
|
||||
# The walk's time budget covers only the walk, but phase 3 runs in the SAME
|
||||
# Celery task, under the same soft limit (tasks/download.py: 1350s). A walk that
|
||||
# finds a lot of work for phase 3 must stop early and leave it to the next chunk,
|
||||
# or phase 3 is killed mid-import: TamadaHeijun's recapture, 2026-09-24, walked
|
||||
# for ~2 min and then spent 20 min importing 431 orphans and relinking ~3000
|
||||
# on-disk files, and died at the soft limit.
|
||||
#
|
||||
# So the walk also stops when its elapsed time PLUS phase 3's estimated cost
|
||||
# would pass CHUNK_TOTAL_SECONDS. Costs measured on the live instance: 431
|
||||
# imports took 976s (~2.3s each: hash, pHash, sidecar, provenance); a relink is
|
||||
# a sha256 over NFS, 0.15s for the 8.8 MB average file, plus a lookup.
|
||||
# test_download_source_task pins CHUNK_TOTAL_SECONDS under the soft limit.
|
||||
CHUNK_TOTAL_SECONDS = 1200.0
|
||||
PHASE3_IMPORT_SECONDS = 2.5
|
||||
PHASE3_RELINK_SECONDS = 0.25
|
||||
|
||||
|
||||
def _parse_published(raw: object) -> datetime | None:
|
||||
"""An ISO-8601 post date from either native client, as aware UTC.
|
||||
@@ -247,6 +264,14 @@ class Ingester:
|
||||
# operator-driven "re-read every body" pass; a horizon there would be a
|
||||
# third overlapping answer to a question that has two.
|
||||
post_meta = getattr(self.client, "post_meta", None)
|
||||
# #4413: optional client seam for a source that is several feeds walked
|
||||
# one after another (a Discord server: every channel and thread). The
|
||||
# tick early-out means "this feed has nothing new", and without the
|
||||
# seam it ends the WHOLE walk — so the first quiet channel would hide
|
||||
# every channel after it. With it, the early-out asks the client to
|
||||
# move on and the walk continues. Absent → the early-out ends the walk,
|
||||
# exactly as before (Patreon and SubscribeStar are one feed each).
|
||||
skip_feed = getattr(self.client, "skip_feed", None)
|
||||
horizon: datetime | None = None
|
||||
if mode == "tick" and revisit_days > 0 and post_meta is not None:
|
||||
horizon = datetime.now(UTC) - timedelta(days=revisit_days)
|
||||
@@ -261,6 +286,9 @@ class Ingester:
|
||||
# source_filehash and (b) link the on-disk image to its Post (#1288) —
|
||||
# WITHOUT re-downloading or unlinking the file. Empty outside recapture.
|
||||
relink: list[tuple[str, str, str]] = []
|
||||
# Media handed to phase 3 for import. Marked seen by phase 3 once the
|
||||
# import has run (`mark_seen_after_import`), not here — see there.
|
||||
fetched: list[tuple[str, str]] = []
|
||||
downloaded = 0
|
||||
errors = 0
|
||||
quarantined = 0
|
||||
@@ -292,6 +320,7 @@ class Ingester:
|
||||
reached_bottom = False
|
||||
budget_hit = False
|
||||
early_out = False
|
||||
feeds_caught_up = 0 # #4413: feeds a tick left early via skip_feed
|
||||
stopped = False # plan #708 B4: operator hit Stop mid-walk
|
||||
cancel_armed = False # latched once we observe a live "running" state
|
||||
|
||||
@@ -313,6 +342,7 @@ class Ingester:
|
||||
written_paths=written,
|
||||
post_record_paths=list(post_records),
|
||||
relink_source_paths=list(relink),
|
||||
mark_seen_after_import=lambda: self._mark_seen(source_id, fetched),
|
||||
stdout="\n".join(log_lines),
|
||||
stderr="",
|
||||
return_code=return_code,
|
||||
@@ -388,7 +418,17 @@ class Ingester:
|
||||
|
||||
# Time-box check at the post boundary (coarse, like a gallery-dl
|
||||
# chunk). Backfill/recovery resume from emitted_cursor next chunk.
|
||||
if time.monotonic() - start >= time_budget_seconds:
|
||||
# The second half is phase 3's share of the task — see
|
||||
# CHUNK_TOTAL_SECONDS. A mid-page stop resumes the same page.
|
||||
elapsed = time.monotonic() - start
|
||||
phase3 = (
|
||||
len(written) * PHASE3_IMPORT_SECONDS
|
||||
+ len(relink) * PHASE3_RELINK_SECONDS
|
||||
)
|
||||
if (
|
||||
elapsed >= time_budget_seconds
|
||||
or elapsed + phase3 >= CHUNK_TOTAL_SECONDS
|
||||
):
|
||||
budget_hit = True
|
||||
break
|
||||
|
||||
@@ -512,6 +552,13 @@ class Ingester:
|
||||
recapture=recapture,
|
||||
)
|
||||
|
||||
# An on-disk file is only "done" if something imported it. One
|
||||
# with no ImageRecord at its path was written by a run that died
|
||||
# before phase 3 — it goes to import, not to the ledger.
|
||||
imported_paths = self._recorded_paths([
|
||||
str(o.path) for o in outcomes
|
||||
if o.status == "skipped_disk" and o.path is not None
|
||||
])
|
||||
to_mark: list[tuple[str, str]] = []
|
||||
to_clear: list[str] = [] # recovered → drop any dead-letter row
|
||||
to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error)
|
||||
@@ -523,11 +570,29 @@ class Ingester:
|
||||
downloaded += 1
|
||||
if outcome.path is not None:
|
||||
written.append(str(outcome.path))
|
||||
to_mark.append((key, media_item.post_id))
|
||||
fetched.append((key, media_item.post_id))
|
||||
to_clear.append(key)
|
||||
consecutive_seen = 0
|
||||
elif (
|
||||
outcome.status == "skipped_disk"
|
||||
and outcome.path is not None
|
||||
and str(outcome.path) not in imported_paths
|
||||
):
|
||||
# On disk, never imported: a prior run wrote it and died
|
||||
# before phase 3. Import it now. Safe to feed to
|
||||
# attach_in_place because no record owns this path —
|
||||
# the unlink below is about a file that IS the record.
|
||||
written.append(str(outcome.path))
|
||||
fetched.append((key, media_item.post_id))
|
||||
to_clear.append(key)
|
||||
skipped_count += 1
|
||||
consecutive_seen += 1
|
||||
log_lines.append(
|
||||
f" post {media_item.post_id} — on disk but never "
|
||||
f"imported: {outcome.path.name}"
|
||||
)
|
||||
elif outcome.status == "skipped_disk":
|
||||
# Already on disk (a prior run). Reconcile the ledger so a
|
||||
# Already on disk and imported. Reconcile the ledger so a
|
||||
# later tick skips it at tier-1 without a disk stat, but
|
||||
# do NOT re-feed it to phase 3 — attach_in_place would see
|
||||
# the duplicate sha256 and unlink the on-disk copy.
|
||||
@@ -618,9 +683,15 @@ class Ingester:
|
||||
})
|
||||
|
||||
if early_out:
|
||||
break
|
||||
if skip_feed is None:
|
||||
break
|
||||
skip_feed()
|
||||
feeds_caught_up += 1
|
||||
early_out = False
|
||||
consecutive_seen = 0
|
||||
else:
|
||||
reached_bottom = True
|
||||
# A walk that left feeds early did not read to their ends.
|
||||
reached_bottom = not feeds_caught_up
|
||||
except self._error_base as exc:
|
||||
# The platform's client-error base — _failure_result (adapter)
|
||||
# maps it to a typed error.
|
||||
@@ -668,6 +739,7 @@ class Ingester:
|
||||
f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
|
||||
if revisited else ""
|
||||
)
|
||||
+ (f", {feeds_caught_up} feed(s) caught up" if feeds_caught_up else "")
|
||||
+ (", reached end" if reached_bottom else "")
|
||||
+ (", time-boxed" if budget_hit else "")
|
||||
)
|
||||
@@ -681,7 +753,13 @@ class Ingester:
|
||||
# next chunk resumes from the emitted cursor. No progress → TIMEOUT,
|
||||
# which feeds download_service's backfill stall-guard. rc<0 mirrors
|
||||
# subprocess TimeoutExpired so completion detection stays false.
|
||||
made_progress = downloaded > 0 or emitted_cursor != resume_cursor
|
||||
# Work handed to phase 3 is progress too: a recapture chunk that
|
||||
# stopped for its imports downloaded nothing, and may not have left
|
||||
# its first page.
|
||||
made_progress = (
|
||||
downloaded > 0 or bool(written) or bool(relink)
|
||||
or emitted_cursor != resume_cursor
|
||||
)
|
||||
if made_progress:
|
||||
return _result(
|
||||
success=False, return_code=-1,
|
||||
@@ -887,6 +965,16 @@ class Ingester:
|
||||
)
|
||||
session.commit()
|
||||
|
||||
def _recorded_paths(self, paths: list[str]) -> set[str]:
|
||||
"""Which of `paths` an ImageRecord already points at."""
|
||||
if not paths:
|
||||
return set()
|
||||
with self.session_factory() as session:
|
||||
rows = session.execute(
|
||||
select(ImageRecord.path).where(ImageRecord.path.in_(paths))
|
||||
).scalars().all()
|
||||
return set(rows)
|
||||
|
||||
def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
|
||||
"""Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
|
||||
|
||||
|
||||
@@ -23,8 +23,10 @@ log = logging.getLogger(__name__)
|
||||
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
||||
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
||||
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
||||
# Add a platform here to cap it to a single concurrent walk.
|
||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
||||
# Add a platform here to cap it to a single concurrent walk. Discord most of
|
||||
# all: every source walks on the operator's ONE user token, and parallel walks
|
||||
# on a user account are both how its rate limit trips and what gets it flagged.
|
||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||
|
||||
_LOCK_PREFIX = "fc:download_lock:"
|
||||
|
||||
|
||||
@@ -115,6 +115,32 @@ async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime
|
||||
return active
|
||||
|
||||
|
||||
def backfill_ready(source: Source) -> bool:
|
||||
"""A deep walk the operator started, with budget left and no failure
|
||||
backing it off — due NOW rather than at its next scheduled check.
|
||||
|
||||
A backfill runs one time-boxed chunk per download (plan #693), and nothing
|
||||
queued the next chunk: each waited for the source's regular interval. At
|
||||
the 8-hour default a freshly armed backfill sat untouched until the next
|
||||
check (the operator armed one on 2026-09-25 and saw nothing happen) and a
|
||||
five-chunk walk took most of two days. The tick's in-flight guard keeps
|
||||
one chunk at a time per source and the platform lock one walk per
|
||||
platform, so "due every tick" means "next chunk as soon as the last one
|
||||
ends".
|
||||
|
||||
The failure gate is what keeps a broken source from retrying every
|
||||
minute: any failed chunk raises `consecutive_failures`, which drops the
|
||||
source back onto its backed-off interval. A chunk that fails to progress
|
||||
twice marks the walk stalled (download_service), which ends it here too.
|
||||
"""
|
||||
co = source.config_overrides or {}
|
||||
return (
|
||||
co.get("_backfill_state") == "running"
|
||||
and (source.backfill_runs_remaining or 0) > 0
|
||||
and not (source.consecutive_failures or 0)
|
||||
)
|
||||
|
||||
|
||||
async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
"""Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
|
||||
|
||||
@@ -123,6 +149,9 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
cooldown is the preventive half of the burst-prevention pair (per-source
|
||||
consecutive_failures backoff handles the offending source itself).
|
||||
|
||||
A running backfill (`backfill_ready`) is due on every tick, and whether
|
||||
or not its artist is on auto-check — the operator started it by hand.
|
||||
|
||||
Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked
|
||||
sources go first, then the longest-since-checked, so the most overdue
|
||||
sources hit Celery's FIFO download queue first. Anti-starvation: if
|
||||
@@ -135,7 +164,6 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
.options(selectinload(Source.artist))
|
||||
.join(Artist, Source.artist_id == Artist.id)
|
||||
.where(Source.enabled.is_(True))
|
||||
.where(Artist.auto_check.is_(True))
|
||||
.order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
|
||||
)).scalars().all()
|
||||
|
||||
@@ -147,6 +175,11 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||
for s in rows:
|
||||
if s.platform in cooldowns:
|
||||
continue
|
||||
if backfill_ready(s):
|
||||
due.append(s)
|
||||
continue
|
||||
if not s.artist.auto_check:
|
||||
continue
|
||||
interval = compute_effective_interval(s, s.artist, settings)
|
||||
if s.last_checked_at is None:
|
||||
due.append(s)
|
||||
@@ -163,6 +196,8 @@ def compute_next_check_at(
|
||||
"""Return the projected datetime of the next check, or None if never checked."""
|
||||
if source.last_checked_at is None:
|
||||
return None
|
||||
if backfill_ready(source):
|
||||
return datetime.now(UTC)
|
||||
interval = compute_effective_interval(source, artist, settings)
|
||||
return source.last_checked_at + timedelta(seconds=interval)
|
||||
|
||||
|
||||
@@ -18,6 +18,7 @@ from ..models import (
|
||||
Source,
|
||||
)
|
||||
from .db_helpers import failing_sources_clause
|
||||
from .download_backends import uses_native_ingester
|
||||
from .gallery_dl import ErrorType
|
||||
from .membership_reconcile import KEPT_KEY, STOPPED_KEY
|
||||
from .membership_roster import gated_reasons_for_sources
|
||||
@@ -125,6 +126,11 @@ class SourceRecord:
|
||||
"backfill_posts": self.backfill_posts,
|
||||
"tier_gated_count": self.tier_gated_count,
|
||||
"gated_reason": self.gated_reason,
|
||||
# Recover / recapture exist only on the native ingester. Sent so the
|
||||
# UI asks the backend's own predicate instead of keeping a copy of
|
||||
# the platform list — the copy said "patreon, subscribestar" for a
|
||||
# day after Discord went native (milestone 428).
|
||||
"native_ingester": uses_native_ingester(self.platform),
|
||||
}
|
||||
|
||||
|
||||
@@ -551,8 +557,8 @@ class SourceService:
|
||||
whole source); the two flags are mutually exclusive, so arming recapture
|
||||
clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
|
||||
fresh from the top. The flag is cleared on completion (download_service)
|
||||
and on stop. Recapture is Patreon-only (the native ingester's post-record
|
||||
capture); inert elsewhere. The UI gates the action to Patreon sources."""
|
||||
and on stop. Recapture needs the native ingester's post-record capture,
|
||||
so the UI offers it on native sources only (`native_ingester`)."""
|
||||
source = (await self.session.execute(
|
||||
select(Source).where(Source.id == source_id)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
@@ -211,11 +211,10 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
|
||||
state.reserved += len(reserved.get(hostname, []))
|
||||
state.consuming.update(q["name"] for q in queues)
|
||||
|
||||
# `pool.max-concurrency` is the number pool_grow/pool_shrink move and
|
||||
# the number the UI shows. Absent on a worker whose stats did not
|
||||
# answer, which leaves pool=None — unknown, not zero.
|
||||
pool = (stats.get(hostname) or {}).get("pool", {}).get("max-concurrency")
|
||||
if isinstance(pool, int):
|
||||
# Absent on a worker whose stats did not answer, which leaves
|
||||
# pool=None — unknown, not zero.
|
||||
pool = pool_size((stats.get(hostname) or {}).get("pool") or {})
|
||||
if pool is not None:
|
||||
state.pools[hostname] = pool
|
||||
|
||||
for state in out.values():
|
||||
@@ -223,6 +222,27 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
|
||||
return out
|
||||
|
||||
|
||||
def pool_size(pool_stats: dict) -> int | None:
|
||||
"""How many processes a prefork pool is running NOW, from `inspect stats`.
|
||||
|
||||
The length of `processes`, not `max-concurrency`. Celery reports
|
||||
`max-concurrency` as the pool's `limit`, set once at boot; `pool_grow` and
|
||||
`pool_shrink` hand straight to billiard and never touch it (celery 5.6
|
||||
`concurrency/prefork.py`: `self.grow = P.grow`). So it read 1 forever on a
|
||||
lane booted at 1, and the sizing sweep, computing `target - 1` on every
|
||||
tick, grew a scheduler capped at 2 to six processes while the System tab
|
||||
showed "6 / 1" (2026-09-24) — and could never shrink one, since 1 - 1 is 0.
|
||||
|
||||
`max-concurrency` is the fallback only for a pool that lists no processes
|
||||
(a non-prefork pool), where it is the best number there is.
|
||||
"""
|
||||
procs = pool_stats.get("processes")
|
||||
if isinstance(procs, list):
|
||||
return len(procs)
|
||||
limit = pool_stats.get("max-concurrency")
|
||||
return limit if isinstance(limit, int) else None
|
||||
|
||||
|
||||
def effective_slots(target: int) -> int:
|
||||
"""What a pool can actually be set to. Never below one process.
|
||||
|
||||
|
||||
@@ -473,6 +473,10 @@ def prune_task_runs() -> dict:
|
||||
(recover_stalled_task_runs) is the mechanism that flips them to
|
||||
terminal state; prune doesn't touch in-flight state.
|
||||
- 'retry' rows: treated as failures (>7d).
|
||||
- The NEWEST row of each task is never deleted, whatever its age: it is
|
||||
what the beat scheduler reads to know when a job last ran (#4408).
|
||||
Without it a weekly job's last run would be pruned after a day, and beat
|
||||
would think it had never run and fire it on every restart.
|
||||
|
||||
Returns dict of how many rows were deleted in each bucket.
|
||||
"""
|
||||
@@ -480,16 +484,19 @@ def prune_task_runs() -> dict:
|
||||
now = datetime.now(UTC)
|
||||
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
|
||||
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
|
||||
newest = select(func.max(TaskRun.id)).group_by(TaskRun.task_name)
|
||||
with SessionLocal() as session:
|
||||
ok_deleted = session.execute(
|
||||
delete(TaskRun)
|
||||
.where(TaskRun.status == "ok")
|
||||
.where(TaskRun.finished_at < ok_cutoff)
|
||||
.where(TaskRun.id.not_in(newest))
|
||||
).rowcount or 0
|
||||
fail_deleted = session.execute(
|
||||
delete(TaskRun)
|
||||
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
|
||||
.where(TaskRun.finished_at < fail_cutoff)
|
||||
.where(TaskRun.id.not_in(newest))
|
||||
).rowcount or 0
|
||||
session.commit()
|
||||
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||
|
||||
+3
-3
@@ -15,7 +15,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||
|
||||
## Secondary runtime image
|
||||
|
||||
node:24-bookworm-slim — `.forgejo/workflows/extension.yml` only.
|
||||
node:24-bookworm-slim — `build.yml`'s `extension-test` job only.
|
||||
|
||||
`.forgejo/workflows/release.yml` runs on `ci-python:3.14` like everything else
|
||||
and installs nothing: it needs git and stdlib python, and builds no image.
|
||||
@@ -29,8 +29,8 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
||||
|
||||
- `pip install -r requirements.txt pytest pytest-asyncio` — in `backend-lint-and-test` and `integration` jobs
|
||||
- `npm install --no-audit --no-fund` — in `frontend-build` job
|
||||
- `npm install --no-audit --no-fund` — in `extension.yml`'s `lint` job (web-ext + vitest)
|
||||
- `unzip` — in `extension.yml`'s "Verify XPI contents" step, installed via apt
|
||||
- `npm install --no-audit --no-fund` — in `build.yml`'s `extension-test` job (web-ext + vitest)
|
||||
- `unzip` — in `extension-test`'s "Verify XPI contents" step, installed via apt
|
||||
only when absent (`node:24-bookworm-slim` may or may not carry it). Debian
|
||||
package, ~2s. Not worth baking into a shared image for a single consumer, per
|
||||
`docs/process.md`'s ">1 project" rule.
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ x-celery-healthcheck: &celery_healthcheck
|
||||
|
||||
services:
|
||||
redis:
|
||||
image: redis:7-alpine
|
||||
image: redis:8-alpine
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
|
||||
@@ -128,7 +128,8 @@ browser.webRequest.onBeforeSendHeaders.addListener(
|
||||
saveDiscordToken(auth.value);
|
||||
}
|
||||
},
|
||||
{ urls: ['https://discord.com/api/*'] },
|
||||
// ptb/canary are Discord's beta clients; their API calls carry the same token.
|
||||
{ urls: ['https://discord.com/api/*', 'https://*.discord.com/api/*'] },
|
||||
['requestHeaders'],
|
||||
);
|
||||
|
||||
@@ -213,7 +214,17 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
if (key === 'discord') {
|
||||
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
||||
await api.uploadCredentials('discord', 'token', discordToken);
|
||||
return { success: true };
|
||||
// Then have FC try it against a Discord source, so a token Discord
|
||||
// has already revoked shows up here rather than at the next check.
|
||||
// A failed verify never undoes the upload: valid=null means FC could
|
||||
// not test (no Discord source yet), not that the token is bad.
|
||||
let verify = null;
|
||||
try {
|
||||
verify = await api.verifyCredential('discord');
|
||||
} catch (e) {
|
||||
verify = { valid: null, reason: e.message };
|
||||
}
|
||||
return { success: true, verify };
|
||||
}
|
||||
return { error: 'Unsupported platform.' };
|
||||
} catch (e) {
|
||||
@@ -256,14 +267,25 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
||||
|
||||
case 'ADD_AS_SOURCE':
|
||||
try {
|
||||
return await api.quickAddSource(msg.url);
|
||||
return await api.quickAddSource(msg.url, {
|
||||
artistId: msg.artistId ?? null,
|
||||
artistName: msg.artistName ?? null,
|
||||
usePlatformName: msg.usePlatformName === true,
|
||||
});
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'SEARCH_ARTISTS':
|
||||
try {
|
||||
return { artists: await api.searchArtists(msg.q || '') };
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
case 'PROBE_SOURCE':
|
||||
try {
|
||||
return await api.probeSource(msg.url);
|
||||
return await api.probeSource(msg.url, { names: msg.names === true });
|
||||
} catch (e) {
|
||||
return { error: e.message };
|
||||
}
|
||||
|
||||
@@ -40,3 +40,63 @@
|
||||
from { transform: translateY(20px); opacity: 0; }
|
||||
to { transform: translateY(0); opacity: 1; }
|
||||
}
|
||||
|
||||
/* Discord Add panel — sits above the chip. Same slate/parchment palette. */
|
||||
.fc-panel {
|
||||
all: revert;
|
||||
position: fixed; bottom: 76px; right: 24px; z-index: 2147483647;
|
||||
box-sizing: border-box; width: 320px; max-width: calc(100vw - 48px);
|
||||
padding: 14px 16px; border-radius: 10px;
|
||||
background: rgb(20, 23, 26); color: rgb(232, 228, 216);
|
||||
border: 1px solid rgb(60, 64, 70);
|
||||
font: 14px/1.4 system-ui, sans-serif;
|
||||
box-shadow: 0 8px 28px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
.fc-panel__title { font-weight: 600; color: rgb(244, 186, 122); }
|
||||
.fc-panel__sub { color: rgb(170, 166, 156); font-size: 12px; margin-bottom: 8px; }
|
||||
.fc-panel__label {
|
||||
margin: 10px 0 4px; font-size: 11px; letter-spacing: 0.06em;
|
||||
text-transform: uppercase; color: rgb(170, 166, 156);
|
||||
}
|
||||
.fc-panel__radio { display: flex; gap: 8px; align-items: center; padding: 2px 0; cursor: pointer; }
|
||||
.fc-panel__radio input { margin: 0; accent-color: rgb(244, 186, 122); }
|
||||
.fc-panel__input {
|
||||
all: revert; box-sizing: border-box; width: 100%;
|
||||
padding: 7px 9px; border-radius: 6px;
|
||||
border: 1px solid rgb(70, 74, 80); background: rgb(12, 14, 16); color: inherit;
|
||||
font: inherit;
|
||||
}
|
||||
.fc-panel__input:focus { outline: 2px solid rgb(244, 186, 122); outline-offset: -1px; }
|
||||
.fc-panel__results { display: flex; flex-direction: column; max-height: 160px; overflow-y: auto; }
|
||||
.fc-panel__result {
|
||||
all: revert; text-align: left; padding: 6px 9px; border: none; border-radius: 4px;
|
||||
background: transparent; color: inherit; font: inherit; cursor: pointer;
|
||||
}
|
||||
.fc-panel__result:hover, .fc-panel__result:focus { background: rgb(36, 40, 46); }
|
||||
.fc-panel__hint { margin-top: 8px; font-size: 12px; color: rgb(170, 166, 156); }
|
||||
.fc-panel__actions { display: flex; justify-content: flex-end; gap: 8px; margin-top: 12px; }
|
||||
.fc-panel__btn {
|
||||
all: revert; padding: 6px 14px; border-radius: 999px; cursor: pointer;
|
||||
border: 1px solid rgb(70, 74, 80); background: transparent; color: inherit;
|
||||
font: 500 13px/1.2 system-ui, sans-serif;
|
||||
}
|
||||
.fc-panel__btn--primary { border-color: rgb(244, 186, 122); background: rgb(244, 186, 122); color: rgb(20, 23, 26); }
|
||||
.fc-panel__btn:disabled { opacity: 0.5; cursor: default; }
|
||||
|
||||
/* Artist autocomplete — the list sits under the field, inside the panel. */
|
||||
.fc-panel__combo { position: relative; }
|
||||
.fc-panel__results {
|
||||
margin-top: 4px; border-radius: 6px;
|
||||
background: rgb(12, 14, 16);
|
||||
}
|
||||
.fc-panel__results:empty { display: none; }
|
||||
.fc-panel__results:not(:empty) { border: 1px solid rgb(70, 74, 80); padding: 3px; }
|
||||
.fc-panel__result { display: flex; justify-content: space-between; align-items: center; width: 100%; box-sizing: border-box; }
|
||||
.fc-panel__result--active { background: rgb(52, 44, 32); outline: 1px solid rgb(244, 186, 122); }
|
||||
.fc-panel__result--new { color: rgb(244, 186, 122); }
|
||||
.fc-panel__result-tag { font-size: 11px; color: rgb(140, 220, 160); }
|
||||
.fc-panel__empty { padding: 6px 9px; font-size: 12px; color: rgb(170, 166, 156); }
|
||||
.fc-panel__hint--match { color: rgb(140, 220, 160); }
|
||||
/* The panel's own [hidden] — its rows are display:flex, which beats the UA's. */
|
||||
.fc-panel [hidden] { display: none !important; }
|
||||
.fc-panel__rename { margin-top: 8px; font-size: 13px; }
|
||||
|
||||
@@ -5,42 +5,61 @@
|
||||
// Cached probe result for the current URL so click-handlers know which
|
||||
// action to dispatch without round-tripping again.
|
||||
let currentProbe = null;
|
||||
// Bumped on every evaluate(): a probe that answers after the operator has
|
||||
// navigated on is for a page they've left, and must not repaint the chip.
|
||||
let generation = 0;
|
||||
let lastUrl = window.location.href;
|
||||
|
||||
evaluate();
|
||||
|
||||
const reEval = () => evaluate();
|
||||
window.addEventListener('popstate', reEval);
|
||||
const origPush = history.pushState;
|
||||
history.pushState = function () { origPush.apply(this, arguments); reEval(); };
|
||||
// SPA navigation. Patreon, SubscribeStar and above all Discord change
|
||||
// channel/page without a reload. Patching history.pushState from here never
|
||||
// worked: a content script runs in an isolated world, so the page's own
|
||||
// pushState is not the function we'd replace. Polling the URL is the one
|
||||
// signal that sees every navigation, and costs a string compare.
|
||||
window.addEventListener('popstate', () => onUrlMaybeChanged());
|
||||
setInterval(onUrlMaybeChanged, 500);
|
||||
|
||||
function onUrlMaybeChanged() {
|
||||
if (window.location.href === lastUrl) return;
|
||||
lastUrl = window.location.href;
|
||||
closePanel();
|
||||
evaluate();
|
||||
}
|
||||
|
||||
async function evaluate() {
|
||||
const mine = ++generation;
|
||||
const url = window.location.href;
|
||||
const platform = getPlatformFromUrl(url);
|
||||
const onArtist = platform && isArtistPage(url, platform);
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
if (!onArtist) {
|
||||
if (btn) btn.remove();
|
||||
removeButton();
|
||||
currentProbe = null;
|
||||
return;
|
||||
}
|
||||
// On artist pages, ask the backend what state the URL is in BEFORE
|
||||
// injecting the button — so the chip can render the right state on
|
||||
// first paint instead of flashing the generic "Add" copy and
|
||||
// updating afterwards.
|
||||
// Ask the backend what state the URL is in BEFORE drawing the button, so
|
||||
// the chip renders the right state on first paint instead of flashing the
|
||||
// generic "Add" copy and updating afterwards.
|
||||
let probe;
|
||||
try {
|
||||
probe = await browser.runtime.sendMessage({ type: 'PROBE_SOURCE', url });
|
||||
} catch (e) {
|
||||
probe = { error: e?.message || 'probe failed' };
|
||||
}
|
||||
if (mine !== generation) return;
|
||||
currentProbe = probe;
|
||||
if (probe?.state === 'unknown_platform') {
|
||||
if (btn) btn.remove();
|
||||
removeButton();
|
||||
return;
|
||||
}
|
||||
renderButton(probe);
|
||||
}
|
||||
|
||||
function removeButton() {
|
||||
document.getElementById('fc-add-source-btn')?.remove();
|
||||
closePanel();
|
||||
}
|
||||
|
||||
function renderButton(probe) {
|
||||
let btn = document.getElementById('fc-add-source-btn');
|
||||
if (!btn) {
|
||||
@@ -51,79 +70,37 @@
|
||||
}
|
||||
// Reset state classes so re-renders (SPA navigation) don't stack.
|
||||
btn.className = 'fc-add-source-btn';
|
||||
btn.classList.add(`fc-add-source-btn--${stateModifier(probe)}`);
|
||||
btn.textContent = labelFor(probe);
|
||||
btn.classList.add(`fc-add-source-btn--${chipState(probe)}`);
|
||||
btn.textContent = chipLabel(probe, PLATFORMS[probe?.platform]?.name || probe?.platform || '');
|
||||
btn.disabled = false;
|
||||
}
|
||||
|
||||
function stateModifier(probe) {
|
||||
if (!probe || probe.error) return 'new';
|
||||
return ({
|
||||
source_match: 'source-match',
|
||||
artist_match: 'artist-match',
|
||||
new: 'new',
|
||||
})[probe.state] || 'new';
|
||||
}
|
||||
|
||||
function labelFor(probe) {
|
||||
if (!probe || probe.error) return '+ Add to FabledCurator';
|
||||
const platformName = platformDisplayName(probe.platform);
|
||||
const artistName = probe.artist?.name;
|
||||
switch (probe.state) {
|
||||
case 'source_match':
|
||||
return `✓ In FabledCurator · ${platformName}`;
|
||||
case 'artist_match':
|
||||
return `+ Add ${platformName} source to ${artistName || 'artist'}`;
|
||||
case 'new':
|
||||
default:
|
||||
return '+ Add to FabledCurator';
|
||||
}
|
||||
}
|
||||
|
||||
function platformDisplayName(key) {
|
||||
return PLATFORMS[key]?.name || key || '';
|
||||
}
|
||||
|
||||
async function onClick() {
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
if (!btn) return;
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
const probe = currentProbe;
|
||||
|
||||
if (probe?.state === 'source_match') {
|
||||
btn.textContent = 'Opening…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'OPEN_ARTIST_PAGE',
|
||||
slug: probe.artist?.slug,
|
||||
});
|
||||
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
}
|
||||
await openArtist(btn, probe.artist?.slug);
|
||||
return;
|
||||
}
|
||||
// Every add goes through the panel, so the operator can match the page to
|
||||
// an artist FabledCurator already has (the same creator is often spelled
|
||||
// differently per platform) instead of minting a duplicate.
|
||||
if (document.getElementById('fc-add-panel')) {
|
||||
closePanel();
|
||||
return;
|
||||
}
|
||||
await openAddPanel(btn, probe);
|
||||
}
|
||||
|
||||
btn.textContent = 'Adding…';
|
||||
async function openArtist(btn, slug) {
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
btn.textContent = 'Opening…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({
|
||||
type: 'ADD_AS_SOURCE',
|
||||
url: window.location.href,
|
||||
});
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
} else {
|
||||
const verb = r.created_source ? 'Added' : 'Already a source for';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
||||
// Re-probe so the chip flips green without waiting for the next
|
||||
// navigation.
|
||||
evaluate();
|
||||
return;
|
||||
}
|
||||
const r = await browser.runtime.sendMessage({ type: 'OPEN_ARTIST_PAGE', slug });
|
||||
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
} finally {
|
||||
@@ -132,6 +109,305 @@
|
||||
}
|
||||
}
|
||||
|
||||
// One add, shared by the one-click chip and the Discord panel. Resolves
|
||||
// true on success.
|
||||
async function add(btn, request) {
|
||||
btn.disabled = true;
|
||||
const original = btn.textContent;
|
||||
btn.textContent = 'Adding…';
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'ADD_AS_SOURCE', ...request });
|
||||
if (r?.error) {
|
||||
showToast(`Error: ${r.error}`, 'error');
|
||||
return false;
|
||||
}
|
||||
const verb = r.created_source ? 'Added to' : 'Already a source for';
|
||||
const renamed = r.renamed_from ? ` — renamed from “${r.renamed_from}”` : '';
|
||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})${renamed}`, 'success');
|
||||
// Re-probe so the chip flips green without waiting for a navigation.
|
||||
evaluate();
|
||||
return true;
|
||||
} catch (e) {
|
||||
showToast(`Error: ${e.message}`, 'error');
|
||||
return false;
|
||||
} finally {
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Add panel ----
|
||||
// Who: the suggested artist, one found by search, or a new one by name —
|
||||
// on every platform. Where (Discord only): this channel or the whole
|
||||
// server. On Patreon, joining an artist known by another name offers the
|
||||
// Patreon name, which the operator treats as canon. Built with
|
||||
// createElement only — server, channel and artist names are other people's
|
||||
// text.
|
||||
|
||||
function el(tag, props = {}, children = []) {
|
||||
const node = document.createElement(tag);
|
||||
const { class: className, text, ...rest } = props;
|
||||
if (className) node.className = className;
|
||||
if (text != null) node.textContent = text;
|
||||
Object.assign(node, rest);
|
||||
for (const c of children) node.appendChild(c);
|
||||
return node;
|
||||
}
|
||||
|
||||
function closePanel() {
|
||||
document.getElementById('fc-add-panel')?.remove();
|
||||
}
|
||||
|
||||
async function openAddPanel(btn, probe) {
|
||||
closePanel();
|
||||
const platformName = PLATFORMS[probe.platform]?.name || probe.platform;
|
||||
// Discord's probe already carries its names. Patreon/SubscribeStar read the
|
||||
// creator's display name only now, when the panel needs it — a request to
|
||||
// the platform the chip's own probe deliberately doesn't make.
|
||||
if (probe.platform !== 'discord') {
|
||||
const original = btn.textContent;
|
||||
btn.disabled = true;
|
||||
btn.textContent = `Reading the ${platformName} name…`;
|
||||
try {
|
||||
const named = await browser.runtime.sendMessage({
|
||||
type: 'PROBE_SOURCE', url: window.location.href, names: true,
|
||||
});
|
||||
if (named && !named.error) probe = named;
|
||||
} catch { /* fall back to the chip's probe: the URL handle */ }
|
||||
btn.disabled = false;
|
||||
btn.textContent = original;
|
||||
if (probe.state === 'source_match') {
|
||||
renderButton(probe);
|
||||
return;
|
||||
}
|
||||
}
|
||||
const d = probe.discord || {};
|
||||
const discord = probe.platform === 'discord';
|
||||
const choice = panelDefaults(probe, window.location.href);
|
||||
|
||||
const scopeRow = (value, label, disabled) => {
|
||||
const input = el('input', {
|
||||
type: 'radio', name: 'fc-discord-scope', value,
|
||||
checked: choice.scope === value, disabled,
|
||||
});
|
||||
input.addEventListener('change', () => { choice.scope = value; refresh(); });
|
||||
return el('label', { class: 'fc-panel__radio' }, [input, el('span', { text: label })]);
|
||||
};
|
||||
|
||||
// The artist field is an autocomplete over FC's artists: it searches as
|
||||
// soon as the panel opens (with the prefilled name) and on every keystroke,
|
||||
// lists the matches under the field, fills in the rest of the top match as
|
||||
// you type (Tab or Enter accepts it, typing on replaces it), and picks an
|
||||
// artist whose name IS the text, spacing and case aside, without being
|
||||
// asked. ↑/↓ walk the list; the last row creates a new artist instead.
|
||||
const nameInput = el('input', {
|
||||
type: 'text', class: 'fc-panel__input', value: choice.artistName,
|
||||
placeholder: 'Search artists or type a new name',
|
||||
autocomplete: 'off', spellcheck: false,
|
||||
role: 'combobox',
|
||||
});
|
||||
nameInput.setAttribute('aria-autocomplete', 'both');
|
||||
nameInput.setAttribute('aria-expanded', 'false');
|
||||
const results = el('div', { class: 'fc-panel__results', role: 'listbox' });
|
||||
const hint = el('div', { class: 'fc-panel__hint' });
|
||||
const addBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Add' });
|
||||
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Cancel' });
|
||||
|
||||
// Patreon is canon: joining an artist known by another name takes the
|
||||
// Patreon name unless this is unticked. Shown only when it would rename.
|
||||
const renameBox = el('input', { type: 'checkbox', checked: choice.adoptPlatformName });
|
||||
const renameText = el('span');
|
||||
const renameRow = el('label', { class: 'fc-panel__radio fc-panel__rename' }, [renameBox, renameText]);
|
||||
renameBox.addEventListener('change', () => { choice.adoptPlatformName = renameBox.checked; refresh(); });
|
||||
|
||||
const sub = discord
|
||||
? serverLabel(d)
|
||||
: [probe.display_name, probe.slug].filter(Boolean).filter((v, i, a) => a.indexOf(v) === i).join(' · ');
|
||||
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||
el('div', { class: 'fc-panel__title', text: `Add ${platformName} source` }),
|
||||
el('div', { class: 'fc-panel__sub', text: sub }),
|
||||
...(discord ? [
|
||||
el('div', { class: 'fc-panel__label', text: 'Follow' }),
|
||||
scopeRow('channel', d.channel_id ? channelLabel(d) : 'this channel', !d.channel_id),
|
||||
scopeRow('server', `Every channel in ${serverLabel(d)}`, false),
|
||||
] : []),
|
||||
el('div', { class: 'fc-panel__label', text: 'Artist' }),
|
||||
el('div', { class: 'fc-panel__combo' }, [nameInput, results]),
|
||||
renameRow,
|
||||
hint,
|
||||
el('div', { class: 'fc-panel__actions' }, [cancelBtn, addBtn]),
|
||||
]);
|
||||
|
||||
// Search state: the rows on screen, which one ↑/↓ has highlighted (-1 =
|
||||
// none), and whether the list is open.
|
||||
let rows = [];
|
||||
let active = -1;
|
||||
let listOpen = false;
|
||||
|
||||
function refresh() {
|
||||
const req = addRequest(choice);
|
||||
addBtn.disabled = !req;
|
||||
if (!req) hint.textContent = 'Pick an artist or type a name.';
|
||||
else if (req.artistId != null) hint.textContent = `✓ Connects to ${choice.artist.name}, already in FabledCurator.`;
|
||||
else if (req.artistName) hint.textContent = `Creates a new artist “${req.artistName}”.`;
|
||||
else hint.textContent = `Creates a new artist, named from the ${platformName} page.`;
|
||||
hint.classList.toggle('fc-panel__hint--match', !!req && req.artistId != null);
|
||||
const offer = renameOffer(choice);
|
||||
renameRow.hidden = !offer;
|
||||
if (offer) renameText.textContent = `Rename “${offer.from}” to the Patreon name “${offer.to}”`;
|
||||
}
|
||||
|
||||
function pick(artist) {
|
||||
choice.artist = artist ? { id: artist.id, name: artist.name } : null;
|
||||
if (artist) {
|
||||
choice.artistName = artist.name;
|
||||
nameInput.value = artist.name;
|
||||
}
|
||||
closeList();
|
||||
refresh();
|
||||
}
|
||||
|
||||
function closeList() {
|
||||
listOpen = false;
|
||||
active = -1;
|
||||
results.replaceChildren();
|
||||
nameInput.setAttribute('aria-expanded', 'false');
|
||||
}
|
||||
|
||||
function renderList() {
|
||||
const typed = nameInput.value.trim();
|
||||
const exact = exactArtistMatch(typed, rows);
|
||||
const items = rows.map((a, i) => {
|
||||
const row = el('button', { type: 'button', class: 'fc-panel__result', role: 'option' }, [
|
||||
el('span', { text: a.name }),
|
||||
]);
|
||||
if (choice.artist && choice.artist.id === a.id) {
|
||||
row.appendChild(el('span', { class: 'fc-panel__result-tag', text: 'selected' }));
|
||||
}
|
||||
row.classList.toggle('fc-panel__result--active', i === active);
|
||||
// mousedown, not click: it fires before the input's blur closes the list.
|
||||
row.addEventListener('mousedown', (e) => { e.preventDefault(); pick(a); });
|
||||
return row;
|
||||
});
|
||||
if (typed && !exact) {
|
||||
const i = rows.length;
|
||||
const create = el('button', { type: 'button', class: 'fc-panel__result fc-panel__result--new', role: 'option',
|
||||
text: `+ New artist “${typed}”` });
|
||||
create.classList.toggle('fc-panel__result--active', i === active);
|
||||
create.addEventListener('mousedown', (e) => { e.preventDefault(); pick(null); choice.artistName = typed; refresh(); });
|
||||
items.push(create);
|
||||
}
|
||||
if (typed && !rows.length) {
|
||||
items.unshift(el('div', { class: 'fc-panel__empty', text: 'No FabledCurator artist matches.' }));
|
||||
}
|
||||
results.replaceChildren(...items);
|
||||
listOpen = items.length > 0;
|
||||
nameInput.setAttribute('aria-expanded', String(listOpen));
|
||||
results.querySelector('.fc-panel__result--active')?.scrollIntoView({ block: 'nearest' });
|
||||
}
|
||||
|
||||
let debounce = null;
|
||||
let searchSeq = 0;
|
||||
// `autofill` is false for deletions: filling the name back in as you
|
||||
// backspace would make it impossible to delete.
|
||||
function search(autofill) {
|
||||
clearTimeout(debounce);
|
||||
const q = nameInput.value.trim();
|
||||
if (!q) { rows = []; closeList(); return; }
|
||||
debounce = setTimeout(async () => {
|
||||
const mine = ++searchSeq;
|
||||
let r;
|
||||
try {
|
||||
r = await browser.runtime.sendMessage({ type: 'SEARCH_ARTISTS', q });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
// Stale: a later keystroke has its own search coming.
|
||||
if (mine !== searchSeq || r?.error || nameInput.value.trim() !== q) return;
|
||||
rows = r.artists || [];
|
||||
active = -1;
|
||||
const exact = exactArtistMatch(q, rows);
|
||||
if (exact && !choice.artist) {
|
||||
choice.artist = { id: exact.id, name: exact.name };
|
||||
} else if (autofill && document.activeElement === nameInput) {
|
||||
const hit = inlineCompletion(nameInput.value, rows);
|
||||
const caret = nameInput.value.length;
|
||||
if (hit && nameInput.selectionStart === caret) {
|
||||
nameInput.value = nameInput.value + hit.name.slice(caret);
|
||||
nameInput.setSelectionRange(caret, hit.name.length);
|
||||
choice.artist = { id: hit.id, name: hit.name };
|
||||
choice.artistName = hit.name;
|
||||
}
|
||||
}
|
||||
renderList();
|
||||
refresh();
|
||||
}, 150);
|
||||
}
|
||||
|
||||
nameInput.addEventListener('input', (e) => {
|
||||
choice.artistName = nameInput.value;
|
||||
// Any edit un-picks: the artist is whatever the field now says.
|
||||
choice.artist = null;
|
||||
refresh();
|
||||
search(!String(e.inputType || '').startsWith('delete'));
|
||||
});
|
||||
nameInput.addEventListener('focus', () => { if (rows.length) renderList(); });
|
||||
nameInput.addEventListener('blur', () => closeList());
|
||||
|
||||
// Keep Discord's global shortcuts from eating keystrokes meant for us.
|
||||
panel.addEventListener('keydown', (e) => {
|
||||
e.stopPropagation();
|
||||
const count = results.querySelectorAll('.fc-panel__result').length;
|
||||
if (e.target === nameInput && listOpen && count) {
|
||||
if (e.key === 'ArrowDown' || e.key === 'ArrowUp') {
|
||||
e.preventDefault();
|
||||
active = e.key === 'ArrowDown' ? (active + 1) % count : (active <= 0 ? count - 1 : active - 1);
|
||||
renderList();
|
||||
return;
|
||||
}
|
||||
if ((e.key === 'Enter' || e.key === 'Tab') && active >= 0) {
|
||||
e.preventDefault();
|
||||
results.querySelectorAll('.fc-panel__result')[active]
|
||||
.dispatchEvent(new MouseEvent('mousedown', { cancelable: true }));
|
||||
return;
|
||||
}
|
||||
if ((e.key === 'Tab' || e.key === 'Enter') && choice.artist
|
||||
&& nameInput.selectionEnd > nameInput.selectionStart) {
|
||||
// Accept the inline autofill — Enter too, but only to accept: the
|
||||
// add itself takes a second Enter, once the hint names the artist.
|
||||
e.preventDefault();
|
||||
pick(choice.artist);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (e.key === 'Escape') {
|
||||
if (listOpen) closeList();
|
||||
else closePanel();
|
||||
return;
|
||||
}
|
||||
if (e.key === 'Enter' && e.target === nameInput && !addBtn.disabled) addBtn.click();
|
||||
});
|
||||
|
||||
cancelBtn.addEventListener('click', closePanel);
|
||||
addBtn.addEventListener('click', async () => {
|
||||
const req = addRequest(choice);
|
||||
if (!req) return;
|
||||
const btn = document.getElementById('fc-add-source-btn');
|
||||
addBtn.disabled = true;
|
||||
const ok = await add(btn || addBtn, req);
|
||||
if (ok) closePanel();
|
||||
else refresh();
|
||||
});
|
||||
|
||||
document.body.appendChild(panel);
|
||||
refresh();
|
||||
nameInput.focus();
|
||||
nameInput.select();
|
||||
// Search what the field opens with — the server's name, usually — so an
|
||||
// artist it already matches is picked before the operator types anything.
|
||||
if (!choice.artist && nameInput.value.trim()) search(false);
|
||||
}
|
||||
|
||||
function showToast(text, kind) {
|
||||
const t = document.createElement('div');
|
||||
t.className = `fc-toast fc-toast--${kind}`;
|
||||
|
||||
+29
-5
@@ -80,6 +80,17 @@ class FabledCuratorAPI {
|
||||
getCredentials() {
|
||||
return this.request('GET', '/credentials');
|
||||
}
|
||||
// Test the STORED credential against one of the platform's sources — the
|
||||
// same check the web UI's Verify button runs. {valid: true|false|null, reason}.
|
||||
verifyCredential(platform) {
|
||||
return this.request('POST', `/credentials/${encodeURIComponent(platform)}/verify`);
|
||||
}
|
||||
|
||||
// Artist search for the Discord Add panel — the web UI's autocomplete.
|
||||
searchArtists(q, limit = 8) {
|
||||
const qs = new URLSearchParams({ q, limit: String(limit) }).toString();
|
||||
return this.request('GET', `/artists/autocomplete?${qs}`);
|
||||
}
|
||||
|
||||
// FC-3a — sources.
|
||||
listSources() {
|
||||
@@ -90,13 +101,26 @@ class FabledCuratorAPI {
|
||||
}
|
||||
|
||||
// FC-3g — extension-specific.
|
||||
quickAddSource(url) {
|
||||
return this.request('POST', '/extension/quick-add-source', { url });
|
||||
// artistId connects the source to an existing artist, artistName to the
|
||||
// artist of that name (created if new); with neither the server derives the
|
||||
// artist from the URL. A Discord channel always sends one.
|
||||
// usePlatformName: a Patreon source joining an existing artist renames it
|
||||
// to the Patreon display name (Patreon is canon; name only, never the slug).
|
||||
quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false } = {}) {
|
||||
const body = { url };
|
||||
if (artistId != null) body.artist_id = artistId;
|
||||
else if (artistName) body.artist_name = artistName;
|
||||
if (usePlatformName) body.use_platform_name = true;
|
||||
return this.request('POST', '/extension/quick-add-source', body);
|
||||
}
|
||||
probeSource(url) {
|
||||
probeSource(url, { names = false } = {}) {
|
||||
// Read-only existence check. Drives the content-script chip's
|
||||
// color/copy BEFORE the operator clicks Add.
|
||||
const qs = new URLSearchParams({ url }).toString();
|
||||
// color/copy BEFORE the operator clicks Add. `names` also reads the
|
||||
// creator's display name from the platform — the Add panel asks for it,
|
||||
// the chip doesn't, so a plain page view never costs a platform request.
|
||||
const params = { url };
|
||||
if (names) params.names = '1';
|
||||
const qs = new URLSearchParams(params).toString();
|
||||
return this.request('GET', `/extension/probe?${qs}`);
|
||||
}
|
||||
// Latest published extension version on this instance — drives the in-app
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
/**
|
||||
* The content script's decisions, kept apart from its DOM so the specs can
|
||||
* load them (test/chip.spec.js): which state the chip shows, what it says,
|
||||
* and what the Add panel starts out proposing and finally sends.
|
||||
*
|
||||
* `probe` is /api/extension/probe's answer; `platformName` is the display
|
||||
* name (PLATFORMS[key].name), passed in so this file needs no other lib.
|
||||
*/
|
||||
|
||||
function chipState(probe) {
|
||||
if (!probe || probe.error) return 'new';
|
||||
return ({ source_match: 'source-match', artist_match: 'artist-match', new: 'new' })[probe.state] || 'new';
|
||||
}
|
||||
|
||||
function chipLabel(probe, platformName) {
|
||||
if (!probe || probe.error) return '+ Add to FabledCurator';
|
||||
const artist = probe.artist?.name || 'artist';
|
||||
if (probe.platform === 'discord') {
|
||||
const d = probe.discord || {};
|
||||
if (probe.state === 'source_match') {
|
||||
return probe.covered_by_server
|
||||
? `✓ Whole server in FabledCurator · ${artist}`
|
||||
: `✓ In FabledCurator · ${artist}`;
|
||||
}
|
||||
// Every other Discord state opens the panel: the artist is always chosen.
|
||||
return d.channel_id ? `+ Add ${channelLabel(d)} to FabledCurator` : '+ Add server to FabledCurator';
|
||||
}
|
||||
switch (probe.state) {
|
||||
case 'source_match':
|
||||
return `✓ In FabledCurator · ${platformName}`;
|
||||
case 'artist_match':
|
||||
return `+ Add ${platformName} source to ${artist}`;
|
||||
default:
|
||||
return '+ Add to FabledCurator';
|
||||
}
|
||||
}
|
||||
|
||||
/** `#name` when the probe could read it, else a neutral "this channel". */
|
||||
function channelLabel(d) {
|
||||
return d.channel_name ? `#${d.channel_name}` : 'this channel';
|
||||
}
|
||||
|
||||
/** `name` when the probe could read it, else "this server". */
|
||||
function serverLabel(d) {
|
||||
return d.server_name || 'this server';
|
||||
}
|
||||
|
||||
/**
|
||||
* What the Add panel opens with, on any platform.
|
||||
*
|
||||
* Discord: the channel is the default scope when there is one — a server
|
||||
* source walks every channel the token can read, which is rarely what a
|
||||
* single art channel wants. The artist is the probe's suggestion (the owner
|
||||
* of another source on this server), else a new one named after the server.
|
||||
*
|
||||
* Patreon / SubscribeStar: the page URL is the source. The artist is the one
|
||||
* whose slug the URL already names (artist_match), else a new one under the
|
||||
* creator's display name — `probe.display_name`, from the probe the panel
|
||||
* makes with names=1 — falling back to the URL handle, which the server
|
||||
* resolves on its own when it is left untouched (`nameIsHandle`).
|
||||
*/
|
||||
function panelDefaults(probe, pageUrl) {
|
||||
const d = probe?.discord || {};
|
||||
const suggested = probe?.state === 'artist_match' && probe.artist ? probe.artist : null;
|
||||
const discord = probe?.platform === 'discord';
|
||||
const shown = probe?.display_name || null;
|
||||
let artistName = '';
|
||||
if (suggested) artistName = suggested.name;
|
||||
else if (discord) artistName = d.server_name || '';
|
||||
else artistName = shown || probe?.slug || '';
|
||||
return {
|
||||
platform: probe?.platform || null,
|
||||
scope: discord ? (d.channel_id ? 'channel' : 'server') : 'page',
|
||||
pageUrl: pageUrl || null,
|
||||
channelUrl: d.channel_url || null,
|
||||
serverUrl: d.server_url || null,
|
||||
artist: suggested ? { id: suggested.id, name: suggested.name } : null,
|
||||
artistName,
|
||||
nameIsHandle: !discord && !suggested && !shown,
|
||||
handle: probe?.slug || '',
|
||||
displayName: shown,
|
||||
// Patreon is canon: joining an artist known by another name takes the
|
||||
// Patreon name, unless the operator unticks it.
|
||||
adoptPlatformName: true,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The rename the panel offers, or null: only on Patreon (the canon name),
|
||||
* only when joining an existing artist, only with a name actually read from
|
||||
* Patreon, and only when it differs from what the artist is called now.
|
||||
*/
|
||||
function renameOffer(choice) {
|
||||
if (choice.platform !== 'patreon' || !choice.artist || !choice.displayName) return null;
|
||||
if (choice.artist.name === choice.displayName) return null;
|
||||
return { from: choice.artist.name, to: choice.displayName };
|
||||
}
|
||||
|
||||
/**
|
||||
* The quick-add body for the panel's current choice. A picked artist goes by
|
||||
* id — names can collide once slugified — and a typed name creates (or
|
||||
* finds) that artist. The URL handle left untouched sends no name, so the
|
||||
* server resolves the display name itself. null when there is nothing valid.
|
||||
*/
|
||||
function addRequest(choice) {
|
||||
let url = choice.pageUrl;
|
||||
if (choice.scope === 'server') url = choice.serverUrl;
|
||||
else if (choice.scope === 'channel') url = choice.channelUrl;
|
||||
if (!url) return null;
|
||||
if (choice.artist && choice.artist.id != null && choice.artist.name === choice.artistName) {
|
||||
const req = { url, artistId: choice.artist.id };
|
||||
if (choice.adoptPlatformName && renameOffer(choice)) req.usePlatformName = true;
|
||||
return req;
|
||||
}
|
||||
const name = (choice.artistName || '').trim();
|
||||
if (!name) return null;
|
||||
if (choice.nameIsHandle && name === choice.handle) return { url };
|
||||
return { url, artistName: name };
|
||||
}
|
||||
|
||||
/**
|
||||
* An artist name reduced to what identifies it: lowercase letters and digits
|
||||
* of any script, nothing else — so "Tamada Heijun", "tamada_heijun" and
|
||||
* "TamadaHeijun" are one name. Mirrors the server's autocomplete (#429).
|
||||
*/
|
||||
function squashName(name) {
|
||||
return String(name || '').toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
|
||||
}
|
||||
|
||||
/** The search result that IS the query, spacing aside, else null. */
|
||||
function exactArtistMatch(query, results) {
|
||||
const q = squashName(query);
|
||||
if (!q) return null;
|
||||
return (results || []).find((a) => squashName(a.name) === q) || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Inline autofill: the first result whose name extends what was typed
|
||||
* (case-insensitive), so the panel can fill in the rest and select it —
|
||||
* typing on overwrites it, Tab or Enter accepts it. null when none does.
|
||||
*/
|
||||
function inlineCompletion(typed, results) {
|
||||
const t = String(typed || '').toLowerCase();
|
||||
if (!t) return null;
|
||||
return (results || []).find((a) => a.name.length > t.length && a.name.toLowerCase().startsWith(t)) || null;
|
||||
}
|
||||
@@ -57,7 +57,8 @@ const PLATFORMS = {
|
||||
domains: ['.discord.com', 'discord.com'],
|
||||
authType: 'token',
|
||||
color: '#5865F2',
|
||||
urlPattern: /^https?:\/\/(www\.)?discord\.com/,
|
||||
// ptb/canary are Discord's beta clients — same channels, same token.
|
||||
urlPattern: /^https?:\/\/((www|ptb|canary)\.)?discord\.com/,
|
||||
note: 'Open Discord in browser to capture token',
|
||||
},
|
||||
};
|
||||
@@ -80,8 +81,23 @@ const PLATFORM_ARTIST_PATTERNS = {
|
||||
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i,
|
||||
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
||||
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
||||
// A Discord URL names a server or channel, not a creator: the backend's slug
|
||||
// is `<server>[/<channel>]` and the Add panel asks which artist it belongs
|
||||
// to. A message jump link still names its channel; DMs (@me) and thread
|
||||
// links don't match. Mirrors extension_service._PLATFORM_PATTERNS.
|
||||
discord: /^https?:\/\/(?:www\.|ptb\.|canary\.)?discord\.com\/channels\/\d+(?:\/\d+)?(?:\/\d+)?\/?(?:[?#].*)?$/i,
|
||||
};
|
||||
|
||||
/**
|
||||
* `{serverId, channelId}` from a Discord channel/server URL the artist
|
||||
* pattern accepts, else null. channelId is null for a whole-server URL.
|
||||
*/
|
||||
function parseDiscordUrl(url) {
|
||||
if (!PLATFORM_ARTIST_PATTERNS.discord.test(url || '')) return null;
|
||||
const m = /\/channels\/(\d+)(?:\/(\d+))?/.exec(url);
|
||||
return m ? { serverId: m[1], channelId: m[2] || null } : null;
|
||||
}
|
||||
|
||||
function getPlatformFromUrl(url) {
|
||||
for (const [key, platform] of Object.entries(PLATFORMS)) {
|
||||
if (platform.urlPattern.test(url)) return key;
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* The popup's wording, kept apart from its DOM so the specs can load it
|
||||
* (test/popup-format.spec.js). Classic script, like the rest of lib/.
|
||||
*/
|
||||
|
||||
/**
|
||||
* One line of state for a source row, from /api/sources' fields, with the
|
||||
* status class the popup colours it by ('ready' | 'error' | 'no-cookies' for
|
||||
* a warning | '' for plain). The most actionable fact wins: an error before
|
||||
* a running backfill, a backfill before the last-checked time.
|
||||
*/
|
||||
function sourceStatus(src, now = Date.now()) {
|
||||
if (!src.enabled) return { text: 'Disabled', kind: '' };
|
||||
if (src.last_error) {
|
||||
const first = String(src.last_error).split('\n')[0].trim();
|
||||
const short = first.length > 90 ? `${first.slice(0, 89)}…` : first;
|
||||
return { text: `Error — ${short}`, kind: 'error' };
|
||||
}
|
||||
if (src.backfill_state === 'running') {
|
||||
const n = src.backfill_chunks || 0;
|
||||
return { text: n ? `Backfilling — ${n} chunk${n === 1 ? '' : 's'} done` : 'Backfill queued', kind: 'ready' };
|
||||
}
|
||||
if (src.backfill_state === 'stalled') return { text: 'Backfill stalled', kind: 'no-cookies' };
|
||||
if (!src.last_checked_at) return { text: 'Not checked yet', kind: '' };
|
||||
return { text: `Checked ${relativeTime(src.last_checked_at, now)}`, kind: '' };
|
||||
}
|
||||
|
||||
// Same buckets and wording as the web UI's canonical formatRelative
|
||||
// (frontend/src/utils/date.js, snippet #3959) — the extension can't import it (classic
|
||||
// scripts, separate package), so it mirrors it: "42s ago", "5m ago", "3h ago",
|
||||
// "2d ago", and "Never" for a missing or unreadable time.
|
||||
function relativeTime(iso, now = Date.now()) {
|
||||
const t = iso ? Date.parse(iso) : NaN;
|
||||
if (Number.isNaN(t)) return 'Never';
|
||||
const abs = Math.abs(now - t) / 1000;
|
||||
let body;
|
||||
if (abs < 60) body = `${Math.floor(abs)}s`;
|
||||
else if (abs < 3600) body = `${Math.floor(abs / 60)}m`;
|
||||
else if (abs < 86400) body = `${Math.floor(abs / 3600)}h`;
|
||||
else body = `${Math.floor(abs / 86400)}d`;
|
||||
return `${body} ago`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The popup message after a Discord token export, from FC's verify of the
|
||||
* stored token: {text, kind} with kind 'success' | 'warning' | 'error'.
|
||||
* valid=null is "FC couldn't test it" (no Discord source yet, or a network
|
||||
* hiccup) — a warning with FC's reason, never a failure.
|
||||
*/
|
||||
function tokenExportMessage(verify) {
|
||||
if (!verify) return { text: 'Discord: token exported', kind: 'success' };
|
||||
if (verify.valid === true) return { text: `Discord: token exported and verified ✓ — ${verify.reason}`, kind: 'success' };
|
||||
if (verify.valid === false) return { text: `Discord: token exported, but FC's check failed — ${verify.reason}`, kind: 'error' };
|
||||
return { text: `Discord: token exported (not verified — ${verify.reason})`, kind: 'warning' };
|
||||
}
|
||||
@@ -56,9 +56,10 @@
|
||||
"*://*.patreon.com/*",
|
||||
"*://*.subscribestar.com/*",
|
||||
"*://*.subscribestar.adult/*",
|
||||
"*://*.hentai-foundry.com/*"
|
||||
"*://*.hentai-foundry.com/*",
|
||||
"*://*.discord.com/*"
|
||||
],
|
||||
"js": ["lib/platforms.js", "content/content-script.js"],
|
||||
"js": ["lib/platforms.js", "lib/chip.js", "content/content-script.js"],
|
||||
"css": ["content/content-script.css"],
|
||||
"run_at": "document_idle"
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
</section>
|
||||
|
||||
<script src="../lib/platforms.js"></script>
|
||||
<script src="../lib/popup-format.js"></script>
|
||||
<script src="popup.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -159,7 +159,11 @@ async function exportPlatformCookies(key, card) {
|
||||
try {
|
||||
const r = await browser.runtime.sendMessage({ type: 'EXPORT_COOKIES', platform: key });
|
||||
if (r.error) showError(r.error);
|
||||
else {
|
||||
else if (key === 'discord') {
|
||||
const m = tokenExportMessage(r.verify);
|
||||
showStatusMessage(m.text, m.kind);
|
||||
await loadPlatformStatus();
|
||||
} else {
|
||||
const n = r.cookieCount ?? null;
|
||||
const verifiedSuffix = r.verified ? ' (verified ✓)' : '';
|
||||
const msg = n !== null
|
||||
@@ -205,7 +209,10 @@ async function loadSources() {
|
||||
c.appendChild(mutedNote('No sources yet.'));
|
||||
return;
|
||||
}
|
||||
for (const src of r.sources) c.appendChild(createSourceRow(src));
|
||||
// Grouped by artist so a creator's Patreon and Discord sit together.
|
||||
const sorted = [...r.sources].sort((a, b) =>
|
||||
(a.artist_name || '').localeCompare(b.artist_name || '') || a.id - b.id);
|
||||
for (const src of sorted) c.appendChild(createSourceRow(src));
|
||||
}
|
||||
|
||||
function createSourceRow(src) {
|
||||
@@ -215,11 +222,16 @@ function createSourceRow(src) {
|
||||
info.className = 'info';
|
||||
const name = document.createElement('div');
|
||||
name.className = 'name';
|
||||
name.textContent = `${src.platform} · #${src.id}`;
|
||||
const platformName = PLATFORMS[src.platform]?.name || src.platform;
|
||||
name.textContent = `${src.artist_name || `Source #${src.id}`} · ${platformName}`;
|
||||
const state = sourceStatus(src);
|
||||
const st = document.createElement('div');
|
||||
st.className = `status ${state.kind}`;
|
||||
st.textContent = state.text;
|
||||
const url = document.createElement('div');
|
||||
url.className = 'url';
|
||||
url.textContent = src.url;
|
||||
info.appendChild(name); info.appendChild(url);
|
||||
info.appendChild(name); info.appendChild(st); info.appendChild(url);
|
||||
const play = document.createElement('button');
|
||||
play.className = 'play';
|
||||
play.textContent = '▶';
|
||||
@@ -229,7 +241,7 @@ function createSourceRow(src) {
|
||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_SOURCE', sourceId: src.id });
|
||||
play.disabled = false;
|
||||
if (r.error) showError(r.error);
|
||||
else showSuccess(`Triggered check for source #${src.id}`);
|
||||
else showSuccess(`Check queued for ${src.artist_name || `source #${src.id}`} (${platformName})`);
|
||||
});
|
||||
row.appendChild(info); row.appendChild(play);
|
||||
return row;
|
||||
|
||||
@@ -134,5 +134,42 @@
|
||||
{ "url": "https://www.hentai-foundry.com/pictures/popular", "why": "gallery listing, not a user" },
|
||||
{ "url": "https://www.hentai-foundry.com/", "why": "site root" }
|
||||
]
|
||||
},
|
||||
|
||||
"discord": {
|
||||
"match": [
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111/222222222222222222",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "a channel: the slug is server/channel -- a Discord URL names a place, not a creator, so the artist is chosen in the Add panel"
|
||||
},
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111",
|
||||
"slug": "111111111111111111",
|
||||
"why": "a whole server"
|
||||
},
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111/222222222222222222/333333333333333333",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "a message jump link still names its channel"
|
||||
},
|
||||
{
|
||||
"url": "https://ptb.discord.com/channels/111111111111111111/222222222222222222",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "the ptb and canary clients serve the same channels"
|
||||
},
|
||||
{
|
||||
"url": "https://discord.com/channels/111111111111111111/222222222222222222/",
|
||||
"slug": "111111111111111111/222222222222222222",
|
||||
"why": "trailing slash is tolerated"
|
||||
}
|
||||
],
|
||||
"no_match": [
|
||||
{ "url": "https://discord.com/channels/@me", "why": "the DM list is not a source" },
|
||||
{ "url": "https://discord.com/channels/@me/222222222222222222", "why": "a DM is not a source" },
|
||||
{ "url": "https://discord.com/app", "why": "the app shell, no server open" },
|
||||
{ "url": "https://discord.com/channels/111111111111111111/222222222222222222/threads/444444444444444444", "why": "thread links are left to the manual Add form" },
|
||||
{ "url": "https://discord.com/servers/111111111111111111", "why": "a server-discovery page, not a channel" }
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { loadLib } from './helpers/loadLib.js'
|
||||
|
||||
const { chipState, chipLabel, panelDefaults, addRequest, renameOffer } = loadLib('chip.js', [
|
||||
'chipState',
|
||||
'chipLabel',
|
||||
'panelDefaults',
|
||||
'addRequest',
|
||||
'renameOffer'
|
||||
])
|
||||
|
||||
const discord = (extra = {}) => ({
|
||||
platform: 'discord',
|
||||
slug: '111/222',
|
||||
discord: {
|
||||
server_id: '111',
|
||||
channel_id: '222',
|
||||
server_name: 'Studio',
|
||||
channel_name: 'drops',
|
||||
server_url: 'https://discord.com/channels/111',
|
||||
channel_url: 'https://discord.com/channels/111/222'
|
||||
},
|
||||
...extra
|
||||
})
|
||||
|
||||
describe('chip state and label', () => {
|
||||
it('keeps the one-click wording on creator platforms', () => {
|
||||
expect(chipLabel({ state: 'new', platform: 'patreon' }, 'Patreon')).toBe('+ Add to FabledCurator')
|
||||
expect(
|
||||
chipLabel({ state: 'artist_match', platform: 'patreon', artist: { name: 'Atole' } }, 'Patreon')
|
||||
).toBe('+ Add Patreon source to Atole')
|
||||
expect(chipLabel({ state: 'source_match', platform: 'patreon' }, 'Patreon')).toBe(
|
||||
'✓ In FabledCurator · Patreon'
|
||||
)
|
||||
})
|
||||
|
||||
it('offers the channel by name on Discord, whatever the suggestion', () => {
|
||||
expect(chipLabel(discord({ state: 'new' }), 'Discord')).toBe('+ Add #drops to FabledCurator')
|
||||
expect(chipLabel(discord({ state: 'artist_match', artist: { name: 'A' } }), 'Discord')).toBe(
|
||||
'+ Add #drops to FabledCurator'
|
||||
)
|
||||
})
|
||||
|
||||
it('says "this channel" when the token could not read the name', () => {
|
||||
const p = discord({ state: 'new' })
|
||||
p.discord.channel_name = null
|
||||
expect(chipLabel(p, 'Discord')).toBe('+ Add this channel to FabledCurator')
|
||||
})
|
||||
|
||||
it('says whose source a Discord channel already is, and when the server covers it', () => {
|
||||
const artist = { name: 'Tamada', slug: 'tamada' }
|
||||
expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: false }), 'Discord')).toBe(
|
||||
'✓ In FabledCurator · Tamada'
|
||||
)
|
||||
expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: true }), 'Discord')).toBe(
|
||||
'✓ Whole server in FabledCurator · Tamada'
|
||||
)
|
||||
})
|
||||
|
||||
it('falls back to the generic add when the probe failed', () => {
|
||||
expect(chipState({ error: 'x' })).toBe('new')
|
||||
expect(chipLabel({ error: 'x' }, '')).toBe('+ Add to FabledCurator')
|
||||
expect(chipState({ state: 'source_match' })).toBe('source-match')
|
||||
})
|
||||
})
|
||||
|
||||
describe('Add panel on Discord', () => {
|
||||
it('opens on the channel with the suggested artist preselected', () => {
|
||||
const d = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
expect(d.scope).toBe('channel')
|
||||
expect(d.artist).toEqual({ id: 7, name: 'Tamada' })
|
||||
expect(d.artistName).toBe('Tamada')
|
||||
})
|
||||
|
||||
it('proposes a new artist named after the server when nothing is suggested', () => {
|
||||
const d = panelDefaults(discord({ state: 'new' }))
|
||||
expect(d.artist).toBe(null)
|
||||
expect(d.artistName).toBe('Studio')
|
||||
})
|
||||
|
||||
it('sends a picked artist by id', () => {
|
||||
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
expect(addRequest(choice)).toEqual({
|
||||
url: 'https://discord.com/channels/111/222',
|
||||
artistId: 7
|
||||
})
|
||||
})
|
||||
|
||||
it('sends a typed name once the picked artist has been edited away', () => {
|
||||
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||
choice.artistName = 'Tamada Alt'
|
||||
expect(addRequest(choice)).toEqual({
|
||||
url: 'https://discord.com/channels/111/222',
|
||||
artistName: 'Tamada Alt'
|
||||
})
|
||||
})
|
||||
|
||||
it('adds the whole server when the operator picks it', () => {
|
||||
const choice = panelDefaults(discord({ state: 'new' }))
|
||||
choice.scope = 'server'
|
||||
expect(addRequest(choice).url).toBe('https://discord.com/channels/111')
|
||||
})
|
||||
|
||||
it('has nothing to send without an artist', () => {
|
||||
const choice = panelDefaults(discord({ state: 'new' }))
|
||||
choice.artistName = ' '
|
||||
expect(addRequest(choice)).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
const { squashName, exactArtistMatch, inlineCompletion } = loadLib('chip.js', [
|
||||
'squashName',
|
||||
'exactArtistMatch',
|
||||
'inlineCompletion'
|
||||
])
|
||||
|
||||
describe('artist matching for the Add panel', () => {
|
||||
const results = [
|
||||
{ id: 1, name: 'TamadaHeijun' },
|
||||
{ id: 2, name: 'Tamago' },
|
||||
{ id: 3, name: 'Sabu Art' }
|
||||
]
|
||||
|
||||
it('treats spacing, case and punctuation as the same name', () => {
|
||||
expect(squashName('Tamada Heijun')).toBe('tamadaheijun')
|
||||
expect(squashName('sabu_art!')).toBe('sabuart')
|
||||
expect(squashName('玉田 平順')).toBe('玉田平順')
|
||||
})
|
||||
|
||||
it('finds the result that is the query, spacing aside', () => {
|
||||
expect(exactArtistMatch('tamada heijun', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||
expect(exactArtistMatch('Tama', results)).toBe(null)
|
||||
expect(exactArtistMatch(' ', results)).toBe(null)
|
||||
})
|
||||
|
||||
it('autofills the first name that extends what was typed', () => {
|
||||
expect(inlineCompletion('tamad', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||
expect(inlineCompletion('Sab', results).name).toBe('Sabu Art')
|
||||
// Nothing to add once the name is complete, or when nothing extends it.
|
||||
expect(inlineCompletion('Sabu Art', results)).toBe(null)
|
||||
expect(inlineCompletion('heijun', results)).toBe(null)
|
||||
expect(inlineCompletion('', results)).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
describe('Add panel on Patreon and SubscribeStar', () => {
|
||||
const PAGE = 'https://www.patreon.com/cw/tamadaheijun'
|
||||
const patreon = (extra = {}) => ({ platform: 'patreon', slug: 'tamadaheijun', ...extra })
|
||||
|
||||
it('opens on the Patreon display name as a new artist', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
expect(c.scope).toBe('page')
|
||||
expect(c.artistName).toBe('Tamada Heijun')
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Tamada Heijun' })
|
||||
})
|
||||
|
||||
it('leaves an untouched URL handle to the server to resolve', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||
expect(c.artistName).toBe('tamadaheijun')
|
||||
expect(addRequest(c)).toEqual({ url: PAGE })
|
||||
c.artistName = 'Someone Else'
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Someone Else' })
|
||||
})
|
||||
|
||||
it('offers the Patreon name when joining an artist known by another name', () => {
|
||||
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
c.artist = { id: 4, name: 'tamada' }
|
||||
c.artistName = 'tamada'
|
||||
expect(renameOffer(c)).toEqual({ from: 'tamada', to: 'Tamada Heijun' })
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4, usePlatformName: true })
|
||||
c.adoptPlatformName = false
|
||||
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4 })
|
||||
})
|
||||
|
||||
it('offers no rename when the names agree or the name was not read', () => {
|
||||
const same = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||
same.artist = { id: 4, name: 'Tamada Heijun' }
|
||||
expect(renameOffer(same)).toBe(null)
|
||||
const unread = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||
unread.artist = { id: 4, name: 'tamada' }
|
||||
expect(renameOffer(unread)).toBe(null)
|
||||
})
|
||||
|
||||
it('never renames from SubscribeStar or Discord — Patreon is the canon', () => {
|
||||
const ss = panelDefaults(
|
||||
{ platform: 'subscribestar', slug: 'tamada', state: 'new', display_name: 'SS Tamada' },
|
||||
'https://subscribestar.adult/tamada'
|
||||
)
|
||||
ss.artist = { id: 4, name: 'Tamada Heijun' }
|
||||
ss.artistName = 'Tamada Heijun'
|
||||
expect(renameOffer(ss)).toBe(null)
|
||||
expect(addRequest(ss)).toEqual({ url: 'https://subscribestar.adult/tamada', artistId: 4 })
|
||||
})
|
||||
|
||||
it('preselects the artist the URL already names', () => {
|
||||
const c = panelDefaults(patreon({ state: 'artist_match', artist: { id: 9, name: 'Tamada' } }), PAGE)
|
||||
expect(c.artist).toEqual({ id: 9, name: 'Tamada' })
|
||||
expect(c.artistName).toBe('Tamada')
|
||||
})
|
||||
})
|
||||
@@ -7,10 +7,14 @@ import { loadLib } from './helpers/loadLib.js'
|
||||
const EXT_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..')
|
||||
const manifest = JSON.parse(readFileSync(path.join(EXT_DIR, 'manifest.json'), 'utf8'))
|
||||
|
||||
const { getPlatformFromUrl, isArtistPage, PLATFORMS, PLATFORM_ARTIST_PATTERNS } = loadLib(
|
||||
'platforms.js',
|
||||
['getPlatformFromUrl', 'isArtistPage', 'PLATFORMS', 'PLATFORM_ARTIST_PATTERNS']
|
||||
)
|
||||
const { getPlatformFromUrl, isArtistPage, parseDiscordUrl, PLATFORMS, PLATFORM_ARTIST_PATTERNS } =
|
||||
loadLib('platforms.js', [
|
||||
'getPlatformFromUrl',
|
||||
'isArtistPage',
|
||||
'parseDiscordUrl',
|
||||
'PLATFORMS',
|
||||
'PLATFORM_ARTIST_PATTERNS'
|
||||
])
|
||||
|
||||
describe('getPlatformFromUrl', () => {
|
||||
it('identifies each platform from a domain URL', () => {
|
||||
@@ -88,8 +92,11 @@ describe('isArtistPage', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('returns false for a platform with no artist pattern (discord)', () => {
|
||||
it('matches Discord server and channel pages, not DMs (milestone 429)', () => {
|
||||
expect(isArtistPage('https://discord.com/channels/111/222', 'discord')).toBe(true)
|
||||
expect(isArtistPage('https://discord.com/channels/111', 'discord')).toBe(true)
|
||||
expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false)
|
||||
expect(isArtistPage('https://discord.com/channels/@me/222', 'discord')).toBe(false)
|
||||
})
|
||||
|
||||
it('returns false for an unknown platform key', () => {
|
||||
@@ -100,8 +107,8 @@ describe('isArtistPage', () => {
|
||||
describe('platform table integrity', () => {
|
||||
it('gives every artist pattern a corresponding platform entry', () => {
|
||||
// A pattern keyed to a platform that no longer exists is dead code that
|
||||
// silently never fires; the reverse (a platform with no pattern) is the
|
||||
// legitimate discord case, so only this direction is an error.
|
||||
// silently never fires; the reverse (a platform with no pattern) would be
|
||||
// a platform the button never offers, a product choice rather than an error.
|
||||
for (const key of Object.keys(PLATFORM_ARTIST_PATTERNS)) {
|
||||
expect(Object.keys(PLATFORMS)).toContain(key)
|
||||
}
|
||||
@@ -125,7 +132,8 @@ describe('platform table integrity', () => {
|
||||
const samples = {
|
||||
patreon: 'https://www.patreon.com/cw/Atole',
|
||||
subscribestar: 'https://subscribestar.adult/someone',
|
||||
hentaifoundry: 'https://www.hentai-foundry.com/user/someone'
|
||||
hentaifoundry: 'https://www.hentai-foundry.com/user/someone',
|
||||
discord: 'https://ptb.discord.com/channels/111/222'
|
||||
}
|
||||
for (const [key, url] of Object.entries(samples)) {
|
||||
expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true)
|
||||
@@ -152,7 +160,7 @@ describe('manifest.json agrees with the platform table', () => {
|
||||
)
|
||||
expect(owner, `no platform claims content-script match "${m}"`).toBeTruthy()
|
||||
// The content script exists to draw the Add-as-source button, so a
|
||||
// platform with no artist pattern (discord) has no business here.
|
||||
// platform with no artist pattern has no business here.
|
||||
expect(
|
||||
PLATFORM_ARTIST_PATTERNS[owner[0]],
|
||||
`"${m}" injects for ${owner[0]}, which has no artist pattern`
|
||||
@@ -232,9 +240,8 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => {
|
||||
|
||||
it('has samples for every platform that has an artist pattern', () => {
|
||||
// The guard's own coverage check: without it, deleting a platform's
|
||||
// samples would make this block pass by testing less. Discord is
|
||||
// deliberately in neither — it is channel-based, with no creator page to
|
||||
// put a button on, so it has no artist pattern on either side.
|
||||
// samples would make this block pass by testing less. Discord joined at
|
||||
// milestone 429 — its slug is server/channel and the artist is chosen.
|
||||
expect(Object.keys(samples).sort()).toEqual(Object.keys(PLATFORM_ARTIST_PATTERNS).sort())
|
||||
})
|
||||
|
||||
@@ -247,3 +254,26 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => {
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('parseDiscordUrl', () => {
|
||||
it('reads the server and channel ids the Add panel offers', () => {
|
||||
expect(parseDiscordUrl('https://discord.com/channels/111/222')).toEqual({
|
||||
serverId: '111',
|
||||
channelId: '222'
|
||||
})
|
||||
expect(parseDiscordUrl('https://discord.com/channels/111/222/333')).toEqual({
|
||||
serverId: '111',
|
||||
channelId: '222'
|
||||
})
|
||||
expect(parseDiscordUrl('https://discord.com/channels/111')).toEqual({
|
||||
serverId: '111',
|
||||
channelId: null
|
||||
})
|
||||
})
|
||||
|
||||
it('returns null for anything the artist pattern rejects', () => {
|
||||
expect(parseDiscordUrl('https://discord.com/channels/@me/222')).toBe(null)
|
||||
expect(parseDiscordUrl('https://discord.com/app')).toBe(null)
|
||||
expect(parseDiscordUrl('')).toBe(null)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import { loadLib } from './helpers/loadLib.js'
|
||||
|
||||
const { sourceStatus, relativeTime, tokenExportMessage } = loadLib('popup-format.js', [
|
||||
'sourceStatus',
|
||||
'relativeTime',
|
||||
'tokenExportMessage'
|
||||
])
|
||||
|
||||
const NOW = Date.parse('2026-09-25T12:00:00Z')
|
||||
const src = (extra = {}) => ({ enabled: true, last_error: null, backfill_state: null, ...extra })
|
||||
|
||||
describe('sourceStatus', () => {
|
||||
it('puts an error first, trimmed to its first line', () => {
|
||||
const s = sourceStatus(src({ last_error: 'Discord rejected the token\nstack…', backfill_state: 'running' }), NOW)
|
||||
expect(s).toEqual({ text: 'Error — Discord rejected the token', kind: 'error' })
|
||||
})
|
||||
|
||||
it('shows a running backfill and its progress', () => {
|
||||
expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 0 }), NOW).text).toBe('Backfill queued')
|
||||
expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 3 }), NOW).text).toBe(
|
||||
'Backfilling — 3 chunks done'
|
||||
)
|
||||
})
|
||||
|
||||
it('says when a source was last checked, or that it never was', () => {
|
||||
expect(sourceStatus(src({ last_checked_at: '2026-09-25T11:55:00Z' }), NOW).text).toBe('Checked 5m ago')
|
||||
expect(sourceStatus(src({ last_checked_at: null }), NOW).text).toBe('Not checked yet')
|
||||
})
|
||||
|
||||
it('shows a disabled source as disabled, whatever else it carries', () => {
|
||||
expect(sourceStatus(src({ enabled: false, last_error: 'x' }), NOW).text).toBe('Disabled')
|
||||
})
|
||||
})
|
||||
|
||||
describe('relativeTime', () => {
|
||||
it('uses the web UI formatRelative buckets', () => {
|
||||
expect(relativeTime('2026-09-25T11:59:18Z', NOW)).toBe('42s ago')
|
||||
expect(relativeTime('2026-09-25T09:00:00Z', NOW)).toBe('3h ago')
|
||||
expect(relativeTime('2026-09-23T12:00:00Z', NOW)).toBe('2d ago')
|
||||
expect(relativeTime('garbage', NOW)).toBe('Never')
|
||||
})
|
||||
})
|
||||
|
||||
describe('tokenExportMessage', () => {
|
||||
it('distinguishes verified, rejected and untested tokens', () => {
|
||||
expect(tokenExportMessage({ valid: true, reason: 'Token valid (me)' }).kind).toBe('success')
|
||||
expect(tokenExportMessage({ valid: false, reason: 'Discord rejected the token' }).kind).toBe('error')
|
||||
const untested = tokenExportMessage({ valid: null, reason: 'No enabled source' })
|
||||
expect(untested.kind).toBe('warning')
|
||||
expect(untested.text).toContain('No enabled source')
|
||||
})
|
||||
})
|
||||
@@ -153,6 +153,14 @@ const health = computed(() => {
|
||||
label: (worst?.detail || 'A part has stopped') + suffix,
|
||||
}
|
||||
}
|
||||
// Running but slow — a GPU agent that fell back to the CPU (#4410). Worth
|
||||
// the amber dot: nothing else anywhere says so.
|
||||
if (overall === 'degraded') {
|
||||
return {
|
||||
icon: 'mdi-speedometer-slow', color: 'warning',
|
||||
label: (worst?.detail || 'A part is running degraded') + suffix,
|
||||
}
|
||||
}
|
||||
if (overall === 'stale') {
|
||||
return {
|
||||
icon: 'mdi-alert', color: 'warning',
|
||||
|
||||
@@ -103,9 +103,9 @@ const stats = computed(() => {
|
||||
padding: 0.5rem 1rem;
|
||||
background: linear-gradient(
|
||||
to bottom,
|
||||
rgba(20, 23, 26, 0.92) 0%,
|
||||
rgba(20, 23, 26, 0.65) 60%,
|
||||
rgba(20, 23, 26, 0) 100%
|
||||
rgba(var(--fc-chrome-rgb), 0.92) 0%,
|
||||
rgba(var(--fc-chrome-rgb), 0.65) 60%,
|
||||
rgba(var(--fc-chrome-rgb), 0) 100%
|
||||
);
|
||||
backdrop-filter: blur(2px);
|
||||
-webkit-backdrop-filter: blur(2px);
|
||||
|
||||
@@ -329,7 +329,7 @@ function pushFilter(mutate) {
|
||||
opaque than the bar/nav so the controls stay legible. */
|
||||
.fc-filterbar-wrap :deep(.v-field),
|
||||
.fc-filterbar-wrap :deep(.v-btn-group) {
|
||||
background-color: rgba(20, 23, 26, 0.72);
|
||||
background-color: rgba(var(--v-theme-background), 0.72);
|
||||
}
|
||||
/* Media toggle (All / Images / Videos) as ONE cohesive segmented control.
|
||||
FC's global VBtn { rounded: 'pill' } default made Vuetify 4 pill-round each
|
||||
|
||||
@@ -213,7 +213,7 @@ function nextFrame() {
|
||||
--fc-side-w: 320px;
|
||||
/* Obsidian haze (#14171A = 20,23,26) — same palette as TopNav,
|
||||
mid-opacity + blur so the page behind shows through faintly. */
|
||||
background: rgba(20, 23, 26, 0.65);
|
||||
background: rgba(var(--v-theme-background), 0.65);
|
||||
backdrop-filter: blur(8px);
|
||||
-webkit-backdrop-filter: blur(8px);
|
||||
outline: none;
|
||||
@@ -221,7 +221,7 @@ function nextFrame() {
|
||||
}
|
||||
.fc-viewer__close, .fc-viewer__nav {
|
||||
position: absolute; top: 50%;
|
||||
background: rgba(20, 23, 26, 0.7);
|
||||
background: rgba(var(--v-theme-background), 0.7);
|
||||
color: rgb(var(--v-theme-parchment, 232 228 216));
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
border-radius: 50%;
|
||||
@@ -359,7 +359,7 @@ function nextFrame() {
|
||||
z-index: 1;
|
||||
/* Opaque obsidian so the scrolling panel never bleeds through the
|
||||
haze behind the pinned image. */
|
||||
background: rgb(20, 23, 26);
|
||||
background: rgb(var(--v-theme-background));
|
||||
}
|
||||
.fc-viewer__side {
|
||||
width: 100%;
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
class="fc-post-card__synthetic" :title="synthesisTitle"
|
||||
>
|
||||
<v-icon icon="mdi-auto-fix" size="x-small" start />
|
||||
grouped by FabledCurator
|
||||
{{ synthesisChip }}
|
||||
</v-chip>
|
||||
<RouterLink
|
||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||
@@ -270,11 +270,18 @@ const plainTitle = computed(() => toPlainText(props.post.post_title))
|
||||
// post dict by hand, must degrade to "not synthetic" rather than throw.
|
||||
const synthesized = computed(() => Boolean(props.post.synthesized_by))
|
||||
const messageCount = computed(() => props.post.synthesis?.message_count ?? 0)
|
||||
// A drop of one message stays a synthetic post (teaser matching only looks at
|
||||
// drops), but there is nothing grouped in it — #4390: "Grouped from 1 Discord
|
||||
// message" described a wrapper, not a grouping. Say what it is instead.
|
||||
const synthesisTitle = computed(() => {
|
||||
const n = messageCount.value
|
||||
if (!n) return 'Grouped from Discord'
|
||||
return `Grouped from ${n} Discord message${n === 1 ? '' : 's'}`
|
||||
if (n === 1) return 'Discord message'
|
||||
return `Grouped from ${n} Discord messages`
|
||||
})
|
||||
const synthesisChip = computed(() =>
|
||||
messageCount.value === 1 ? 'from Discord' : 'grouped by FabledCurator'
|
||||
)
|
||||
|
||||
const hero = computed(() => images.value[0])
|
||||
|
||||
|
||||
@@ -368,6 +368,7 @@ function step(lane, delta) {
|
||||
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
|
||||
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
|
||||
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
|
||||
.fc-sys__dot--degraded { background: rgb(var(--v-theme-warning)); }
|
||||
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
|
||||
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
|
||||
|
||||
|
||||
@@ -45,8 +45,9 @@
|
||||
>
|
||||
<v-list-item-title>Recapture post text & links</v-list-item-title>
|
||||
<v-list-item-subtitle>
|
||||
Re-grab every post's body + external links and localize inline images
|
||||
already on disk — without re-downloading media
|
||||
Re-grab every post's body + external links, localize inline images
|
||||
already on disk, and import any downloaded file that never reached the
|
||||
library — without re-downloading media
|
||||
</v-list-item-subtitle>
|
||||
</v-list-item>
|
||||
<v-divider v-if="isNative && !running" />
|
||||
@@ -75,10 +76,10 @@ const running = computed(() => props.source.backfill_state === 'running')
|
||||
const recovering = computed(() => !!props.source.backfill_bypass_seen)
|
||||
const recapturing = computed(() => !!props.source.backfill_recapture)
|
||||
// Recover / recapture are native-ingester features (ledger-bypass re-walk and
|
||||
// post-text re-grab), available to every native platform — not just Patreon.
|
||||
// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS.
|
||||
const NATIVE_PLATFORMS = ['patreon', 'subscribestar']
|
||||
const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform))
|
||||
// post-text re-grab), available on every native platform. The backend says
|
||||
// which those are (`native_ingester`); a copied list here went stale when
|
||||
// Discord moved over.
|
||||
const isNative = computed(() => !!props.source.native_ingester)
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
|
||||
@@ -679,7 +679,7 @@ async function onRecover(source) {
|
||||
}
|
||||
}
|
||||
|
||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||
// #830: arm a recapture walk (native platforms) — re-grab every post's body +
|
||||
// external links and localize on-disk inline images, without re-downloading
|
||||
// media. Reuses the backfill lifecycle/badge; stop via the same Stop control.
|
||||
async function onRecapture(source) {
|
||||
|
||||
@@ -10,7 +10,22 @@ export class ApiError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
async function request(method, url, { body, params, signal } = {}) {
|
||||
const OPTIONS = new Set(['body', 'params', 'signal'])
|
||||
|
||||
async function request(method, url, opts = {}) {
|
||||
// Refuse an option this wrapper does not know. `api.post(url, { id: 1 })`
|
||||
// reads naturally and sends NO body — the payload lands in the options bag
|
||||
// and is dropped — so the server answers `invalid_body` and the caller
|
||||
// looks broken server-side. It shipped exactly that way in the Subscriptions
|
||||
// "Add subscription" button (2026-09-24). Failing here names the mistake.
|
||||
const unknown = Object.keys(opts).filter((k) => !OPTIONS.has(k))
|
||||
if (unknown.length) {
|
||||
throw new TypeError(
|
||||
`useApi ${method} ${url}: unknown option(s) ${unknown.join(', ')} — ` +
|
||||
'a request payload goes under `body`, a query under `params`'
|
||||
)
|
||||
}
|
||||
const { body, params, signal } = opts
|
||||
let fullUrl = url
|
||||
if (params) {
|
||||
const search = new URLSearchParams()
|
||||
|
||||
+17
-1
@@ -94,9 +94,25 @@ const routes = [
|
||||
const history =
|
||||
typeof window !== 'undefined' ? createWebHistory() : createMemoryHistory()
|
||||
|
||||
// Where the window lands after a navigation. Without this the router keeps the
|
||||
// window's scroll offset, so a view entered from halfway down another one opened
|
||||
// halfway down itself, its heading and tab strip under the nav (operator-flagged
|
||||
// 2026-09-24, Settings entered from a scrolled feed).
|
||||
// - back/forward returns to where that entry was left;
|
||||
// - the SAME view with a new query (a gallery filter, a Browse or
|
||||
// Subscriptions tab) keeps its place — that is not a new page;
|
||||
// - a new view opens at its top.
|
||||
export function scrollBehavior(to, from, savedPosition) {
|
||||
if (savedPosition) return savedPosition
|
||||
if (from.matched.length && to.path === from.path) return false
|
||||
if (to.hash) return { el: to.hash }
|
||||
return { top: 0 }
|
||||
}
|
||||
|
||||
const router = createRouter({
|
||||
history,
|
||||
routes
|
||||
routes,
|
||||
scrollBehavior
|
||||
})
|
||||
|
||||
const DEFAULT_TITLE = 'FabledCurator'
|
||||
|
||||
@@ -28,7 +28,7 @@ export const useMembershipReconcileStore = defineStore('membershipReconcile', ()
|
||||
async function adopt (membershipId) {
|
||||
try {
|
||||
const res = await api.post('/api/sources/reconciliation/adopt', {
|
||||
membership_id: membershipId
|
||||
body: { membership_id: membershipId }
|
||||
})
|
||||
toast({
|
||||
text: res.already_tracked
|
||||
|
||||
@@ -140,7 +140,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
||||
_patchSource(body)
|
||||
return body
|
||||
}
|
||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
||||
// #830: arm a recapture walk (native platforms) — re-grab every post's body +
|
||||
// external links and localize on-disk inline images, WITHOUT re-downloading
|
||||
// media. Shares the backfill lifecycle/badge; stop via stopBackfill.
|
||||
async function recaptureSource(id, artistIdHint = null) {
|
||||
|
||||
@@ -24,7 +24,8 @@ export function queueKey(queues) {
|
||||
}
|
||||
|
||||
// Worst first. A stopped datastore is why someone opened this tab.
|
||||
export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 }
|
||||
// `degraded`: checking in, but slow — a GPU agent on the CPU (#4410).
|
||||
export const SEVERITY = { down: 4, stale: 3, degraded: 2, unknown: 1, ok: 0 }
|
||||
|
||||
export function kindLabel(kind) {
|
||||
if (kind === 'celery') return 'worker lane'
|
||||
|
||||
@@ -369,7 +369,7 @@ onUnmounted(() => {
|
||||
|
||||
/* Center viewer. */
|
||||
.fc-ex__viewer {
|
||||
background: rgb(20, 23, 26);
|
||||
background: rgb(var(--v-theme-background));
|
||||
display: flex; flex-direction: column; min-width: 0; min-height: 0;
|
||||
}
|
||||
.fc-ex__canvas { flex: 1 1 auto; display: flex; min-height: 0; min-width: 0; }
|
||||
|
||||
@@ -276,7 +276,7 @@ onUnmounted(() => {
|
||||
.fc-reader__thumb img { width: 100%; height: auto; display: block; }
|
||||
.fc-reader__thumbnum {
|
||||
position: absolute; bottom: 4px; right: 4px;
|
||||
background: rgba(20, 23, 26, 0.8); color: rgb(var(--v-theme-on-surface));
|
||||
background: rgba(var(--v-theme-background), 0.8); color: rgb(var(--v-theme-on-surface));
|
||||
padding: 1px 5px; border-radius: 4px; font-size: 0.7rem;
|
||||
}
|
||||
.fc-reader__content {
|
||||
@@ -302,7 +302,7 @@ onUnmounted(() => {
|
||||
.fc-reader__indicator {
|
||||
position: fixed; bottom: 1.5rem; right: 1.5rem;
|
||||
padding: 4px 10px; border-radius: 999px;
|
||||
background: rgba(20, 23, 26, 0.6);
|
||||
background: rgba(var(--v-theme-background), 0.6);
|
||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||
font-size: 0.8rem; color: rgb(var(--v-theme-on-surface-variant));
|
||||
z-index: 100; pointer-events: none;
|
||||
|
||||
@@ -338,18 +338,18 @@ onMounted(() => {
|
||||
position: absolute; top: 6px; left: 6px;
|
||||
display: inline-flex; align-items: center; gap: 2px;
|
||||
padding: 1px 6px; border-radius: 999px; font-size: 11px;
|
||||
background: rgba(20, 23, 26, 0.75);
|
||||
background: rgba(var(--v-theme-background), 0.75);
|
||||
color: rgb(var(--v-theme-warning, var(--v-theme-accent)));
|
||||
}
|
||||
/* Kebab sits top-right of the cover, opposite the gap badge. Tinted backing so
|
||||
it stays legible over any cover image. */
|
||||
.fc-sbcard__kebab {
|
||||
position: absolute; top: 4px; right: 4px;
|
||||
background: rgba(20, 23, 26, 0.6) !important;
|
||||
background: rgba(var(--v-theme-background), 0.6) !important;
|
||||
border-radius: 50%;
|
||||
color: rgb(var(--v-theme-on-surface));
|
||||
}
|
||||
.fc-sbcard__kebab:hover { background: rgba(20, 23, 26, 0.85) !important; }
|
||||
.fc-sbcard__kebab:hover { background: rgba(var(--v-theme-background), 0.85) !important; }
|
||||
.fc-sbcard__body { padding: 8px 10px; }
|
||||
.fc-sbcard__name {
|
||||
font-family: 'Fraunces', Georgia, serif; font-size: 15px; font-weight: 600;
|
||||
|
||||
@@ -1,23 +1,12 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useAdminStore } from '../src/stores/admin.js'
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
// Covers the two helpers the admin store actions route through (DRY Finding C,
|
||||
// #753): _dryRunPost (URL + dry_run body, sourceId→source_id) and _guard
|
||||
// (lastError capture + rethrow). The store had no frontend test before.
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function lastCallBody(calls) {
|
||||
return JSON.parse(calls.at(-1).init.body)
|
||||
}
|
||||
|
||||
@@ -149,13 +149,16 @@ describe('PostCard', () => {
|
||||
expect(w.find('.fc-post-card__assoc').exists()).toBe(false)
|
||||
})
|
||||
|
||||
it('singularises a one-message drop', () => {
|
||||
it('does not call a one-message drop a grouping (#4390)', () => {
|
||||
const w = mountComponent(PostCard, {
|
||||
props: { post: { ...SYNTH, synthesis: { message_count: 1 } } },
|
||||
pinia: freshPinia(),
|
||||
})
|
||||
expect(w.text()).toContain('Grouped from 1 Discord message')
|
||||
expect(w.text()).not.toContain('1 Discord messages')
|
||||
expect(w.text()).toContain('Discord message')
|
||||
expect(w.text()).not.toContain('Grouped from')
|
||||
// The marker stays — FC still wrote this post — but names the source.
|
||||
expect(w.find('.fc-post-card__synthetic').text()).toContain('from Discord')
|
||||
expect(w.text()).not.toContain('grouped by FabledCurator')
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { useApi } from '../../src/composables/useApi.js'
|
||||
|
||||
// 2026-09-24: `api.post(url, { membership_id })` shipped in the Subscriptions
|
||||
// "Add subscription" button. It reads naturally, sends NO body, and the
|
||||
// server's `invalid_body` made the bug look like the backend's. The wrapper now
|
||||
// refuses an option it does not know, so the mistake names itself.
|
||||
describe('useApi', () => {
|
||||
afterEach(() => { vi.unstubAllGlobals() })
|
||||
|
||||
function stubFetch () {
|
||||
const fetch = vi.fn().mockResolvedValue({
|
||||
ok: true, status: 200, statusText: 'OK', text: () => Promise.resolve('{}'),
|
||||
})
|
||||
vi.stubGlobal('fetch', fetch)
|
||||
return fetch
|
||||
}
|
||||
|
||||
it('refuses a payload passed as options instead of under body', async () => {
|
||||
const fetch = stubFetch()
|
||||
await expect(useApi().post('/api/x', { membership_id: 1 }))
|
||||
.rejects.toThrow(/membership_id.*body/)
|
||||
expect(fetch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('sends a body passed under body', async () => {
|
||||
const fetch = stubFetch()
|
||||
await useApi().post('/api/x', { body: { membership_id: 1 } })
|
||||
expect(JSON.parse(fetch.mock.calls[0][1].body)).toEqual({ membership_id: 1 })
|
||||
})
|
||||
|
||||
it('still accepts no options at all', async () => {
|
||||
stubFetch()
|
||||
await expect(useApi().get('/api/x')).resolves.toEqual({})
|
||||
})
|
||||
})
|
||||
@@ -1,17 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useCredentialsStore } from '../src/stores/credentials.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status, statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('credentials store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useDbMaintenanceStore } from '../src/stores/dbMaintenance.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('dbMaintenance store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,17 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useDownloadsStore } from '../src/stores/downloads.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status, statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('downloads store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { cloneFilter, filterToQuery, useGalleryStore } from '../src/stores/gallery.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
const EMPTY = { images: [], date_groups: [], next_cursor: null }
|
||||
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useGallerySelectionStore } from '../src/stores/gallerySelection.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body))
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('gallerySelection store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
// The obsidian surface written out as a number instead of read from its token.
|
||||
//
|
||||
// fabled-tokens.js is the palette's single copy ("do not hand-edit hexes;
|
||||
// re-mirror here"). Thirteen hand-typed `rgba(20, 23, 26, …)` across six
|
||||
// components meant a changed obsidian would miss most of the UI (#3108).
|
||||
// Components read `--v-theme-background` (Vuetify maps background → obsidian)
|
||||
// or, for nav-style chrome, `--fc-chrome-rgb`. Comments may still name it.
|
||||
|
||||
import { readdirSync, readFileSync, statSync } from 'node:fs'
|
||||
import { join, relative } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
const SRC = fileURLToPath(new URL('../src', import.meta.url))
|
||||
const LITERAL = /rgba?\(\s*20\s*,\s*23\s*,\s*26\b/
|
||||
|
||||
function vueFiles (dir) {
|
||||
return readdirSync(dir).flatMap((name) => {
|
||||
const path = join(dir, name)
|
||||
if (statSync(path).isDirectory()) return vueFiles(path)
|
||||
return name.endsWith('.vue') ? [path] : []
|
||||
})
|
||||
}
|
||||
|
||||
function stripComments (text) {
|
||||
return text.replace(/\/\*[\s\S]*?\*\//g, '').replace(/<!--[\s\S]*?-->/g, '')
|
||||
}
|
||||
|
||||
describe('palette literals', () => {
|
||||
it('no component hand-types the obsidian surface', () => {
|
||||
const offenders = vueFiles(SRC).flatMap((path) =>
|
||||
stripComments(readFileSync(path, 'utf8'))
|
||||
.split('\n')
|
||||
.filter((line) => LITERAL.test(line))
|
||||
.map((line) => `${relative(SRC, path)}: ${line.trim()}`)
|
||||
)
|
||||
expect(offenders).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -1,17 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { usePlatformsStore } from '../src/stores/platforms.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status, statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('platforms store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useProvenanceStore } from '../src/stores/provenance.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body))
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('provenance store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect } from 'vitest'
|
||||
import router, { FRONT_DOOR } from '../src/router.js'
|
||||
import router, { FRONT_DOOR, scrollBehavior } from '../src/router.js'
|
||||
|
||||
describe('router', () => {
|
||||
it('FRONT_DOOR is the post feed', () => {
|
||||
@@ -74,4 +74,26 @@ describe('router', () => {
|
||||
expect(r.name).toBe('series-read')
|
||||
expect(r.meta.immersive).toBe(true)
|
||||
})
|
||||
|
||||
describe('scrollBehavior', () => {
|
||||
const at = (path) => ({ path, hash: '', matched: [{}] })
|
||||
|
||||
it('opens a new view at its top, not at the last view\'s offset', () => {
|
||||
expect(scrollBehavior(at('/settings'), at('/latest'), null)).toEqual({ top: 0 })
|
||||
})
|
||||
|
||||
it('back and forward return to where the entry was left', () => {
|
||||
const saved = { left: 0, top: 1234 }
|
||||
expect(scrollBehavior(at('/latest'), at('/settings'), saved)).toBe(saved)
|
||||
})
|
||||
|
||||
it('a new query on the same view keeps its place', () => {
|
||||
expect(scrollBehavior(at('/gallery'), at('/gallery'), null)).toBe(false)
|
||||
})
|
||||
|
||||
it('the first navigation of a page load opens at the top', () => {
|
||||
const initial = { path: '/', hash: '', matched: [] }
|
||||
expect(scrollBehavior(at('/latest'), initial, null)).toEqual({ top: 0 })
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,17 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useSeriesManageStore, moveItem } from '../src/stores/seriesManage.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status, statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body))
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
// FC-6.x: a flat page run + cosmetic chapter dividers.
|
||||
const SERIES_BODY = {
|
||||
|
||||
@@ -6,17 +6,7 @@ import {
|
||||
progressPct,
|
||||
clampPage
|
||||
} from '../src/stores/seriesReader.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status, statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body))
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
const M = [
|
||||
{ page_number: 1, top: 0, height: 100 },
|
||||
|
||||
@@ -1,17 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useSourcesStore } from '../src/stores/sources.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status, statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('sources store', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,21 +1,10 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useSuggestionsStore } from '../src/stores/suggestions.js'
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
vi.mock('../src/utils/toast.js', () => ({ toast: vi.fn() }))
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Every suggestion is a canonical DB tag now (tagging-v2): a real id, flagged
|
||||
// above/below its head's suggest threshold. No raw / creates-new / alias cases.
|
||||
const sugg = (over = {}) => ({
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
// Replace globalThis.fetch with a stub answering from `handler(url, init)`,
|
||||
// which returns { status, body }. Fifteen specs carried their own copy of this
|
||||
// (#3109); a response shape change now has one place to go.
|
||||
import { vi } from 'vitest'
|
||||
|
||||
export function stubFetch (handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,18 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useTagDirectoryStore } from '../src/stores/tagDirectory.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body))
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('tagDirectory store: rename / merge', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,18 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { useTagStore } from '../src/stores/tags.js'
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
describe('tags store: setFandom', () => {
|
||||
beforeEach(() => setActivePinia(createPinia()))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
||||
import { setActivePinia, createPinia } from 'pinia'
|
||||
import { laneStuckFor, useSystemActivityStore } from '../src/stores/systemActivity.js'
|
||||
import { stubFetch } from './support/stubFetch.js'
|
||||
|
||||
// Milestone 422 step 4. Covers the store half of the worker-lane dial — the
|
||||
// part that decides what the card can tell the operator.
|
||||
@@ -18,18 +19,6 @@ import { laneStuckFor, useSystemActivityStore } from '../src/stores/systemActivi
|
||||
// as one. A control that silently does nothing is worse than one that
|
||||
// refuses out loud.
|
||||
|
||||
function stubFetch(handler) {
|
||||
globalThis.fetch = vi.fn(async (url, init) => {
|
||||
const { status, body } = handler(url, init)
|
||||
return {
|
||||
ok: status >= 200 && status < 300,
|
||||
status,
|
||||
statusText: String(status),
|
||||
text: async () => (body == null ? '' : JSON.stringify(body)),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
const LANES_BODY = {
|
||||
lanes: [
|
||||
{
|
||||
|
||||
+11
-11
@@ -8,21 +8,21 @@
|
||||
# so Dockerfile uses the +cpu wheels from
|
||||
# https://download.pytorch.org/whl/cpu instead.
|
||||
#
|
||||
# IMPORTANT: torchvision 0.27 declares requires_python "!=3.14.1,>=3.10" —
|
||||
# Python 3.14.1 specifically is excluded due to a known incompatibility.
|
||||
# The python-ci runner pulls python:3.14-bookworm (latest patch); if that
|
||||
# resolves to 3.14.1 the install will fail. Pin a specific Python patch in
|
||||
# the runner image (CI-Runner/CI-python/Dockerfile) if this becomes a
|
||||
# blocker. 3.14.0 and 3.14.2+ are fine.
|
||||
# torchvision declares requires_python "!=3.14.1" (0.27 through 0.29). The
|
||||
# image's python:3.14-slim is past that patch, so it only bites a build pinned
|
||||
# to exactly 3.14.1.
|
||||
#
|
||||
# No caps below: rule 154 wants a named breakage for one, and none of the
|
||||
# `<N` caps these lines used to carry had one. opencv-python-headless and
|
||||
# onnxruntime were dropped (#1451): listed since the 2026-05-14 scaffold,
|
||||
# imported by nothing in backend/ — ONNX inference lives in the GPU agent.
|
||||
|
||||
transformers>=5.8,<6.0
|
||||
onnxruntime>=1.26,<2.0
|
||||
huggingface-hub>=1.14,<2.0
|
||||
opencv-python-headless>=4.13,<5.0
|
||||
transformers>=5.8
|
||||
huggingface-hub>=1.14
|
||||
|
||||
# scikit-learn powers the tag-eval (#1130) head-vs-centroid comparison: logistic
|
||||
# regression + cross-validated precision/recall/AP. Battle-tested metrics matter
|
||||
# because that eval's whole purpose is producing trustworthy numbers. numpy is
|
||||
# left to resolve transitively (torch/transformers/sklearn all pull it) to avoid
|
||||
# pinning against their constraints.
|
||||
scikit-learn>=1.7,<2.0
|
||||
scikit-learn>=1.7
|
||||
|
||||
+3
-1
@@ -11,7 +11,9 @@ pgvector>=0.5,<0.6
|
||||
|
||||
# Task queue
|
||||
celery>=5.6,<5.7
|
||||
redis>=7.4,<8.0
|
||||
# Uncapped (rule 154). 8.0 only changed type hints. kombu's `redis` extra
|
||||
# still says <6.5, but celery is installed without that extra, so it never applies.
|
||||
redis>=8.1
|
||||
|
||||
# Crypto for credential storage (lands in FC-3, but pinned now for stability)
|
||||
cryptography>=49,<50
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Throwaway rows for tests: one copy of the placeholder columns (#3109).
|
||||
|
||||
`ImageRecord` needs a unique path and sha256 plus six NOT NULL columns no test
|
||||
cares about. Fifteen modules re-typed the same builder, so a new NOT NULL column
|
||||
meant fifteen separate patches. The tests that need these rows import them from
|
||||
here, usually under their old local name (`make_image as _img`), so their call
|
||||
sites did not change.
|
||||
"""
|
||||
|
||||
from backend.app.models import ImageRecord, Tag, TagKind
|
||||
|
||||
|
||||
def image_row(sha: str, emb=None, **overrides) -> ImageRecord:
|
||||
"""An unsaved ImageRecord keyed by `sha`; `emb` is its SigLIP embedding."""
|
||||
fields = {
|
||||
"path": f"/images/{sha}.jpg", "sha256": sha, "size_bytes": 1,
|
||||
"mime": "image/jpeg", "width": 1, "height": 1,
|
||||
"origin": "imported_filesystem", "integrity_status": "unknown",
|
||||
}
|
||||
if emb is not None:
|
||||
fields["siglip_embedding"] = emb
|
||||
fields.update(overrides)
|
||||
return ImageRecord(**fields)
|
||||
|
||||
|
||||
def make_image(db, sha: str, emb=None, **overrides) -> ImageRecord:
|
||||
"""image_row, added and flushed on a sync session."""
|
||||
img = image_row(sha, emb, **overrides)
|
||||
db.add(img)
|
||||
db.flush()
|
||||
return img
|
||||
|
||||
|
||||
async def make_image_async(db, sha: str, emb=None, **overrides) -> ImageRecord:
|
||||
"""image_row, added and flushed on an async session."""
|
||||
img = image_row(sha, emb, **overrides)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
return img
|
||||
|
||||
|
||||
def make_tag(db, name: str, **overrides) -> Tag:
|
||||
"""A general tag, added and flushed on a sync session."""
|
||||
tag = Tag(**{"name": name, "kind": TagKind.general, **overrides})
|
||||
db.add(tag)
|
||||
db.flush()
|
||||
return tag
|
||||
@@ -0,0 +1,122 @@
|
||||
"""The agent's startup report of which runtime landed on the GPU (#1451).
|
||||
|
||||
Both runtimes fall back to the CPU without raising, so the report is the only
|
||||
thing that says so. These pin the distinction it exists for: onnxruntime
|
||||
listing the CUDA provider is not the same as the provider being able to load
|
||||
its libraries.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
|
||||
from agent.fc_agent import accel
|
||||
|
||||
|
||||
def _ort(providers, preload=None):
|
||||
mod = types.SimpleNamespace(
|
||||
__version__="1.30.0",
|
||||
__file__="/site/onnxruntime/__init__.py",
|
||||
get_available_providers=lambda: providers,
|
||||
)
|
||||
if preload is not None:
|
||||
mod.preload_dlls = preload
|
||||
return mod
|
||||
|
||||
|
||||
def _imp(mod):
|
||||
return lambda name: mod
|
||||
|
||||
|
||||
class _Cudart:
|
||||
"""libcudart as ctypes sees it: cudaGetDeviceCount writes through a pointer."""
|
||||
|
||||
def __init__(self, rc=0, count=1):
|
||||
self.rc = rc
|
||||
self.count = count
|
||||
self.cudaGetErrorString = lambda rc: b"unknown error"
|
||||
|
||||
def cudaGetDeviceCount(self, ref):
|
||||
ref._obj.value = self.count
|
||||
return self.rc
|
||||
|
||||
|
||||
def _loader(calls, cudart):
|
||||
def load(path, mode=0):
|
||||
calls.append(path)
|
||||
return cudart if path == "libcudart.so.13" else None
|
||||
return load
|
||||
|
||||
|
||||
GPU_BUILD = ["CUDAExecutionProvider", "CPUExecutionProvider"]
|
||||
|
||||
|
||||
def test_onnx_on_gpu_when_the_provider_loads_and_a_device_answers():
|
||||
calls = []
|
||||
s = accel.onnx_status(
|
||||
_imp(_ort(GPU_BUILD, lambda: calls.append("preload"))),
|
||||
load=_loader(calls, _Cudart()),
|
||||
)
|
||||
assert s["device"] == "cuda"
|
||||
assert calls[0] == "preload"
|
||||
assert any(c.endswith("capi/libonnxruntime_providers_cuda.so") for c in calls)
|
||||
|
||||
|
||||
def test_onnx_libraries_loading_is_not_a_gpu_when_cuda_cannot_initialise():
|
||||
"""The 2026-09-24 case: every library resolved, cuInit failed."""
|
||||
s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=_loader([], _Cudart(rc=999)))
|
||||
assert s["device"] == "cpu"
|
||||
assert "unknown error" in s["error"]
|
||||
|
||||
|
||||
def test_onnx_listed_but_unloadable_reports_cpu_with_the_reason():
|
||||
def load(path, mode=0):
|
||||
if path.endswith("providers_cuda.so"):
|
||||
raise OSError("libcudart.so.13: cannot open shared object file")
|
||||
|
||||
s = accel.onnx_status(_imp(_ort(GPU_BUILD)), load=load)
|
||||
assert s["device"] == "cpu"
|
||||
assert "libcudart.so.13" in s["error"]
|
||||
|
||||
|
||||
def test_onnx_cpu_build_never_tries_the_cuda_library():
|
||||
def load(path, mode=0):
|
||||
raise AssertionError("a CPU build has no CUDA provider to load")
|
||||
|
||||
s = accel.onnx_status(_imp(_ort(["CPUExecutionProvider"])), load=load)
|
||||
assert s["device"] == "cpu"
|
||||
|
||||
|
||||
def test_torch_reports_cpu_when_cuda_is_unavailable():
|
||||
torch = types.SimpleNamespace(
|
||||
__version__="2.14.0+cu130",
|
||||
version=types.SimpleNamespace(cuda="13.0"),
|
||||
cuda=types.SimpleNamespace(is_available=lambda: False),
|
||||
)
|
||||
s = accel.torch_status(_imp(torch))
|
||||
assert s == {"version": "2.14.0+cu130", "cuda_build": "13.0", "device": "cpu"}
|
||||
|
||||
|
||||
def test_a_missing_runtime_is_reported_not_raised():
|
||||
def imp(name):
|
||||
raise ImportError(f"No module named {name!r}")
|
||||
|
||||
assert accel.torch_status(imp)["device"] == "unavailable"
|
||||
assert accel.onnx_status(imp)["device"] == "unavailable"
|
||||
|
||||
|
||||
def test_summary_is_what_the_server_stores(monkeypatch):
|
||||
"""Device plus a bounded reason, per runtime — sent on every lease."""
|
||||
monkeypatch.setattr(accel, "LAST", {
|
||||
"torch": {"version": "2.14.0", "device": "cuda", "gpu": "RTX"},
|
||||
"onnx": {"version": "1.30.0", "device": "cpu", "error": "e" * 500},
|
||||
})
|
||||
assert accel.summary() == {
|
||||
"torch": {"device": "cuda"},
|
||||
"onnx": {"device": "cpu", "error": "e" * 200},
|
||||
}
|
||||
|
||||
|
||||
def test_summary_before_the_report_is_none(monkeypatch):
|
||||
monkeypatch.setattr(accel, "LAST", {})
|
||||
assert accel.summary() is None
|
||||
@@ -0,0 +1,59 @@
|
||||
"""A GPU agent that fell back to the CPU reads as DEGRADED, not running (#4410).
|
||||
|
||||
Both runtimes fall back without raising, so a CPU-bound agent leases, works
|
||||
and checks in exactly like a healthy one. On 2026-09-24 one had been doing so
|
||||
since a driver update left a stale CDI spec; the only sign was a line in the
|
||||
agent's own log. The agent now sends its startup report on every lease and
|
||||
heartbeat, and the System view derives the state from it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from backend.app.api.gpu import _accel_detail
|
||||
from backend.app.api.system_health import _SEVERITY, _learned_state
|
||||
|
||||
GPU = {"torch": {"device": "cuda"}, "onnx": {"device": "cuda"}}
|
||||
CPU = {
|
||||
"torch": {"device": "cpu"},
|
||||
"onnx": {"device": "cpu", "error": "cudaGetDeviceCount: unknown error (999)"},
|
||||
}
|
||||
|
||||
|
||||
def test_a_running_agent_on_the_gpu_is_ok():
|
||||
state, detail = _learned_state("GPU agent", "ok", 5, {"accel": GPU})
|
||||
assert state == "ok"
|
||||
assert detail == "GPU agent is running"
|
||||
|
||||
|
||||
def test_a_running_agent_on_the_cpu_is_degraded_and_says_why():
|
||||
state, detail = _learned_state("GPU agent", "ok", 5, {"accel": CPU})
|
||||
assert state == "degraded"
|
||||
assert "onnx (cudaGetDeviceCount: unknown error (999))" in detail
|
||||
assert "torch (cpu)" in detail
|
||||
|
||||
|
||||
def test_an_agent_that_never_reported_is_not_called_degraded():
|
||||
"""An older agent build sends no `accel`: that is not-yet-known, not slow."""
|
||||
assert _learned_state("GPU agent", "ok", 5, {})[0] == "ok"
|
||||
|
||||
|
||||
def test_stopped_outranks_degraded():
|
||||
"""Whether a quiet agent is still running is the more urgent question."""
|
||||
state, _ = _learned_state("GPU agent", "down", 900, {"accel": CPU})
|
||||
assert state == "down"
|
||||
assert _SEVERITY["stale"] > _SEVERITY["degraded"] > _SEVERITY["unknown"]
|
||||
|
||||
|
||||
def test_the_lease_keeps_only_a_bounded_report():
|
||||
"""Written on every lease by a client the server does not control."""
|
||||
body = {"accel": {
|
||||
"torch": {"device": "cpu", "error": "x" * 5000, "extra": "dropped"},
|
||||
"onnx": "not a dict",
|
||||
}}
|
||||
kept = _accel_detail(body)["accel"]
|
||||
assert kept == {"torch": {"device": "cpu", "error": "x" * 200}}
|
||||
|
||||
|
||||
def test_a_lease_without_a_report_adds_nothing():
|
||||
assert _accel_detail({}) == {}
|
||||
assert _accel_detail({"accel": None}) == {}
|
||||
+1
-10
@@ -4,6 +4,7 @@ import pytest
|
||||
from backend.app.models import ImageRecord, ImageRegion, TagKind
|
||||
from backend.app.models.tag import image_tag
|
||||
from backend.app.services.tag_service import TagService
|
||||
from tests.factories import make_image_async as _img
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
@@ -14,16 +15,6 @@ def _ccip(slot: int) -> list[float]:
|
||||
return v
|
||||
|
||||
|
||||
async def _img(db, sha) -> ImageRecord:
|
||||
img = ImageRecord(
|
||||
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
|
||||
width=1, height=1, origin="imported_filesystem", integrity_status="unknown",
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
return img
|
||||
|
||||
|
||||
async def _figure(db, image_id, ccip):
|
||||
db.add(ImageRegion(
|
||||
image_record_id=image_id, kind="figure", rx=0.0, ry=0.0, rw=1.0, rh=1.0,
|
||||
|
||||
@@ -355,6 +355,292 @@ async def test_quick_add_source_attaches_to_existing_artist(client, ext_key, db,
|
||||
assert artist_count == 1 # no duplicate created
|
||||
|
||||
|
||||
# --- Discord: channels are added to a CHOSEN artist (milestone 429) ---
|
||||
|
||||
|
||||
_GUILD = "111111111111111111"
|
||||
_CHAN = "222222222222222222"
|
||||
_OTHER_CHAN = "333333333333333333"
|
||||
|
||||
|
||||
async def _discord_source(db, name, url):
|
||||
artist = Artist(name=name, slug=name.lower(), is_subscription=True)
|
||||
db.add(artist)
|
||||
await db.flush()
|
||||
src = Source(artist_id=artist.id, platform="discord", url=url,
|
||||
enabled=True, config_overrides={})
|
||||
db.add(src)
|
||||
await db.commit()
|
||||
return artist, src
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_discord_channel_is_added_to_the_artist_the_operator_picked(
|
||||
client, ext_key, db, db_sync,
|
||||
):
|
||||
artist = Artist(name="Tamada", slug="tamada", is_subscription=True)
|
||||
db.add(artist)
|
||||
await db.commit()
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
# A jump link on the ptb host still names the channel.
|
||||
json={"url": f"https://ptb.discord.com/channels/{_GUILD}/{_CHAN}/999",
|
||||
"artist_id": artist.id},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 201, await resp.get_json()
|
||||
body = await resp.get_json()
|
||||
assert body["artist"]["id"] == artist.id
|
||||
assert body["created_artist"] is False
|
||||
# Stored canonical, so the manual form and the ingester read the same URL.
|
||||
assert body["source"]["url"] == f"https://discord.com/channels/{_GUILD}/{_CHAN}"
|
||||
assert body["source"]["platform"] == "discord"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_discord_add_can_name_a_new_artist(client, ext_key):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": f"https://discord.com/channels/{_GUILD}", "artist_name": "Studio Q"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 201
|
||||
body = await resp.get_json()
|
||||
assert body["artist"]["name"] == "Studio Q"
|
||||
assert body["created_artist"] is True
|
||||
assert body["source"]["url"] == f"https://discord.com/channels/{_GUILD}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_discord_add_with_no_artist_and_no_token_names_the_server(client, ext_key):
|
||||
"""No stored token means no server name to read; the fallback still names
|
||||
a readable artist rather than slugifying the ids."""
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": f"https://discord.com/channels/{_GUILD}/{_CHAN}"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 201
|
||||
assert (await resp.get_json())["artist"]["name"] == f"Discord {_GUILD}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_re_adding_a_discord_channel_keeps_its_artist(client, ext_key, db):
|
||||
"""Identity by source (#130), compared by ids: a row stored with a
|
||||
trailing slash is the same channel, and naming another artist does not
|
||||
move it."""
|
||||
owner, src = await _discord_source(
|
||||
db, "Owner", f"https://discord.com/channels/{_GUILD}/{_CHAN}/",
|
||||
)
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": f"https://discord.com/channels/{_GUILD}/{_CHAN}",
|
||||
"artist_name": "Someone Else"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = await resp.get_json()
|
||||
assert body["source"]["id"] == src.id
|
||||
assert body["artist"]["id"] == owner.id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_with_a_missing_artist_id_is_404(client, ext_key):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": f"https://discord.com/channels/{_GUILD}/{_CHAN}", "artist_id": 987654},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quick_add_rejects_a_non_integer_artist_id(client, ext_key):
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": f"https://discord.com/channels/{_GUILD}/{_CHAN}", "artist_id": "7"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 400
|
||||
|
||||
|
||||
async def _probe(client, ext_key, url):
|
||||
resp = await client.get(
|
||||
"/api/extension/probe", query_string={"url": url},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
return await resp.get_json()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_a_fresh_discord_channel_is_new_with_both_urls(client, ext_key):
|
||||
body = await _probe(client, ext_key, f"https://discord.com/channels/{_GUILD}/{_CHAN}")
|
||||
assert body["state"] == "new"
|
||||
assert body["platform"] == "discord"
|
||||
d = body["discord"]
|
||||
assert d["server_id"] == _GUILD and d["channel_id"] == _CHAN
|
||||
assert d["server_url"] == f"https://discord.com/channels/{_GUILD}"
|
||||
assert d["channel_url"] == f"https://discord.com/channels/{_GUILD}/{_CHAN}"
|
||||
# No token stored → no names, and no error.
|
||||
assert d["server_name"] is None and d["channel_name"] is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_suggests_the_artist_who_owns_another_channel_on_the_server(
|
||||
client, ext_key, db,
|
||||
):
|
||||
owner, _ = await _discord_source(
|
||||
db, "Owner", f"https://discord.com/channels/{_GUILD}/{_OTHER_CHAN}",
|
||||
)
|
||||
body = await _probe(client, ext_key, f"https://discord.com/channels/{_GUILD}/{_CHAN}")
|
||||
assert body["state"] == "artist_match"
|
||||
assert body["artist"]["id"] == owner.id
|
||||
assert "source" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_finds_the_channel_under_any_artist(client, ext_key, db):
|
||||
owner, src = await _discord_source(
|
||||
db, "Owner", f"https://discord.com/channels/{_GUILD}/{_CHAN}",
|
||||
)
|
||||
body = await _probe(client, ext_key, f"https://discord.com/channels/{_GUILD}/{_CHAN}/555")
|
||||
assert body["state"] == "source_match"
|
||||
assert body["source"]["id"] == src.id
|
||||
assert body["artist"]["id"] == owner.id
|
||||
assert body["covered_by_server"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_a_channel_is_covered_by_a_whole_server_source(client, ext_key, db):
|
||||
_, src = await _discord_source(db, "Owner", f"https://discord.com/channels/{_GUILD}")
|
||||
body = await _probe(client, ext_key, f"https://discord.com/channels/{_GUILD}/{_CHAN}")
|
||||
assert body["state"] == "source_match"
|
||||
assert body["source"]["id"] == src.id
|
||||
assert body["covered_by_server"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_a_discord_dm_is_not_a_source(client, ext_key):
|
||||
body = await _probe(client, ext_key, f"https://discord.com/channels/@me/{_CHAN}")
|
||||
assert body["state"] == "unknown_platform"
|
||||
|
||||
|
||||
# --- Patreon is canon: the Add panel's names and the rename (milestone 429) ---
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def platform_names(monkeypatch):
|
||||
"""Stub both platforms' display-name lookups (no network in tests)."""
|
||||
from backend.app.services import patreon_resolver
|
||||
from backend.app.services.credential_service import CredentialService
|
||||
from backend.app.services.subscribestar_client import SubscribeStarClient
|
||||
|
||||
async def _cookies(self, platform):
|
||||
return "/tmp/cookies.txt"
|
||||
|
||||
names = {"patreon": "Tamada Heijun", "subscribestar": "SS Tamada"}
|
||||
monkeypatch.setattr(CredentialService, "get_cookies_path", _cookies)
|
||||
monkeypatch.setattr(
|
||||
patreon_resolver, "resolve_display_name", lambda v, c: names["patreon"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SubscribeStarClient, "resolve_display_name", lambda self, u: names["subscribestar"],
|
||||
)
|
||||
return names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_probe_with_names_reads_the_patreon_display_name(client, ext_key, platform_names):
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
query_string={"url": "https://www.patreon.com/tamadaheijun", "names": "1"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
body = await resp.get_json()
|
||||
assert body["state"] == "new"
|
||||
assert body["display_name"] == "Tamada Heijun"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_plain_probe_does_not_look_the_name_up(client, ext_key, platform_names):
|
||||
resp = await client.get(
|
||||
"/api/extension/probe",
|
||||
query_string={"url": "https://www.patreon.com/tamadaheijun"},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert "display_name" not in await resp.get_json()
|
||||
|
||||
|
||||
async def _artist(db, name, slug):
|
||||
artist = Artist(name=name, slug=slug, is_subscription=True)
|
||||
db.add(artist)
|
||||
await db.commit()
|
||||
return artist
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_patreon_source_renames_the_artist_it_joins_to_the_patreon_name(
|
||||
client, ext_key, db, db_sync, platform_names,
|
||||
):
|
||||
artist = await _artist(db, "tamada", "tamada")
|
||||
resp = await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://www.patreon.com/tamadaheijun",
|
||||
"artist_id": artist.id, "use_platform_name": True},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)
|
||||
assert resp.status_code == 201
|
||||
body = await resp.get_json()
|
||||
assert body["artist"]["name"] == "Tamada Heijun"
|
||||
assert body["renamed_from"] == "tamada"
|
||||
# Name only: the slug, and every path keyed off it, stays.
|
||||
row = db_sync.execute(select(Artist.name, Artist.slug).where(Artist.id == artist.id)).one()
|
||||
assert tuple(row) == ("Tamada Heijun", "tamada")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_rename_without_the_flag(client, ext_key, db, platform_names):
|
||||
artist = await _artist(db, "tamada", "tamada")
|
||||
body = await (await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://www.patreon.com/tamadaheijun", "artist_id": artist.id},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)).get_json()
|
||||
assert body["artist"]["name"] == "tamada"
|
||||
assert "renamed_from" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unreadable_patreon_name_never_renames_to_the_handle(
|
||||
client, ext_key, db, platform_names,
|
||||
):
|
||||
platform_names["patreon"] = None
|
||||
artist = await _artist(db, "Tamada", "tamada")
|
||||
body = await (await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://www.patreon.com/tamadaheijun",
|
||||
"artist_id": artist.id, "use_platform_name": True},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)).get_json()
|
||||
assert body["artist"]["name"] == "Tamada"
|
||||
assert "renamed_from" not in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_patreon_names_are_canon(client, ext_key, db, platform_names):
|
||||
"""A SubscribeStar source joins the picked artist under the name it has."""
|
||||
artist = await _artist(db, "Tamada Heijun", "tamada-heijun")
|
||||
body = await (await client.post(
|
||||
"/api/extension/quick-add-source",
|
||||
json={"url": "https://subscribestar.adult/tamada",
|
||||
"artist_id": artist.id, "use_platform_name": True},
|
||||
headers={"X-Extension-Key": ext_key},
|
||||
)).get_json()
|
||||
assert body["artist"]["name"] == "Tamada Heijun"
|
||||
assert "renamed_from" not in body
|
||||
|
||||
|
||||
# --- /api/extension/manifest ---------------------------------------
|
||||
|
||||
|
||||
|
||||
+1
-10
@@ -7,20 +7,11 @@ from sqlalchemy import func, select
|
||||
from backend.app.models import GpuJob, ImageRecord
|
||||
from backend.app.services.ml.gpu_jobs import GpuJobService
|
||||
from backend.app.services.ml.regions import RegionService
|
||||
from tests.factories import make_image_async as _img
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
async def _img(db, sha) -> ImageRecord:
|
||||
img = ImageRecord(
|
||||
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
|
||||
width=1, height=1, origin="imported_filesystem", integrity_status="unknown",
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
return img
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_endpoints_require_bearer(client, db):
|
||||
resp = await client.post("/api/gpu/jobs/lease", json={"agent_id": "a1"})
|
||||
|
||||
@@ -5,20 +5,11 @@ from backend.app.models import ImageRecord, TagHead, TagKind
|
||||
from backend.app.models.tag import image_tag
|
||||
from backend.app.models.tag_suggestion_rejection import TagSuggestionRejection
|
||||
from backend.app.services.tag_service import TagService
|
||||
from tests.factories import make_image_async as _img
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
|
||||
async def _img(db, sha) -> ImageRecord:
|
||||
img = ImageRecord(
|
||||
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
|
||||
width=1, height=1, origin="imported_filesystem", integrity_status="unknown",
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
return img
|
||||
|
||||
|
||||
async def _apply(db, image_id, tag_id, source):
|
||||
await db.execute(image_tag.insert().values(
|
||||
image_record_id=image_id, tag_id=tag_id, source=source,
|
||||
|
||||
@@ -50,6 +50,34 @@ async def test_autocomplete_ranks_exact_prefix_substring(db):
|
||||
assert "Bob" not in names
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_ignores_spacing_and_punctuation(db):
|
||||
"""The same creator is spelled differently per platform — a Discord server
|
||||
"Tamada Heijun" must find the Patreon artist "TamadaHeijun" (milestone 429)."""
|
||||
db.add_all([
|
||||
Artist(name="TamadaHeijun", slug="tamadaheijun"),
|
||||
Artist(name="Sabu Art", slug="sabu-art"),
|
||||
Artist(name="Bob", slug="bob"),
|
||||
])
|
||||
await db.flush()
|
||||
svc = ArtistService(db)
|
||||
assert [r.name for r in await svc.autocomplete("Tamada Heijun")] == ["TamadaHeijun"]
|
||||
assert [r.name for r in await svc.autocomplete("sabu_art")] == ["Sabu Art"]
|
||||
# All punctuation: no squashed arm, so it does not match everyone.
|
||||
assert await svc.autocomplete("--") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_ranks_an_exact_match_ignoring_spacing_above_a_prefix(db):
|
||||
db.add_all([
|
||||
Artist(name="Sabu Artworks", slug="sabu-artworks"),
|
||||
Artist(name="SabuArt", slug="sabuart"),
|
||||
])
|
||||
await db.flush()
|
||||
names = [r.name for r in await ArtistService(db).autocomplete("sabu art")]
|
||||
assert names == ["SabuArt", "Sabu Artworks"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_autocomplete_empty_query_returns_empty(db):
|
||||
svc = ArtistService(db)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
"""#4408: beat takes each job's last run from task_run, not a shelve file.
|
||||
|
||||
The default scheduler forgot everything on each container recreate and waited
|
||||
a full interval before any job, so with several redeploys a day no daily or
|
||||
weekly job had run since 2026-09-21.
|
||||
"""
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import pytest
|
||||
from celery.beat import ScheduleEntry
|
||||
from celery.schedules import schedule
|
||||
|
||||
from backend.app.beat_scheduler import NEVER, last_runs, seed
|
||||
from backend.app.celery_app import celery
|
||||
from backend.app.models import TaskRun
|
||||
|
||||
DAY = 86400.0
|
||||
|
||||
|
||||
def _entry(task, every=DAY):
|
||||
return ScheduleEntry(name=task, task=task, schedule=schedule(every, app=celery), app=celery)
|
||||
|
||||
|
||||
def test_a_job_that_ran_recently_waits_only_the_remainder():
|
||||
entry = _entry("t.daily")
|
||||
seed([entry], {"t.daily": datetime.now(UTC) - timedelta(hours=20)})
|
||||
due, next_in = entry.is_due()
|
||||
assert not due
|
||||
assert 3 * 3600 < next_in <= 4 * 3600 + 5
|
||||
|
||||
|
||||
def test_an_overdue_job_is_due_at_once():
|
||||
"""The live case: daily jobs last ran three days before the restart."""
|
||||
entry = _entry("t.daily")
|
||||
seed([entry], {"t.daily": datetime.now(UTC) - timedelta(days=3)})
|
||||
assert entry.is_due()[0]
|
||||
|
||||
|
||||
def test_a_job_with_no_recorded_run_is_due_now():
|
||||
entry = _entry("t.never")
|
||||
seed([entry], {})
|
||||
assert entry.last_run_at == NEVER
|
||||
assert entry.is_due()[0]
|
||||
|
||||
|
||||
def test_the_scheduler_is_the_one_celery_uses():
|
||||
assert celery.conf.beat_scheduler == "backend.app.beat_scheduler:TaskRunScheduler"
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_last_runs_reads_the_newest_start_per_task(db_sync):
|
||||
now = datetime.now(UTC)
|
||||
for name, ago in [("t.a", 30), ("t.a", 2), ("t.b", 5)]:
|
||||
db_sync.add(TaskRun(
|
||||
celery_task_id="x", queue="maintenance", task_name=name,
|
||||
started_at=now - timedelta(hours=ago), status="ok",
|
||||
))
|
||||
db_sync.flush()
|
||||
|
||||
got = last_runs(db_sync, ["t.a", "t.b", "t.c"])
|
||||
|
||||
assert set(got) == {"t.a", "t.b"}
|
||||
assert abs((got["t.a"] - (now - timedelta(hours=2))).total_seconds()) < 1
|
||||
+1
-10
@@ -13,6 +13,7 @@ from backend.app.models import (
|
||||
from backend.app.models.tag import image_tag
|
||||
from backend.app.services.ml.ccip import match_image
|
||||
from backend.app.services.tag_service import TagService
|
||||
from tests.factories import make_image_async as _img
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
@@ -23,16 +24,6 @@ def _ccip(slot: int) -> list[float]:
|
||||
return v
|
||||
|
||||
|
||||
async def _img(db, sha) -> ImageRecord:
|
||||
img = ImageRecord(
|
||||
path=f"/images/{sha}.jpg", sha256=sha, size_bytes=1, mime="image/jpeg",
|
||||
width=1, height=1, origin="imported_filesystem", integrity_status="unknown",
|
||||
)
|
||||
db.add(img)
|
||||
await db.flush()
|
||||
return img
|
||||
|
||||
|
||||
async def _figure(db, image_id, ccip):
|
||||
db.add(ImageRegion(
|
||||
image_record_id=image_id, kind="figure",
|
||||
|
||||
@@ -36,3 +36,17 @@ def test_queue_for_mirrors_external_to_download():
|
||||
celery.conf.task_routes["backend.app.tasks.external.*"]["queue"]
|
||||
== "download"
|
||||
)
|
||||
|
||||
|
||||
def test_backfill_phash_runs_on_the_long_lane():
|
||||
"""It lives in maintenance.py, so the quick-lane glob matches it too —
|
||||
the router must pick the exact name. A 35-minute rehash on the scheduler
|
||||
lane blocked the minute ticks behind it (2026-09-24)."""
|
||||
route = celery.amqp.router.route(
|
||||
{}, "backend.app.tasks.maintenance.backfill_phash",
|
||||
)
|
||||
assert route["queue"].name == "maintenance_long"
|
||||
quick = celery.amqp.router.route(
|
||||
{}, "backend.app.tasks.maintenance.recover_stalled_task_runs",
|
||||
)
|
||||
assert quick["queue"].name == "maintenance"
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user