Compare commits
296
Commits
2f9e35390e
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b0a6c5f4ab | ||
|
|
31c94d6beb | ||
|
|
e92497cff3 | ||
|
|
3f5224800e | ||
|
|
3c3045190d | ||
|
|
0928d2f914 | ||
|
|
ef591edf10 | ||
|
|
1efece0f21 | ||
|
|
20aeac81df | ||
|
|
d6d3184361 | ||
|
|
29c22afcb7 | ||
|
|
063778e6b3 | ||
|
|
42a60e3e74 | ||
|
|
621c2b8315 | ||
|
|
ca802bd885 | ||
|
|
e2eeb63115 | ||
|
|
e704c70f32 | ||
|
|
0fad744bfb | ||
|
|
32874ca678 | ||
|
|
8891e20203 | ||
|
|
edd2daa16a | ||
|
|
23e062dd4a | ||
|
|
a360d69ee8 | ||
|
|
dfd28a0aa6 | ||
|
|
efcb548ebf | ||
|
|
4fa7975963 | ||
|
|
7b1570f2a5 | ||
|
|
83e1382812 | ||
|
|
423275a1e5 | ||
|
|
eeb9263125 | ||
|
|
4c75dd0f88 | ||
|
|
97045279a3 | ||
|
|
e5bdcd2596 | ||
|
|
84e5448941 | ||
|
|
058fa85606 | ||
|
|
cc53d8db7b | ||
|
|
e39ec1c550 | ||
|
|
42a40d71a4 | ||
|
|
f127c50207 | ||
|
|
7a09dc3cda | ||
|
|
2587421f5b | ||
|
|
ff70f837d0 | ||
|
|
0842df46e9 | ||
|
|
f2e4ee4d17 | ||
|
|
31020d9395 | ||
|
|
ee59133b42 | ||
|
|
7cc34b5724 | ||
|
|
20cb813809 | ||
|
|
3aa899ad29 | ||
|
|
da2091a875 | ||
|
|
49d18c1757 | ||
|
|
6bf205407b | ||
|
|
32d37cd453 | ||
|
|
e2246e861e | ||
|
|
b4e7b98390 | ||
|
|
c3dac235a9 | ||
|
|
69d9c14e35 | ||
|
|
057bab1011 | ||
|
|
a8fdbd86bc | ||
|
|
0421fd3109 | ||
|
|
d01f33dea6 | ||
|
|
adab33694d | ||
|
|
3e4d39b111 | ||
|
|
b93a5fc5b4 | ||
|
|
725bf15f88 | ||
|
|
b14818303c | ||
|
|
499720d87e | ||
|
|
bd24f4e876 | ||
|
|
0e15c44c51 | ||
|
|
6d98dfc0ec | ||
|
|
6c76f08b69 | ||
|
|
7b1019ba82 | ||
|
|
0d204e6837 | ||
|
|
8300029741 | ||
|
|
b5b437ca80 | ||
|
|
ce0dac3524 | ||
|
|
9b5ec86222 | ||
|
|
89c83ee5de | ||
|
|
d3192f1843 | ||
|
|
5a5694f200 | ||
|
|
bb1a938cc0 | ||
|
|
fc0293029d | ||
|
|
7d1c701b67 | ||
|
|
b638382cd5 | ||
|
|
17903068b4 | ||
|
|
31d400ab0a | ||
|
|
d45ce5e426 | ||
|
|
77bfc32b02 | ||
|
|
d13efe1b69 | ||
|
|
ca6d26d236 | ||
|
|
5366047d55 | ||
|
|
2923257529 | ||
|
|
934731e9ef | ||
|
|
4c67c116b2 | ||
|
|
f8b667604f | ||
|
|
11572f469a | ||
|
|
ba3fd2a118 | ||
|
|
9bf095179a | ||
|
|
cefc064e0f | ||
|
|
06757daf80 | ||
|
|
a723ef436b | ||
|
|
d68f39ca50 | ||
|
|
a66a695977 | ||
|
|
5289fa3879 | ||
|
|
ebac34e17b | ||
|
|
4c3ba20198 | ||
|
|
d2acec61ae | ||
|
|
15ae5ef4fa | ||
|
|
e3ceefc820 | ||
|
|
e52090a4cf | ||
|
|
bfba8045e4 | ||
|
|
a708357436 | ||
|
|
d9354ac1e1 | ||
|
|
1d48770793 | ||
|
|
489e6aaaee | ||
|
|
ed20df905b | ||
|
|
5ba9871ef0 | ||
|
|
2a820d0848 | ||
|
|
2f9aa3d86c | ||
|
|
560a5000a2 | ||
|
|
7ddad231f8 | ||
|
|
a78f7eaace | ||
|
|
71337b0ba4 | ||
|
|
3996205f3b | ||
|
|
9f9db01456 | ||
|
|
216b7fc743 | ||
|
|
fbb76e6f36 | ||
|
|
5ef1478ade | ||
|
|
88ff4147e1 | ||
|
|
1ea02ad44c | ||
|
|
937cfb65b4 | ||
|
|
baac851220 | ||
|
|
a28c33281a | ||
|
|
40be0a9323 | ||
|
|
2c6bf26bfc | ||
|
|
3c1e76bd44 | ||
|
|
831c5b1c10 | ||
|
|
76b6af4903 | ||
|
|
92494ec4ed | ||
|
|
bdfc17477c | ||
|
|
6cd3153bf4 | ||
|
|
5f2853168a | ||
|
|
9a979ee808 | ||
|
|
3138f912fd | ||
|
|
9df874e396 | ||
|
|
6915a7590a | ||
|
|
5e8c28236a | ||
|
|
7e3c0f0b74 | ||
|
|
5d0c7ba706 | ||
|
|
18300e1f8a | ||
|
|
d52ac0a0e2 | ||
|
|
401fe8213e | ||
|
|
e8774d7953 | ||
|
|
bc0f00c51b | ||
|
|
1bef68aa29 | ||
|
|
1a4bc2f981 | ||
|
|
862ace69d6 | ||
|
|
abf88b1a15 | ||
|
|
c05dcafbea | ||
|
|
55e8632dab | ||
|
|
825e6b90bf | ||
|
|
fb012c557c | ||
|
|
66593ab895 | ||
|
|
b266a54ad3 | ||
|
|
ad803b646f | ||
|
|
1f5da3d283 | ||
|
|
93034f580d | ||
|
|
9b9b12f410 | ||
|
|
376d310693 | ||
|
|
bc69495a16 | ||
|
|
478f898e72 | ||
|
|
38a5e7f332 | ||
|
|
57fe15c267 | ||
|
|
eb3231ef10 | ||
|
|
e9af459c0d | ||
|
|
6f02806aec | ||
|
|
a1d19bd96a | ||
|
|
26827ff38f | ||
|
|
26dcfaf6c2 | ||
|
|
9b1b0369cc | ||
|
|
18123fb9cb | ||
|
|
2e806f202f | ||
|
|
18d5c05639 | ||
|
|
11ddfc3876 | ||
|
|
2b8ce86622 | ||
|
|
49bee77cdc | ||
|
|
c209e3b37e | ||
|
|
cffdd93418 | ||
|
|
fd84be40dd | ||
|
|
79f510d7f8 | ||
|
|
59181069da | ||
|
|
428ecd8642 | ||
|
|
ed1e04b831 | ||
|
|
f5156bd847 | ||
|
|
dfc3922d24 | ||
|
|
3eb08e926b | ||
|
|
9e81ced359 | ||
|
|
11e9f5af60 | ||
|
|
909fa37b15 | ||
|
|
dfab8f65ff | ||
|
|
618f7cdc36 | ||
|
|
028ea33a7c | ||
|
|
444c1fb075 | ||
|
|
26c68b0a75 | ||
|
|
e75427b19a | ||
|
|
5447fab987 | ||
|
|
bad37e07b2 | ||
|
|
2bfc9936a1 | ||
|
|
4c6406ee18 | ||
|
|
bb47e80b3e | ||
|
|
dc1083b5e0 | ||
|
|
e46893fefd | ||
|
|
0666e15211 | ||
|
|
747390631d | ||
|
|
e0d2a20588 | ||
|
|
1d84f67418 | ||
|
|
91265df3d6 | ||
|
|
11acdb0322 | ||
|
|
2eb9fd5dd0 | ||
|
|
01e5ce1410 | ||
|
|
3bb94674cf | ||
|
|
a75c602175 | ||
|
|
ef8f4f7193 | ||
|
|
ec3d27b219 | ||
|
|
03bd3b2eda | ||
|
|
7395e77d75 | ||
|
|
575d817919 | ||
|
|
2a8f7cd8b6 | ||
|
|
83f8af8090 | ||
|
|
9a2617c1a2 | ||
|
|
81688815a0 | ||
|
|
773128c3bf | ||
|
|
ce7b154ae9 | ||
|
|
9430a9d9c3 | ||
|
|
23aee56ce3 | ||
|
|
711abea567 | ||
|
|
844bb86802 | ||
|
|
a8f6a464aa | ||
|
|
ab9922ad2e | ||
|
|
0533807669 | ||
|
|
279dff3fb6 | ||
|
|
37e66cddc4 | ||
|
|
9cf6b2d363 | ||
|
|
6ef0fed41f | ||
|
|
89b48f8f35 | ||
|
|
d60e0b9494 | ||
|
|
9c27a2d3c7 | ||
|
|
93e37681b7 | ||
|
|
64ca858574 | ||
|
|
9d0c0b7da8 | ||
|
|
8e4d252ae4 | ||
|
|
fdd3e01f56 | ||
|
|
c82fb308b6 | ||
|
|
8cf8d2ca4d | ||
|
|
b1d58bc3b8 | ||
|
|
65386f02a0 | ||
|
|
667b05f14e | ||
|
|
856e9104b4 | ||
|
|
0397642b21 | ||
|
|
237575447d | ||
|
|
ed358757dc | ||
|
|
d181f4afb8 | ||
|
|
2886fa4997 | ||
|
|
f256f587ee | ||
|
|
384d8d5e50 | ||
|
|
319e8c1d18 | ||
|
|
9075d8eadd | ||
|
|
88e53e5b86 | ||
|
|
37e8b796a1 | ||
|
|
4e82208926 | ||
|
|
52fff00353 | ||
|
|
c14338cbce | ||
|
|
8c36dd28b0 | ||
|
|
88cfb3dd02 | ||
|
|
5d4f223b71 | ||
|
|
05090c6e85 | ||
|
|
3a577d5ade | ||
|
|
f4fe02e346 | ||
|
|
e766197d99 | ||
|
|
3872e1dda9 | ||
|
|
9814f3dbaf | ||
|
|
b214460fdb | ||
|
|
ac55d0e8d8 | ||
|
|
89a89e0ded | ||
|
|
4e9aac2c05 | ||
|
|
2879ac6f2b | ||
|
|
b8dce6c483 | ||
|
|
d1c0b82a22 | ||
|
|
5526b8dc78 | ||
|
|
16eb7075c4 | ||
|
|
885dcf64f3 | ||
|
|
f2f6b6d25e | ||
|
|
0822240fde | ||
|
|
27f7f3fd01 | ||
|
|
c5bf564f53 | ||
|
|
602c7d275d |
@@ -81,8 +81,18 @@ jobs:
|
|||||||
set -eux
|
set -eux
|
||||||
# Same service-IP dance as build.yml's integration job; see the long
|
# Same service-IP dance as build.yml's integration job; see the long
|
||||||
# comment there for why the job name must stay separator-free.
|
# comment there for why the job name must stay separator-free.
|
||||||
PG=$(docker ps --filter "name=compare" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
# Only THIS job's services. The runner's docker daemon is shared, so another
|
||||||
test -n "$PG"
|
# repo's job can be running beside this one, and a job-name filter matches its
|
||||||
|
# containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
|
||||||
|
# Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
|
||||||
|
# <n> from our own container (its id is in the /etc/hostname bind mount) and
|
||||||
|
# require exactly one match.
|
||||||
|
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
|
||||||
|
SELF=${SELF:-$(hostname)}
|
||||||
|
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
|
||||||
|
test -n "$TASK"
|
||||||
|
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
|
||||||
|
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
|
||||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||||
test -n "$PG_IP"
|
test -n "$PG_IP"
|
||||||
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
echo "PG_CONTAINER=$PG" >> "$GITHUB_ENV"
|
||||||
|
|||||||
+275
-87
@@ -142,6 +142,16 @@ concurrency:
|
|||||||
# Deriving it per job invites the two halves to disagree: sign-extension would
|
# Deriving it per job invites the two halves to disagree: sign-extension would
|
||||||
# derive dev's extension version while build-web bundled main's, and the
|
# derive dev's extension version while build-web bundled main's, and the
|
||||||
# release download would 404 on a version that exists perfectly well.
|
# release download would 404 on a version that exists perfectly well.
|
||||||
|
#
|
||||||
|
# On every other trigger it is the COMMIT that fired (`github.sha`), never the
|
||||||
|
# branch name. A branch is re-resolved by each job's checkout when that job
|
||||||
|
# starts, so a push landing mid-run moved the later jobs onto the new tip: run
|
||||||
|
# 7499 signed 423275a's extension, then build-web checked out 83e1382 (pushed
|
||||||
|
# while 7499 ran), derived a version nobody had signed, and 404'd on the
|
||||||
|
# download (#4427). The guard failed closed that time; a job without one would
|
||||||
|
# have published a commit the run's own lanes never tested — the lanes check
|
||||||
|
# out `github.sha` by default, so pinning here makes the publish build exactly
|
||||||
|
# what they passed.
|
||||||
# IS THIS A BASE REFRESH? Asked in five places and previously spelled five
|
# IS THIS A BASE REFRESH? Asked in five places and previously spelled five
|
||||||
# ways — `github.event_name == 'schedule'` in an `if:`, `$GITHUB_EVENT_NAME` in
|
# ways — `github.event_name == 'schedule'` in an `if:`, `$GITHUB_EVENT_NAME` in
|
||||||
# one shell, an `EVENT:` env passed into another, and a bare expression on
|
# one shell, an `EVENT:` env passed into another, and a bare expression on
|
||||||
@@ -178,7 +188,16 @@ concurrency:
|
|||||||
# where a step-level `if:` needs the answer before any shell runs.
|
# where a step-level `if:` needs the answer before any shell runs.
|
||||||
env:
|
env:
|
||||||
IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }}
|
IS_REFRESH: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'true' || 'false' }}
|
||||||
BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.ref }}
|
BUILD_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || github.sha }}
|
||||||
|
# What the six LANES check out. Empty — the checkout default, the triggering
|
||||||
|
# commit (or a PR's merge ref) — on every trigger but the refresh, where it is
|
||||||
|
# `main`: the refresh publishes main, so the gate has to test main (#4430). It
|
||||||
|
# is not BUILD_REF itself because a pull_request run's `github.sha` is a merge
|
||||||
|
# commit the default checkout reaches through its ref, not by sha. Each job
|
||||||
|
# still resolves `main` when it starts, so a merge to main during the ~5 min of
|
||||||
|
# a Sunday-06:00 refresh could put the lanes and the build one commit apart;
|
||||||
|
# the build jobs' own guards assert the branch, not the commit.
|
||||||
|
LANE_REF: ${{ (github.event_name == 'schedule' || format('{0}', github.event.inputs.refresh) == 'true') && 'main' || '' }}
|
||||||
|
|
||||||
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
# Requires repo secret RELEASE_TOKEN — a Forgejo PAT with scopes:
|
||||||
# - write:package, read:package (for docker push to git.fabledsword.com)
|
# - write:package, read:package (for docker push to git.fabledsword.com)
|
||||||
@@ -219,6 +238,8 @@ jobs:
|
|||||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ env.LANE_REF }}
|
||||||
- name: Ruff lint
|
- name: Ruff lint
|
||||||
# agent/ included so the GPU-agent is linted before its image is built
|
# agent/ included so the GPU-agent is linted before its image is built
|
||||||
# (build.yml only `docker build`s it — this is where it gets checked).
|
# (build.yml only `docker build`s it — this is where it gets checked).
|
||||||
@@ -242,7 +263,7 @@ jobs:
|
|||||||
# rather than left running beside the new mechanism (rule 22).
|
# rather than left running beside the new mechanism (rule 22).
|
||||||
#
|
#
|
||||||
# Two things are still worth asserting, and this is the only lane that can:
|
# Two things are still worth asserting, and this is the only lane that can:
|
||||||
# the extension.yml suite runs on node:24-slim, which is exactly why
|
# the extension-test lane runs on node:24-slim, which is exactly why
|
||||||
# version.spec.js sticks to packaging.sh's git-free subcommands.
|
# version.spec.js sticks to packaging.sh's git-free subcommands.
|
||||||
# 1. the derivation actually resolves on this commit
|
# 1. the derivation actually resolves on this commit
|
||||||
# 2. the derived string is one AMO will accept, checked against Mozilla's
|
# 2. the derived string is one AMO will accept, checked against Mozilla's
|
||||||
@@ -265,6 +286,7 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
|
ref: ${{ env.LANE_REF }}
|
||||||
# The derivation needs real history: a depth-1 clone sees one commit
|
# The derivation needs real history: a depth-1 clone sees one commit
|
||||||
# and produces a wrong, too-low value RATHER THAN FAILING. Checking
|
# and produces a wrong, too-low value RATHER THAN FAILING. Checking
|
||||||
# that here is half the point of the lane.
|
# that here is half the point of the lane.
|
||||||
@@ -320,6 +342,7 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
|
ref: ${{ env.LANE_REF }}
|
||||||
# Full history for tests/test_artifact_identity.py, which derives
|
# Full history for tests/test_artifact_identity.py, which derives
|
||||||
# each artifact's revision to check the identity scheme. On a
|
# each artifact's revision to check the identity scheme. On a
|
||||||
# depth-1 clone that derivation either fails or returns the tip sha
|
# depth-1 clone that derivation either fails or returns the tip sha
|
||||||
@@ -366,6 +389,8 @@ jobs:
|
|||||||
working-directory: frontend
|
working-directory: frontend
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ env.LANE_REF }}
|
||||||
# No package-lock.json is tracked yet (we don't run npm locally per
|
# No package-lock.json is tracked yet (we don't run npm locally per
|
||||||
# feedback-no-local-runs). Using `npm install` instead of `npm ci`.
|
# feedback-no-local-runs). Using `npm install` instead of `npm ci`.
|
||||||
# If we want strict lockfile-based reproducibility later, commit a
|
# If we want strict lockfile-based reproducibility later, commit a
|
||||||
@@ -378,6 +403,78 @@ jobs:
|
|||||||
- run: npm run test:unit
|
- run: npm run test:unit
|
||||||
- run: npm run build
|
- run: npm run build
|
||||||
|
|
||||||
|
# The extension's lane: web-ext lint, the vitest suite, and the check that
|
||||||
|
# asks web-ext what it ACTUALLY packaged. Moved in from `extension.yml`
|
||||||
|
# (deleted) at milestone 429 — the same move `ci.yml` made on 2026-09-23 and
|
||||||
|
# for the same reason: a separate workflow cannot gate this one, so a red
|
||||||
|
# extension suite still let `sign-extension` sign and `build-web` ship the
|
||||||
|
# XPI. It is a lane in THE GATE now. Its old path filter is gone with it; a
|
||||||
|
# filter here would make the lane skip, and a skipped lane blocks the publish.
|
||||||
|
#
|
||||||
|
# The vitest suite is also the JS half of the #3093 artist-pattern mirror —
|
||||||
|
# the Python half runs in backend-lint-and-test — so until this move one half
|
||||||
|
# of that guard could fail without stopping anything.
|
||||||
|
extension-test:
|
||||||
|
runs-on: python-ci
|
||||||
|
container:
|
||||||
|
image: node:24-bookworm-slim
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ env.LANE_REF }}
|
||||||
|
# Not --no-save: vitest and web-ext are both real devDependencies now,
|
||||||
|
# and the suite needs vitest resolvable from node_modules.
|
||||||
|
- name: Install dev dependencies
|
||||||
|
run: cd extension && npm install --no-audit --no-fund
|
||||||
|
- name: Lint
|
||||||
|
run: cd extension && npm run lint
|
||||||
|
# Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
|
||||||
|
# package version-consistency checks. No browser, no network.
|
||||||
|
- name: Unit tests
|
||||||
|
run: cd extension && npm run test:unit
|
||||||
|
|
||||||
|
# Everything else about packaging is asserted against our own declaration
|
||||||
|
# of what ships. This is the only check that asks web-ext what it ACTUALLY
|
||||||
|
# put in the archive. Until now that was an unverified assumption about
|
||||||
|
# glob semantics — and a fragile one: `test/**` reaches web-ext intact
|
||||||
|
# only because callers `set -f` first, so losing that quoting would
|
||||||
|
# silently start shipping dev files with no other signal.
|
||||||
|
- name: Verify XPI contents
|
||||||
|
run: |
|
||||||
|
set -eu
|
||||||
|
command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
|
||||||
|
cd extension
|
||||||
|
npm run build
|
||||||
|
ZIP=$(ls web-ext-artifacts/*.zip | head -1)
|
||||||
|
echo "=== packaged entries in $ZIP ==="
|
||||||
|
unzip -Z1 "$ZIP" | sort
|
||||||
|
echo "=== end ==="
|
||||||
|
ENTRIES=$(unzip -Z1 "$ZIP")
|
||||||
|
fail=0
|
||||||
|
# Must NOT ship: repo infrastructure with no business in a user's browser.
|
||||||
|
for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
|
||||||
|
if echo "$ENTRIES" | grep -q "^$pat"; then
|
||||||
|
echo "ERROR: '$pat' was packaged into the XPI but must not be"
|
||||||
|
fail=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
# Must ship: if an exclusion pattern ever over-matches, the extension
|
||||||
|
# breaks at runtime rather than at build time, so assert presence too.
|
||||||
|
for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js' 'lib/chip.js' 'lib/popup-format.js'; do
|
||||||
|
if ! echo "$ENTRIES" | grep -q "^$req$"; then
|
||||||
|
echo "ERROR: '$req' is missing from the XPI"
|
||||||
|
fail=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
|
||||||
|
if ! echo "$ENTRIES" | grep -q "^$dir"; then
|
||||||
|
echo "ERROR: nothing from '$dir' was packaged"
|
||||||
|
fail=1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
[ "$fail" -eq 0 ] || exit 1
|
||||||
|
echo "XPI contents verified."
|
||||||
|
|
||||||
# Single integration job — collapsed from a 3-way shard split on 2026-06-04.
|
# Single integration job — collapsed from a 3-way shard split on 2026-06-04.
|
||||||
# The shards existed to parallelize ~8.5min of integration tests; once the
|
# The shards existed to parallelize ~8.5min of integration tests; once the
|
||||||
# throwaway Postgres runs with fsync OFF (the durability step below) the whole
|
# throwaway Postgres runs with fsync OFF (the durability step below) the whole
|
||||||
@@ -419,7 +516,7 @@ jobs:
|
|||||||
--health-timeout 5s
|
--health-timeout 5s
|
||||||
--health-retries 10
|
--health-retries 10
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:8-alpine
|
||||||
options: >-
|
options: >-
|
||||||
--health-cmd "redis-cli ping"
|
--health-cmd "redis-cli ping"
|
||||||
--health-interval 10s
|
--health-interval 10s
|
||||||
@@ -427,15 +524,28 @@ jobs:
|
|||||||
--health-retries 10
|
--health-retries 10
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
ref: ${{ env.LANE_REF }}
|
||||||
- name: Integration suite (resolve service IPs, migrate, test)
|
- name: Integration suite (resolve service IPs, migrate, test)
|
||||||
run: |
|
run: |
|
||||||
set -eux
|
set -eux
|
||||||
echo "=== container landscape (diagnostic for filter scoping) ==="
|
echo "=== container landscape (diagnostic for filter scoping) ==="
|
||||||
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
docker ps -a --format '{{.ID}} {{.Image}} -> {{.Names}}'
|
||||||
echo "=== end landscape ==="
|
echo "=== end landscape ==="
|
||||||
PG=$(docker ps --filter "name=integration" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
# Only THIS job's services. The runner's docker daemon is shared, so another
|
||||||
RD=$(docker ps --filter "name=integration" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
# repo's job can be running beside this one, and a job-name filter matches its
|
||||||
test -n "$PG" && test -n "$RD"
|
# containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
|
||||||
|
# Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
|
||||||
|
# <n> from our own container (its id is in the /etc/hostname bind mount) and
|
||||||
|
# require exactly one match.
|
||||||
|
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
|
||||||
|
SELF=${SELF:-$(hostname)}
|
||||||
|
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
|
||||||
|
test -n "$TASK"
|
||||||
|
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
|
||||||
|
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
|
||||||
|
RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
|
||||||
|
test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
|
||||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||||
test -n "$PG_IP" && test -n "$RD_IP"
|
test -n "$PG_IP" && test -n "$RD_IP"
|
||||||
@@ -521,7 +631,8 @@ jobs:
|
|||||||
# itself through a job-level `if:` that could not read `env`, and a design
|
# itself through a job-level `if:` that could not read `env`, and a design
|
||||||
# where only a FAILED gate blocks would have published unverified images
|
# where only a FAILED gate blocks would have published unverified images
|
||||||
# while reporting success.
|
# while reporting success.
|
||||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
|
needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
|
||||||
|
integration]
|
||||||
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
||||||
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
||||||
# it is stated on each publishing job rather than inferred from a `needs`
|
# it is stated on each publishing job rather than inferred from a `needs`
|
||||||
@@ -534,8 +645,8 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
# Not the triggering ref — see the `env:` block at the top. On a
|
# Not the triggering ref — see the `env:` block at the top. On a
|
||||||
# scheduled refresh this is `main`; on everything else it is the ref
|
# scheduled refresh this is `main`; on everything else it is the
|
||||||
# that fired, so this is a no-op on every ordinary path.
|
# commit that fired, the same one the lanes above tested.
|
||||||
ref: ${{ env.BUILD_REF }}
|
ref: ${{ env.BUILD_REF }}
|
||||||
# Full history is load-bearing, not a convenience: the version this
|
# Full history is load-bearing, not a convenience: the version this
|
||||||
# job signs is derived from the commit TIME of the newest packaged
|
# job signs is derived from the commit TIME of the newest packaged
|
||||||
@@ -676,6 +787,15 @@ jobs:
|
|||||||
print("guard: no ext-* release yet — nothing to go backwards from")
|
print("guard: no ext-* release yet — nothing to go backwards from")
|
||||||
raise SystemExit(0)
|
raise SystemExit(0)
|
||||||
|
|
||||||
|
# A version that is already signed is reused from its release, not
|
||||||
|
# signed again, so it cannot be a downgrade. The weekly refresh builds
|
||||||
|
# main, whose version trails whatever dev has signed since; refusing
|
||||||
|
# it here would stop :latest from ever being refreshed while dev is
|
||||||
|
# ahead.
|
||||||
|
if any(v == derived for v, _ in signed):
|
||||||
|
print(f"guard: {derived_s} is already signed — reused, not re-signed")
|
||||||
|
raise SystemExit(0)
|
||||||
|
|
||||||
hi, hi_tag = signed[-1]
|
hi, hi_tag = signed[-1]
|
||||||
print(f"guard: derived={derived_s} highest already signed={hi_tag}")
|
print(f"guard: derived={derived_s} highest already signed={hi_tag}")
|
||||||
if derived < hi:
|
if derived < hi:
|
||||||
@@ -850,13 +970,16 @@ jobs:
|
|||||||
# reuse hit — deliberately NOT folded into `digest`, which the :c-<sha>
|
# reuse hit — deliberately NOT folded into `digest`, which the :c-<sha>
|
||||||
# repoint reads and which must keep meaning "what this run built" (#4290).
|
# repoint reads and which must keep meaning "what this run built" (#4290).
|
||||||
published_digest: ${{ steps.reuse.outputs.published_digest }}
|
published_digest: ${{ steps.reuse.outputs.published_digest }}
|
||||||
|
# Every tag this commit should end up under (channel, plus :c-<sha> on
|
||||||
|
# main). `promote` writes them from `digest` once the smoke has passed.
|
||||||
|
tags: ${{ steps.tag.outputs.tags }}
|
||||||
# A plain `needs` — no `always()`. That expression existed to let a
|
# A plain `needs` — no `always()`. That expression existed to let a
|
||||||
# SKIPPED sign-extension through on a tag push while still blocking a
|
# SKIPPED sign-extension through on a tag push while still blocking a
|
||||||
# FAILED one. With no tag trigger, sign-extension always runs, so the
|
# FAILED one. With no tag trigger, sign-extension always runs, so the
|
||||||
# default behaviour is exactly what we want: a failed sign skips build-web
|
# default behaviour is exactly what we want: a failed sign skips build-web
|
||||||
# rather than shipping an image without its XPI.
|
# rather than shipping an image without its XPI.
|
||||||
needs: [sign-extension, lint, extension-version, backend-lint-and-test,
|
needs: [sign-extension, lint, extension-version, backend-lint-and-test,
|
||||||
frontend-build, integration]
|
frontend-build, extension-test, integration]
|
||||||
# The lanes in that list are THE GATE (2026-09-23) — see sign-extension's
|
# The lanes in that list are THE GATE (2026-09-23) — see sign-extension's
|
||||||
# copy of this comment for why, including the property that a SKIPPED lane
|
# copy of this comment for why, including the property that a SKIPPED lane
|
||||||
# blocks as firmly as a failing one. They are repeated here rather than
|
# blocks as firmly as a failing one. They are repeated here rather than
|
||||||
@@ -871,8 +994,8 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
# Not the triggering ref — see the `env:` block at the top. On a
|
# Not the triggering ref — see the `env:` block at the top. On a
|
||||||
# scheduled refresh this is `main`; on everything else it is the ref
|
# scheduled refresh this is `main`; on everything else it is the
|
||||||
# that fired, so this is a no-op on every ordinary path.
|
# commit that fired, the same one the lanes above tested.
|
||||||
ref: ${{ env.BUILD_REF }}
|
ref: ${{ env.BUILD_REF }}
|
||||||
# Full history: this job RE-DERIVES the extension version rather than
|
# Full history: this job RE-DERIVES the extension version rather than
|
||||||
# being handed it, and a depth-1 clone derives a wrong, too-low value
|
# being handed it, and a depth-1 clone derives a wrong, too-low value
|
||||||
@@ -1108,11 +1231,13 @@ jobs:
|
|||||||
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||||
# whether the channel then has to be written separately.
|
# whether the channel then has to be written separately.
|
||||||
#
|
#
|
||||||
# On a push the build writes the channel tag directly: the bytes came
|
# Every build writes a CANDIDATE tag, never the channel (#4310):
|
||||||
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
# :dev-candidate / :latest-candidate on a push, :refresh-candidate on
|
||||||
# it behind.
|
# the refresh. `promote` moves the channel only after smoke-web has
|
||||||
|
# booted the bytes. The lanes prove the SOURCE; only the smoke proves
|
||||||
|
# the BYTES, and the two are not the same claim.
|
||||||
#
|
#
|
||||||
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
# The refresh is the case that made this obvious. A
|
||||||
# refresh rebuilds against freshly resolved base images, and the web
|
# refresh rebuilds against freshly resolved base images, and the web
|
||||||
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||||
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||||
@@ -1142,7 +1267,10 @@ jobs:
|
|||||||
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||||
echo "promote=true" >> "$GITHUB_OUTPUT"
|
echo "promote=true" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
# A push builds to a per-channel candidate too (#4310). The channel
|
||||||
|
# tag moves in `promote`, after smoke-web has booted these bytes —
|
||||||
|
# so a broken image never reaches the tag deployments follow.
|
||||||
|
echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
|
||||||
echo "promote=false" >> "$GITHUB_OUTPUT"
|
echo "promote=false" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
@@ -1418,6 +1546,16 @@ jobs:
|
|||||||
TAGS: ${{ steps.tag.outputs.tags }}
|
TAGS: ${{ steps.tag.outputs.tags }}
|
||||||
run: |
|
run: |
|
||||||
set -euf
|
set -euf
|
||||||
|
# A BUILD publishes nothing from here (#4310). Its bytes sit on the
|
||||||
|
# candidate tag until smoke-web has booted them; `promote` then
|
||||||
|
# writes the channel tag AND :c-<sha> from this run's digest. Writing
|
||||||
|
# :c-<sha> here would publish an immutable rollback tag for bytes
|
||||||
|
# that might then fail the smoke.
|
||||||
|
if [ -n "${BUILT_DIGEST:-}" ]; then
|
||||||
|
echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
|
||||||
|
echo "repoint: after the smoke; nothing to write here."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
||||||
#
|
#
|
||||||
# This step used to copy from the channel tag by NAME. Nothing
|
# This step used to copy from the channel tag by NAME. Nothing
|
||||||
@@ -1511,14 +1649,9 @@ jobs:
|
|||||||
# source tree, and not a static inspection: `ffmpeg -version` exiting 0 would
|
# source tree, and not a static inspection: `ffmpeg -version` exiting 0 would
|
||||||
# pass while a codec removal broke every thumbnail in the library.
|
# pass while a codec removal broke every thumbnail in the library.
|
||||||
#
|
#
|
||||||
# Refresh-only. On a push the bytes came from a commit, and a commit is what
|
# It gates `promote` on every trigger (#4310). The lanes above prove the
|
||||||
# the lanes above already test — and since 2026-09-23 they gate the build, so
|
# source; they cannot see the bytes, and a Dockerfile or base change breaks
|
||||||
# those bytes could not exist without them having passed.
|
# the bytes without touching the source.
|
||||||
#
|
|
||||||
# Reports a verdict; it does not yet gate the promote (milestone 362 step 4).
|
|
||||||
# Landing the gate and the thing it gates in one change would mean the first
|
|
||||||
# time anyone saw this job run would also be the first time it could stop a
|
|
||||||
# publish.
|
|
||||||
smoke-web:
|
smoke-web:
|
||||||
needs: [build-web]
|
needs: [build-web]
|
||||||
# Every run that actually BUILT something, not just the weekly refresh.
|
# Every run that actually BUILT something, not just the weekly refresh.
|
||||||
@@ -1530,13 +1663,10 @@ jobs:
|
|||||||
# and were smoked when they were built. Re-smoking them would burn two
|
# and were smoked when they were built. Re-smoking them would burn two
|
||||||
# minutes to re-learn a fact.
|
# minutes to re-learn a fact.
|
||||||
#
|
#
|
||||||
# HONEST LIMIT, and it is the reason #4299 exists: on a push this runs
|
# This GATES the publish on every trigger (#4310): builds land on a
|
||||||
# AFTER build-web has written the channel tag, so it detects rather than
|
# candidate tag, and `promote` needs this job, so :dev / :latest and a
|
||||||
# gates. Rule 164's verify_with asks for the check BETWEEN build and push.
|
# built :c-<sha> move only after it passes. Rule 164's verify_with — the
|
||||||
# Closing that needs the push path to adopt the candidate-then-promote
|
# check BETWEEN build and push.
|
||||||
# shape the refresh already has — per-channel candidate tags, promote
|
|
||||||
# learning its channel, and the :c-<sha> repoint moving after the gate.
|
|
||||||
# That is a redesign of the production publish path and is its own task.
|
|
||||||
#
|
#
|
||||||
# NO `if:` — this job always runs (#4323). It used to be gated on the
|
# NO `if:` — this job always runs (#4323). It used to be gated on the
|
||||||
# build having published something, which skipped it on a reuse hit. That
|
# build having published something, which skipped it on a reuse hit. That
|
||||||
@@ -1583,7 +1713,7 @@ jobs:
|
|||||||
--health-timeout 5s
|
--health-timeout 5s
|
||||||
--health-retries 10
|
--health-retries 10
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:8-alpine
|
||||||
options: >-
|
options: >-
|
||||||
--health-cmd "redis-cli ping"
|
--health-cmd "redis-cli ping"
|
||||||
--health-interval 10s
|
--health-interval 10s
|
||||||
@@ -1608,9 +1738,20 @@ jobs:
|
|||||||
# Service discovery mirrors the integration lane above: these jobs run
|
# Service discovery mirrors the integration lane above: these jobs run
|
||||||
# in a container against a mounted docker socket, so the services are
|
# in a container against a mounted docker socket, so the services are
|
||||||
# SIBLINGS reachable by IP, not by hostname.
|
# SIBLINGS reachable by IP, not by hostname.
|
||||||
PG=$(docker ps --filter "name=smoke" --filter "ancestor=pgvector/pgvector:pg16" -q | head -n1)
|
# Only THIS job's services. The runner's docker daemon is shared, so another
|
||||||
RD=$(docker ps --filter "name=smoke" --filter "ancestor=redis:7-alpine" -q | head -n1)
|
# repo's job can be running beside this one, and a job-name filter matches its
|
||||||
test -n "$PG" && test -n "$RD"
|
# containers too (Steward run 8358 and Inkwell run 8653 each took another repo's
|
||||||
|
# Postgres). act names every container of a task GITEA-ACTIONS-TASK-<n>-...: read
|
||||||
|
# <n> from our own container (its id is in the /etc/hostname bind mount) and
|
||||||
|
# require exactly one match.
|
||||||
|
SELF=$(grep -o '/containers/[0-9a-f]\{64\}/' /proc/self/mountinfo | head -n1 | cut -d/ -f3 || true)
|
||||||
|
SELF=${SELF:-$(hostname)}
|
||||||
|
TASK=$(docker inspect -f '{{.Name}}' "$SELF" | grep -o 'GITEA-ACTIONS-TASK-[0-9]*-')
|
||||||
|
test -n "$TASK"
|
||||||
|
PG=$(docker ps --filter "name=$TASK" --filter "ancestor=pgvector/pgvector:pg16" -q)
|
||||||
|
test "$(printf '%s\n' "$PG" | grep -c .)" -eq 1
|
||||||
|
RD=$(docker ps --filter "name=$TASK" --filter "ancestor=redis:8-alpine" -q)
|
||||||
|
test "$(printf '%s\n' "$RD" | grep -c .)" -eq 1
|
||||||
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
PG_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$PG")
|
||||||
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
RD_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$RD")
|
||||||
test -n "$PG_IP" && test -n "$RD_IP"
|
test -n "$PG_IP" && test -n "$RD_IP"
|
||||||
@@ -1954,88 +2095,113 @@ jobs:
|
|||||||
# smoked it, and it should not be read as if it were.
|
# smoked it, and it should not be read as if it were.
|
||||||
promote:
|
promote:
|
||||||
needs: [build-web, build-agent, smoke-web]
|
needs: [build-web, build-agent, smoke-web]
|
||||||
# Only a refresh publishes through a candidate; a push writes its channel
|
# THE PUBLISH (#4310). Both image jobs build to a candidate tag — a push to
|
||||||
# tag directly from the build. Reads the same reuse-step decision the build
|
# :dev-candidate / :latest-candidate, the weekly refresh to
|
||||||
# took, via a job output — a job's `if:` cannot see the `env` context.
|
# :refresh-candidate — and nothing names those bytes under a tag anybody
|
||||||
if: needs.build-web.outputs.candidate == 'true'
|
# deploys until this job runs. It runs only when every job in `needs`
|
||||||
|
# SUCCEEDED (a job-level `if:` without a status function implies
|
||||||
|
# success()), so a failed smoke — or a skipped one, which is not the same
|
||||||
|
# as a passing one (run 5290) — leaves :dev / :latest on the last build
|
||||||
|
# that worked. Rule 164's verify_with: the check sits BETWEEN build and
|
||||||
|
# push.
|
||||||
|
#
|
||||||
|
# Per image, from the DIGEST that image's job built (#4290): a tag can
|
||||||
|
# move between the build and this job, a digest cannot. An image whose job
|
||||||
|
# hit reuse built nothing and has nothing to promote — its build job
|
||||||
|
# already wrote its :c-<sha> from the channel tag, which "hit" proved
|
||||||
|
# carries this commit.
|
||||||
|
if: github.event_name != 'pull_request'
|
||||||
runs-on: python-ci
|
runs-on: python-ci
|
||||||
container:
|
container:
|
||||||
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
image: git.fabledsword.com/bvandeusen/ci-python:3.14
|
||||||
steps:
|
steps:
|
||||||
- name: Point the channel tags at the smoked candidates
|
- name: Point this commit's tags at the smoked builds
|
||||||
env:
|
env:
|
||||||
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
TOKEN: ${{ secrets.RELEASE_TOKEN }}
|
||||||
ACTOR: ${{ github.actor }}
|
ACTOR: ${{ github.actor }}
|
||||||
|
WEB_DIGEST: ${{ needs.build-web.outputs.digest }}
|
||||||
|
WEB_TAGS: ${{ needs.build-web.outputs.tags }}
|
||||||
|
AGENT_DIGEST: ${{ needs.build-agent.outputs.digest }}
|
||||||
|
AGENT_TAGS: ${{ needs.build-agent.outputs.tags }}
|
||||||
run: |
|
run: |
|
||||||
set -eu
|
set -eu
|
||||||
# `latest` is not a guess: a refresh always builds `main` (BUILD_REF),
|
|
||||||
# and the "must have checked out main" guard in every build job fails
|
|
||||||
# the run if that did not hold. So the channel is main's.
|
|
||||||
TAG=latest
|
|
||||||
FAILED=""
|
FAILED=""
|
||||||
|
MOVED=0
|
||||||
|
|
||||||
for NAME in fabledcurator fabledcurator-agent; do
|
promote() {
|
||||||
|
NAME="$1"; DIGEST="$2"; TAGS="$3"
|
||||||
REPO="bvandeusen/$NAME"
|
REPO="bvandeusen/$NAME"
|
||||||
echo "promote: $REPO"
|
if [ -z "$DIGEST" ]; then
|
||||||
|
echo "promote: $NAME — no build this run (reuse hit); nothing to publish"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
echo "promote: $NAME $DIGEST"
|
||||||
|
|
||||||
# Registry auth is its own token exchange — `docker login`
|
# Registry auth is its own token exchange — `docker login`
|
||||||
# authenticates the docker client, not curl. Deadline on every call
|
# authenticates the docker client, not curl. Deadline on every call
|
||||||
# (rule 156): a registry that stops answering must fail this step,
|
# (rule 156): a registry that stops answering must fail this step,
|
||||||
# not hang the weekly refresh until the job times out.
|
# not hang the run until the job times out.
|
||||||
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
|
BEARER=$(curl -fsS --max-time 30 -u "$ACTOR:$TOKEN" \
|
||||||
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
|
"https://git.fabledsword.com/v2/token?scope=repository:$REPO:pull,push&service=git.fabledsword.com" \
|
||||||
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
|
| python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])')
|
||||||
|
|
||||||
# Ask for the IMAGE manifest media types only. Offering the index
|
# Ask for the IMAGE manifest media types only. Offering the index
|
||||||
# types too would let the registry hand back an index if one ever
|
# types too would let the registry hand back an index, and we would
|
||||||
# existed at this tag, and we would faithfully copy the thing this
|
# faithfully copy the thing this approach exists to avoid creating.
|
||||||
# whole approach exists to avoid creating.
|
|
||||||
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
|
ACCEPT='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json'
|
||||||
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
|
CT=$(curl -fsS --max-time 60 -o manifest.json -D headers.txt \
|
||||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||||
"https://git.fabledsword.com/v2/$REPO/manifests/refresh-candidate" \
|
"https://git.fabledsword.com/v2/$REPO/manifests/$DIGEST" \
|
||||||
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
|
&& tr -d '\r' < headers.txt | awk -F': ' '/^[Cc]ontent-[Tt]ype:/{print $2}')
|
||||||
test -n "$CT"
|
test -n "$CT"
|
||||||
SRC=$(tr -d '\r' < headers.txt | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
|
||||||
echo "promote: candidate $SRC ($CT)"
|
|
||||||
|
|
||||||
# NOT `imagetools create`. That wraps its source in an INDEX, and
|
# NOT `imagetools create`. That wraps its source in an INDEX, and
|
||||||
# `.Image.Config.Labels` does not resolve through one — the
|
# `.Image.Config.Labels` does not resolve through one — the
|
||||||
# fc.revision the reuse check reads off the channel tag would come
|
# fc.revision the reuse check reads off the channel tag would come
|
||||||
# back empty, every later push would miss and rebuild, and nothing
|
# back empty, every later push would miss and rebuild, and nothing
|
||||||
# would go red (#3183, run 4751). A manifest PUT is what "make this
|
# would go red (#3183, run 4751). A manifest PUT is what "make this
|
||||||
# tag name that image" means at the registry: same bytes, same media
|
# tag name that image" means at the registry: same bytes, same
|
||||||
# type, same digest, no layer transfer.
|
# media type, same digest, no layer transfer. (It also makes a
|
||||||
curl -fsS --max-time 120 -X PUT \
|
# built :c-<sha> a plain image rather than the index the old
|
||||||
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
|
# repoint left.)
|
||||||
--data-binary @manifest.json \
|
IFS=,
|
||||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
|
for REF in $TAGS; do
|
||||||
|
TAG="${REF##*:}"
|
||||||
|
curl -fsS --max-time 120 -X PUT \
|
||||||
|
-H "Authorization: Bearer $BEARER" -H "Content-Type: $CT" \
|
||||||
|
--data-binary @manifest.json \
|
||||||
|
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG"
|
||||||
|
|
||||||
# Read it back. A PUT that returned 2xx but landed something else is
|
# Read it back. A PUT that returned 2xx but landed something else
|
||||||
# exactly the silent-and-plausible failure this pipeline keeps
|
# is exactly the silent-and-plausible failure this pipeline keeps
|
||||||
# producing, and the check costs one request.
|
# producing, and the check costs one request.
|
||||||
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
|
NOW=$(curl -fsS --max-time 30 -o /dev/null -D - \
|
||||||
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
-H "Authorization: Bearer $BEARER" -H "Accept: $ACCEPT" \
|
||||||
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
|
"https://git.fabledsword.com/v2/$REPO/manifests/$TAG" \
|
||||||
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
| tr -d '\r' | awk -F': ' '/^[Dd]ocker-[Cc]ontent-[Dd]igest:/{print $2}')
|
||||||
if [ "$NOW" != "$SRC" ]; then
|
if [ "$NOW" != "$DIGEST" ]; then
|
||||||
echo "promote: FAILED — $NAME:$TAG is $NOW, expected $SRC" >&2
|
echo "promote: FAILED — $NAME:$TAG is $NOW, expected $DIGEST" >&2
|
||||||
FAILED="$FAILED $NAME"
|
FAILED="$FAILED $NAME:$TAG"
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
echo "promote: $NAME:$TAG now names $NOW"
|
echo "promote: $NAME:$TAG now names $NOW"
|
||||||
done
|
MOVED=$((MOVED + 1))
|
||||||
|
done
|
||||||
|
unset IFS
|
||||||
|
}
|
||||||
|
|
||||||
|
promote fabledcurator "$WEB_DIGEST" "$WEB_TAGS"
|
||||||
|
promote fabledcurator-agent "$AGENT_DIGEST" "$AGENT_TAGS"
|
||||||
|
|
||||||
if [ -n "$FAILED" ]; then
|
if [ -n "$FAILED" ]; then
|
||||||
echo "" >&2
|
echo "" >&2
|
||||||
echo "promote: FAILED for:$FAILED" >&2
|
echo "promote: FAILED for:$FAILED" >&2
|
||||||
echo "promote: the channel tags are now INCONSISTENT — some images" >&2
|
echo "promote: the tags are now INCONSISTENT — some moved and some" >&2
|
||||||
echo "promote: moved and some did not. Re-run this refresh; the" >&2
|
echo "promote: did not. Re-run this workflow; the builds are still" >&2
|
||||||
echo "promote: candidates are still published and the promote is" >&2
|
echo "promote: published by digest and the promote is idempotent." >&2
|
||||||
echo "promote: idempotent." >&2
|
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
echo "promote: both channel tags moved"
|
echo "promote: $MOVED tag(s) written"
|
||||||
build-agent:
|
build-agent:
|
||||||
# THE GATE (2026-09-23). Every lane above must have PASSED before this job
|
# THE GATE (2026-09-23). Every lane above must have PASSED before this job
|
||||||
# exists at all — so a red suite does not produce an image, let alone push
|
# exists at all — so a red suite does not produce an image, let alone push
|
||||||
@@ -2048,7 +2214,14 @@ jobs:
|
|||||||
# itself through a job-level `if:` that could not read `env`, and a design
|
# itself through a job-level `if:` that could not read `env`, and a design
|
||||||
# where only a FAILED gate blocks would have published unverified images
|
# where only a FAILED gate blocks would have published unverified images
|
||||||
# while reporting success.
|
# while reporting success.
|
||||||
needs: [lint, extension-version, backend-lint-and-test, frontend-build, integration]
|
needs: [lint, extension-version, backend-lint-and-test, frontend-build, extension-test,
|
||||||
|
integration]
|
||||||
|
# What `promote` needs to publish this image once the smoke has passed:
|
||||||
|
# the manifest this run built (empty on a reuse hit) and every tag it
|
||||||
|
# belongs under. Same meaning as build-web's outputs of the same names.
|
||||||
|
outputs:
|
||||||
|
digest: ${{ steps.build.outputs.digest }}
|
||||||
|
tags: ${{ steps.tag.outputs.tags }}
|
||||||
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
# A pull_request run is the lanes and nothing else. This is the ONLY thing
|
||||||
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
# separating "validate a Renovate bump" from "publish a Renovate bump", so
|
||||||
# it is stated on each publishing job rather than inferred from a `needs`
|
# it is stated on each publishing job rather than inferred from a `needs`
|
||||||
@@ -2061,8 +2234,8 @@ jobs:
|
|||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
with:
|
with:
|
||||||
# Not the triggering ref — see the `env:` block at the top. On a
|
# Not the triggering ref — see the `env:` block at the top. On a
|
||||||
# scheduled refresh this is `main`; on everything else it is the ref
|
# scheduled refresh this is `main`; on everything else it is the
|
||||||
# that fired, so this is a no-op on every ordinary path.
|
# commit that fired, the same one the lanes above tested.
|
||||||
ref: ${{ env.BUILD_REF }}
|
ref: ${{ env.BUILD_REF }}
|
||||||
# Full history: this job derives its artifact's version from the
|
# Full history: this job derives its artifact's version from the
|
||||||
# commit its shipped files last changed in (milestone 313). A
|
# commit its shipped files last changed in (milestone 313). A
|
||||||
@@ -2240,11 +2413,13 @@ jobs:
|
|||||||
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
# WHERE THE BUILD PUBLISHES, which is not always the channel — and
|
||||||
# whether the channel then has to be written separately.
|
# whether the channel then has to be written separately.
|
||||||
#
|
#
|
||||||
# On a push the build writes the channel tag directly: the bytes came
|
# Every build writes a CANDIDATE tag, never the channel (#4310):
|
||||||
# from a commit, and a commit is the thing CI tests. Nothing to hold
|
# :dev-candidate / :latest-candidate on a push, :refresh-candidate on
|
||||||
# it behind.
|
# the refresh. `promote` moves the channel only after smoke-web has
|
||||||
|
# booted the bytes. The lanes prove the SOURCE; only the smoke proves
|
||||||
|
# the BYTES, and the two are not the same claim.
|
||||||
#
|
#
|
||||||
# On the scheduled refresh it writes a CANDIDATE tag instead. A
|
# The refresh is the case that made this obvious. A
|
||||||
# refresh rebuilds against freshly resolved base images, and the web
|
# refresh rebuilds against freshly resolved base images, and the web
|
||||||
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
# image's runtime is a line of UNPINNED Debian packages (ffmpeg,
|
||||||
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
# libjpeg62-turbo, libpq5, megatools…) re-resolved on every build.
|
||||||
@@ -2273,7 +2448,10 @@ jobs:
|
|||||||
if [ "${IS_REFRESH:-}" = "true" ]; then
|
if [ "${IS_REFRESH:-}" = "true" ]; then
|
||||||
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
echo "build_ref=$IMAGE:refresh-candidate" >> "$GITHUB_OUTPUT"
|
||||||
else
|
else
|
||||||
echo "build_ref=$IMAGE:$T" >> "$GITHUB_OUTPUT"
|
# A push builds to a per-channel candidate too (#4310). The channel
|
||||||
|
# tag moves in `promote`, after smoke-web has booted these bytes —
|
||||||
|
# so a broken image never reaches the tag deployments follow.
|
||||||
|
echo "build_ref=$IMAGE:$T-candidate" >> "$GITHUB_OUTPUT"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
# Compare VALUES, never exit codes. Measured on buildx v0.36.1
|
||||||
@@ -2464,6 +2642,16 @@ jobs:
|
|||||||
TAGS: ${{ steps.tag.outputs.tags }}
|
TAGS: ${{ steps.tag.outputs.tags }}
|
||||||
run: |
|
run: |
|
||||||
set -euf
|
set -euf
|
||||||
|
# A BUILD publishes nothing from here (#4310). Its bytes sit on the
|
||||||
|
# candidate tag until smoke-web has booted them; `promote` then
|
||||||
|
# writes the channel tag AND :c-<sha> from this run's digest. Writing
|
||||||
|
# :c-<sha> here would publish an immutable rollback tag for bytes
|
||||||
|
# that might then fail the smoke.
|
||||||
|
if [ -n "${BUILT_DIGEST:-}" ]; then
|
||||||
|
echo "repoint: built $BUILT_DIGEST this run — promote publishes it"
|
||||||
|
echo "repoint: after the smoke; nothing to write here."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
# WHAT WE COPY FROM, which is not what we EXCLUDE (#4290).
|
||||||
#
|
#
|
||||||
# This step used to copy from the channel tag by NAME. Nothing
|
# This step used to copy from the channel tag by NAME. Nothing
|
||||||
|
|||||||
@@ -1,86 +0,0 @@
|
|||||||
name: extension
|
|
||||||
# Lint + unit tests. Deliberately NOT a publishing lane, which is why it is
|
|
||||||
# not part of build.yml's gate. The sign-and-publish dance moved into build.yml's
|
|
||||||
# `sign-extension` job (2026-05-25) — `:latest` now always bundles the XPI
|
|
||||||
# because sign-extension runs as a build-web dependency in the SAME workflow,
|
|
||||||
# eliminating the prior race between build.yml and a separate extension.yml.
|
|
||||||
# Signed XPIs are cached in Forgejo Release Assets named `ext-<version>`.
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [dev, main]
|
|
||||||
paths:
|
|
||||||
- 'extension/**'
|
|
||||||
- '.forgejo/workflows/extension.yml'
|
|
||||||
# test/version.spec.js asserts things ABOUT build.yml — that it does not
|
|
||||||
# inline the packaged-file set, and that build.yml derives
|
|
||||||
# the shipped version rather than reading it out of the repo. A
|
|
||||||
# workflow-only edit can therefore break this suite, so it has to trigger
|
|
||||||
# it. build.yml joined the list at milestone 271 step 5, when the spec
|
|
||||||
# started asserting against it.
|
|
||||||
- '.forgejo/workflows/build.yml'
|
|
||||||
pull_request:
|
|
||||||
branches: [main]
|
|
||||||
paths:
|
|
||||||
- 'extension/**'
|
|
||||||
- '.forgejo/workflows/build.yml'
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
lint:
|
|
||||||
runs-on: python-ci
|
|
||||||
container:
|
|
||||||
image: node:24-bookworm-slim
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@v4
|
|
||||||
# Not --no-save: vitest and web-ext are both real devDependencies now,
|
|
||||||
# and the suite needs vitest resolvable from node_modules.
|
|
||||||
- name: Install dev dependencies
|
|
||||||
run: cd extension && npm install --no-audit --no-fund
|
|
||||||
- name: Lint
|
|
||||||
run: cd extension && npm run lint
|
|
||||||
# Pure-logic specs over lib/url.js and lib/platforms.js plus manifest /
|
|
||||||
# package version-consistency checks. No browser, no network.
|
|
||||||
- name: Unit tests
|
|
||||||
run: cd extension && npm run test:unit
|
|
||||||
|
|
||||||
# Everything else about packaging is asserted against our own declaration
|
|
||||||
# of what ships. This is the only check that asks web-ext what it ACTUALLY
|
|
||||||
# put in the archive. Until now that was an unverified assumption about
|
|
||||||
# glob semantics — and a fragile one: `test/**` reaches web-ext intact
|
|
||||||
# only because callers `set -f` first, so losing that quoting would
|
|
||||||
# silently start shipping dev files with no other signal.
|
|
||||||
- name: Verify XPI contents
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
command -v unzip >/dev/null 2>&1 || { apt-get update -qq && apt-get install -y -qq unzip; }
|
|
||||||
cd extension
|
|
||||||
npm run build
|
|
||||||
ZIP=$(ls web-ext-artifacts/*.zip | head -1)
|
|
||||||
echo "=== packaged entries in $ZIP ==="
|
|
||||||
unzip -Z1 "$ZIP" | sort
|
|
||||||
echo "=== end ==="
|
|
||||||
ENTRIES=$(unzip -Z1 "$ZIP")
|
|
||||||
fail=0
|
|
||||||
# Must NOT ship: repo infrastructure with no business in a user's browser.
|
|
||||||
for pat in 'test/' 'scripts/' 'vitest.config.js' 'package.json' 'package-lock.json' 'README.md' 'node_modules/' 'web-ext-artifacts/'; do
|
|
||||||
if echo "$ENTRIES" | grep -q "^$pat"; then
|
|
||||||
echo "ERROR: '$pat' was packaged into the XPI but must not be"
|
|
||||||
fail=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
# Must ship: if an exclusion pattern ever over-matches, the extension
|
|
||||||
# breaks at runtime rather than at build time, so assert presence too.
|
|
||||||
for req in 'manifest.json' 'lib/url.js' 'lib/api.js' 'lib/platforms.js' 'lib/cookies.js'; do
|
|
||||||
if ! echo "$ENTRIES" | grep -q "^$req$"; then
|
|
||||||
echo "ERROR: '$req' is missing from the XPI"
|
|
||||||
fail=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
for dir in 'background/' 'popup/' 'options/' 'content/' 'icons/'; do
|
|
||||||
if ! echo "$ENTRIES" | grep -q "^$dir"; then
|
|
||||||
echo "ERROR: nothing from '$dir' was packaged"
|
|
||||||
fail=1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
[ "$fail" -eq 0 ] || exit 1
|
|
||||||
echo "XPI contents verified."
|
|
||||||
+3
-7
@@ -36,11 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
libwebp7 \
|
libwebp7 \
|
||||||
libpng16-16 \
|
libpng16-16 \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
# opencv-python-headless (via requirements-ml.txt) links these even in its
|
|
||||||
# headless build. Came from Dockerfile.ml when the images merged
|
|
||||||
# (milestone 422 step 6).
|
|
||||||
libgl1 \
|
|
||||||
libglib2.0-0 \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
@@ -61,7 +56,8 @@ RUN pip install -r requirements.txt
|
|||||||
#
|
#
|
||||||
# torch 2.12.1+cpu wheel 192.3 MB
|
# torch 2.12.1+cpu wheel 192.3 MB
|
||||||
# torchvision 0.27.1+cpu 1.8 MB
|
# torchvision 0.27.1+cpu 1.8 MB
|
||||||
# transformers / onnxruntime / opencv / sklearn and friends
|
# transformers / onnxruntime / opencv / sklearn and friends (opencv and
|
||||||
|
# onnxruntime since dropped, #1451 — nothing here imported them)
|
||||||
# 62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB
|
# 62.0, 35.3, 23.6, 16.7, 12.3, 9.2, 6.9 MB
|
||||||
# largest newly-pushed layer 222.07 MB
|
# largest newly-pushed layer 222.07 MB
|
||||||
#
|
#
|
||||||
@@ -85,7 +81,7 @@ RUN pip install -r requirements.txt
|
|||||||
# CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the
|
# CPU-only torch from the PyTorch CPU index. Nothing here uses a GPU — the
|
||||||
# GPU agent is a separate service with its own image.
|
# GPU agent is a separate service with its own image.
|
||||||
RUN pip install --index-url https://download.pytorch.org/whl/cpu \
|
RUN pip install --index-url https://download.pytorch.org/whl/cpu \
|
||||||
"torch>=2.12,<3.0" "torchvision>=0.27,<0.28"
|
"torch>=2.14" "torchvision>=0.29"
|
||||||
RUN pip install -r requirements-ml.txt
|
RUN pip install -r requirements-ml.txt
|
||||||
|
|
||||||
# Where the model lands. Deliberately NOT a VOLUME instruction: that mints an
|
# Where the model lands. Deliberately NOT a VOLUME instruction: that mints an
|
||||||
|
|||||||
@@ -22,7 +22,8 @@ through afterwards.
|
|||||||
- **ML tagging.** Runs image models in-container to suggest tags, group
|
- **ML tagging.** Runs image models in-container to suggest tags, group
|
||||||
characters, find near-duplicates and power similarity search. Suggestions are
|
characters, find near-duplicates and power similarity search. Suggestions are
|
||||||
reviewable — it proposes, you confirm, and it learns which proposals you keep
|
reviewable — it proposes, you confirm, and it learns which proposals you keep
|
||||||
rejecting.
|
rejecting. It ships switched off: turn it on under Settings → System when
|
||||||
|
you want it, and it fetches its model weights then.
|
||||||
- **Deduplication and provenance.** Everything that arrives is hashed and
|
- **Deduplication and provenance.** Everything that arrives is hashed and
|
||||||
deduplicated by content, metadata sidecars are read wherever the source
|
deduplicated by content, metadata sidecars are read wherever the source
|
||||||
writes them, and every file keeps a record of where it came from.
|
writes them, and every file keeps a record of where it came from.
|
||||||
@@ -119,9 +120,12 @@ needs every credential entered again by hand.
|
|||||||
|
|
||||||
A few other things are worth knowing about the first few minutes:
|
A few other things are worth knowing about the first few minutes:
|
||||||
|
|
||||||
- **The ML worker downloads its model weights on first boot**, several GB from
|
- **ML tagging starts switched off, and nothing is downloaded at boot.** Give
|
||||||
HuggingFace into `./models`. Until that finishes, tagging is queued rather
|
the ML lane a slot under **Settings → System** and it fetches its model
|
||||||
than broken. It is idempotent — a restart resumes rather than refetches.
|
weights then — several GB from HuggingFace into `./models`, shown as a job
|
||||||
|
under **Settings → Activity** that you can watch and retry. Until it
|
||||||
|
finishes, tagging is queued rather than broken, and the fetch only takes
|
||||||
|
what is missing, so turning the lane off and on again does not refetch.
|
||||||
- **The gallery starts empty**, and that is the expected state. Add a creator
|
- **The gallery starts empty**, and that is the expected state. Add a creator
|
||||||
under **Subscriptions** and it fills as posts come down.
|
under **Subscriptions** and it fills as posts come down.
|
||||||
- **If you already have a library on disk**, there is no screen that imports
|
- **If you already have a library on disk**, there is no screen that imports
|
||||||
@@ -271,11 +275,12 @@ Four deployable pieces, built by `.forgejo/workflows/build.yml`:
|
|||||||
|
|
||||||
## CI / Forgejo setup
|
## CI / Forgejo setup
|
||||||
|
|
||||||
Three workflows: `build.yml` (the five verification lanes — lint,
|
Two workflows that matter here: `build.yml` (the six verification lanes — lint,
|
||||||
extension-version check, backend unit tests, frontend build, integration — and
|
extension-version check, backend unit tests, frontend build, extension lint +
|
||||||
then sign + publish), `extension.yml` (extension lint, vitest, XPI content
|
vitest + XPI content check, integration — and then sign + publish), and
|
||||||
verification), and `release.yml`, which runs only on a `v*` tag and publishes a
|
`release.yml`, which runs only on a `v*` tag and publishes a changelog without
|
||||||
changelog without building anything.
|
building anything. The extension lane was its own `extension.yml` until
|
||||||
|
milestone 429, which let a red extension suite sign and ship the XPI anyway.
|
||||||
|
|
||||||
**The lanes and the publish are one workflow on purpose.** They were two
|
**The lanes and the publish are one workflow on purpose.** They were two
|
||||||
(`ci.yml` and `build.yml`) until 2026-09-23, on the same push trigger, which
|
(`ci.yml` and `build.yml`) until 2026-09-23, on the same push trigger, which
|
||||||
|
|||||||
+23
-10
@@ -1,10 +1,21 @@
|
|||||||
# FabledCurator GPU agent — runs on the desktop with the GPU.
|
# FabledCurator GPU agent — runs on the desktop with the GPU.
|
||||||
# CUDA 12.9 + cuDNN 9 runtime so onnxruntime-gpu can use the card (it needs
|
#
|
||||||
# cuDNN 9 — the plain -runtime image lacks it: "libcudnn.so.9: cannot open
|
# The `base` flavour, not `cudnn-runtime`: CUDA and cuDNN arrive as the
|
||||||
# shared object file"); ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
|
# `nvidia-*` pip packages torch and onnxruntime-gpu depend on, so the base only
|
||||||
# Stays on the CUDA-12 / cuDNN-9 line the default onnxruntime-gpu + torch are
|
# has to hand the container the driver (it sets NVIDIA_VISIBLE_DEVICES /
|
||||||
# built against (CUDA 13 has only nascent ONNX Runtime support).
|
# NVIDIA_DRIVER_CAPABILITIES for the Container Toolkit). Until #1451 this was
|
||||||
FROM nvidia/cuda:12.9.2-cudnn-runtime-ubuntu24.04
|
# `12.9.2-cudnn-runtime` under a `torch==2.6.0+cu124` — and requirements.txt then
|
||||||
|
# REPLACED that torch with PyPI's CUDA-13 build (ultralytics pulls torchvision,
|
||||||
|
# which pulls its matching torch), beside a CUDA-13 onnxruntime-gpu. The image
|
||||||
|
# ran CUDA 13 on a CUDA-12 base, carrying ~3 GB of base libraries and a ~3 GB
|
||||||
|
# torch nothing loaded: 10 GB compressed.
|
||||||
|
#
|
||||||
|
# 13.0 because that is the line both wheels are built for (torch's cu130 index,
|
||||||
|
# onnxruntime-gpu's `nvidia-cuda-runtime~=13.0`). Needs an NVIDIA driver that
|
||||||
|
# supports CUDA 13 (580+); fc_agent/accel.py logs at startup whether torch and
|
||||||
|
# onnxruntime actually got the GPU, since both fall back to the CPU silently.
|
||||||
|
# ffmpeg for video frames. Ubuntu 24.04 → Python 3.12.
|
||||||
|
FROM nvidia/cuda:13.0.3-base-ubuntu24.04
|
||||||
|
|
||||||
# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
|
# PIP_BREAK_SYSTEM_PACKAGES: Ubuntu 24.04 marks its system Python as externally
|
||||||
# managed (PEP 668), so a global `pip install` errors without this. It's a
|
# managed (PEP 668), so a global `pip install` errors without this. It's a
|
||||||
@@ -16,10 +27,12 @@ RUN apt-get update \
|
|||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
# torch from the CUDA-12.4 wheel index; its wheels bundle their own CUDA + cuDNN
|
# torch AND torchvision from the cu130 index, together and first. Installing
|
||||||
# so they run on the 12.9 base and coexist with onnxruntime-gpu. Installed first
|
# torch alone is what let the next step swap it out: ultralytics needs
|
||||||
# + separately so the GPU build of torch is deterministic and layer-cached.
|
# torchvision, PyPI's torchvision pins its own torch, and pip replaced ours to
|
||||||
RUN pip3 install --no-cache-dir torch==2.6.0 --index-url https://download.pytorch.org/whl/cu124
|
# match. With both present, requirements.txt finds them satisfied.
|
||||||
|
RUN pip3 install --no-cache-dir --index-url https://download.pytorch.org/whl/cu130 \
|
||||||
|
torch torchvision
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip3 install --no-cache-dir -r requirements.txt
|
RUN pip3 install --no-cache-dir -r requirements.txt
|
||||||
COPY fc_agent ./fc_agent
|
COPY fc_agent ./fc_agent
|
||||||
|
|||||||
+16
-1
@@ -15,7 +15,22 @@ sudo pacman -S nvidia-container-toolkit
|
|||||||
sudo nvidia-ctk runtime configure --runtime=docker
|
sudo nvidia-ctk runtime configure --runtime=docker
|
||||||
sudo systemctl restart docker
|
sudo systemctl restart docker
|
||||||
# verify:
|
# verify:
|
||||||
docker run --rm --gpus all nvidia/cuda:12.4.1-base-ubuntu22.04 nvidia-smi
|
docker run --rm --gpus all nvidia/cuda:13.0.3-base-ubuntu24.04 nvidia-smi
|
||||||
|
# the header's CUDA version must be 13.0 or later (driver 580+)
|
||||||
|
```
|
||||||
|
|
||||||
|
### After a driver update: regenerate the CDI spec
|
||||||
|
If the agent's first log lines say `accel: torch is NOT on the GPU` or report
|
||||||
|
`cudaGetDeviceCount: unknown error (999)` while `nvidia-smi` still works, the
|
||||||
|
toolkit's saved device list (`/etc/cdi/nvidia.yaml`) is out of date. The
|
||||||
|
`nvidia-uvm` device number changes between driver versions, and a spec
|
||||||
|
generated before the update hands the container a device node that no longer
|
||||||
|
exists (2026-09-24: host `511,0`, container `235,0`). Compare
|
||||||
|
`ls -l /dev/nvidia-uvm` on the host with the same inside the container, then:
|
||||||
|
```sh
|
||||||
|
sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
|
||||||
|
# if your toolkit ships it, this keeps it current on every driver update:
|
||||||
|
sudo systemctl enable --now nvidia-cdi-refresh.path
|
||||||
```
|
```
|
||||||
|
|
||||||
## 1. Get a token
|
## 1. Get a token
|
||||||
|
|||||||
@@ -0,0 +1,124 @@
|
|||||||
|
"""Which accelerator each runtime actually got — reported once, at startup.
|
||||||
|
|
||||||
|
The agent has two GPU runtimes and both fall back to the CPU without raising:
|
||||||
|
torch when the driver is too old for its CUDA build, and onnxruntime (the imgutils
|
||||||
|
detector + CCIP models) when its CUDA provider cannot load its libraries. A
|
||||||
|
fallback shows up only as slower work, and nothing reported it. On 2026-09-24 the
|
||||||
|
image turned out to be running a CUDA-13 torch and onnxruntime on a CUDA-12 base
|
||||||
|
(#1451), and whether the ONNX half was on the GPU could not be answered from
|
||||||
|
anything the agent had ever logged.
|
||||||
|
|
||||||
|
Also the fix for the likeliest way the ONNX half misses: onnxruntime-gpu's CUDA
|
||||||
|
provider finds libcudart/cuBLAS/cuDNN only on the loader path, and in this image
|
||||||
|
they live in the `nvidia-*` pip packages torch installs. `preload_dlls()` (ORT
|
||||||
|
1.21+) loads them from there, so the provider resolves them by soname.
|
||||||
|
|
||||||
|
Stdlib-only at import, so the unit suite can import it — torch and onnxruntime
|
||||||
|
are imported inside the functions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ctypes
|
||||||
|
import importlib
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
log = logging.getLogger("fc_agent.accel")
|
||||||
|
|
||||||
|
# Filled by report(); /status carries it so the page can show it too.
|
||||||
|
LAST: dict = {}
|
||||||
|
|
||||||
|
|
||||||
|
def torch_status(imp=importlib.import_module) -> dict:
|
||||||
|
try:
|
||||||
|
torch = imp("torch")
|
||||||
|
except Exception as e:
|
||||||
|
return {"device": "unavailable", "error": str(e)}
|
||||||
|
out = {"version": torch.__version__, "cuda_build": torch.version.cuda}
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
out["device"] = "cuda"
|
||||||
|
out["gpu"] = torch.cuda.get_device_name(0)
|
||||||
|
else:
|
||||||
|
out["device"] = "cpu"
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def onnx_status(imp=importlib.import_module, load=ctypes.CDLL) -> dict:
|
||||||
|
try:
|
||||||
|
ort = imp("onnxruntime")
|
||||||
|
except Exception as e:
|
||||||
|
return {"device": "unavailable", "error": str(e)}
|
||||||
|
out = {"version": ort.__version__, "providers": list(ort.get_available_providers())}
|
||||||
|
if "CUDAExecutionProvider" not in out["providers"]:
|
||||||
|
out["device"] = "cpu"
|
||||||
|
return out
|
||||||
|
preload = getattr(ort, "preload_dlls", None)
|
||||||
|
if preload is not None:
|
||||||
|
try:
|
||||||
|
preload()
|
||||||
|
except Exception as e:
|
||||||
|
out["preload_error"] = str(e)
|
||||||
|
# "Available" only means the build HAS the provider. Loading its library is
|
||||||
|
# what resolves libcudart/cuBLAS/cuDNN — the step that fails when they are
|
||||||
|
# missing, and the one a session would otherwise fail silently on.
|
||||||
|
capi = Path(ort.__file__).parent / "capi"
|
||||||
|
try:
|
||||||
|
load(str(capi / "libonnxruntime_providers_shared.so"), mode=ctypes.RTLD_GLOBAL)
|
||||||
|
load(str(capi / "libonnxruntime_providers_cuda.so"))
|
||||||
|
except OSError as e:
|
||||||
|
out["device"] = "cpu"
|
||||||
|
out["error"] = str(e)
|
||||||
|
return out
|
||||||
|
# Loading proves the libraries resolve, NOT that a GPU can be used: on
|
||||||
|
# 2026-09-24 this reported "onnx on GPU" beside torch failing cuInit with
|
||||||
|
# "CUDA unknown error" (a driver update awaiting a reboot). Asking the CUDA
|
||||||
|
# runtime for a device initialises the driver the provider would use.
|
||||||
|
error = _cuda_device_error(load)
|
||||||
|
out["device"] = "cpu" if error else "cuda"
|
||||||
|
if error:
|
||||||
|
out["error"] = error
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _cuda_device_error(load=ctypes.CDLL) -> str | None:
|
||||||
|
"""None when the CUDA runtime can reach a device, else why it cannot."""
|
||||||
|
try:
|
||||||
|
cudart = load("libcudart.so.13")
|
||||||
|
except OSError as e:
|
||||||
|
return str(e)
|
||||||
|
count = ctypes.c_int(0)
|
||||||
|
rc = cudart.cudaGetDeviceCount(ctypes.byref(count))
|
||||||
|
if rc != 0:
|
||||||
|
cudart.cudaGetErrorString.restype = ctypes.c_char_p
|
||||||
|
return f"cudaGetDeviceCount: {cudart.cudaGetErrorString(rc).decode()} ({rc})"
|
||||||
|
return None if count.value > 0 else "no CUDA device visible"
|
||||||
|
|
||||||
|
|
||||||
|
def summary() -> dict | None:
|
||||||
|
"""The report as FabledCurator stores it: each runtime's device, and why
|
||||||
|
when it is not the GPU. Sent on every lease and heartbeat, so the System
|
||||||
|
view can call a running agent that fell back to the CPU "degraded" rather
|
||||||
|
than "running" — the 2026-09-24 fallback went unseen for weeks because
|
||||||
|
only this agent's own log said so. None before report() has run."""
|
||||||
|
if not LAST:
|
||||||
|
return None
|
||||||
|
out = {}
|
||||||
|
for name, s in LAST.items():
|
||||||
|
entry = {"device": s.get("device")}
|
||||||
|
if s.get("error"):
|
||||||
|
entry["error"] = str(s["error"])[:200]
|
||||||
|
out[name] = entry
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def report() -> dict:
|
||||||
|
"""Check both runtimes, log the result, and keep it for /status."""
|
||||||
|
LAST.clear()
|
||||||
|
LAST.update(torch=torch_status(), onnx=onnx_status())
|
||||||
|
for name, s in LAST.items():
|
||||||
|
if s.get("device") == "cuda":
|
||||||
|
log.info("accel: %s on GPU (%s)", name, s)
|
||||||
|
else:
|
||||||
|
log.warning("accel: %s is NOT on the GPU — work runs on the CPU (%s)", name, s)
|
||||||
|
return dict(LAST)
|
||||||
+28
-3
@@ -11,7 +11,7 @@ import logging
|
|||||||
from fastapi import FastAPI, Request
|
from fastapi import FastAPI, Request
|
||||||
from fastapi.responses import HTMLResponse, JSONResponse
|
from fastapi.responses import HTMLResponse, JSONResponse
|
||||||
|
|
||||||
from . import logbuf
|
from . import accel, logbuf
|
||||||
from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version
|
from .build_info import FC_CHANNEL, FC_REVISION, FC_VERSION, build_id, display_version
|
||||||
from .config import Config
|
from .config import Config
|
||||||
from .gpu import read_gpu
|
from .gpu import read_gpu
|
||||||
@@ -47,6 +47,9 @@ async def _no_store(request, call_next):
|
|||||||
|
|
||||||
@app.on_event("startup")
|
@app.on_event("startup")
|
||||||
def _maybe_autostart() -> None:
|
def _maybe_autostart() -> None:
|
||||||
|
# Before the worker: the report also preloads the CUDA libraries the ONNX
|
||||||
|
# models need, and it says in the log which runtimes landed on the GPU.
|
||||||
|
accel.report()
|
||||||
# With AUTO_START set, a container restart (host reboot, or `restart:
|
# With AUTO_START set, a container restart (host reboot, or `restart:
|
||||||
# unless-stopped` after a crash) resumes the worker on its own — the slots
|
# unless-stopped` after a crash) resumes the worker on its own — the slots
|
||||||
# then ride out a still-down curator via lease backoff. Lets the agent
|
# then ride out a still-down curator via lease backoff. Lets the agent
|
||||||
@@ -137,6 +140,7 @@ def status():
|
|||||||
s["version"] = FC_VERSION or None
|
s["version"] = FC_VERSION or None
|
||||||
s["channel"] = FC_CHANNEL or None
|
s["channel"] = FC_CHANNEL or None
|
||||||
s["revision"] = FC_REVISION or None
|
s["revision"] = FC_REVISION or None
|
||||||
|
s["accel"] = accel.LAST or None
|
||||||
return JSONResponse(s)
|
return JSONResponse(s)
|
||||||
|
|
||||||
|
|
||||||
@@ -188,7 +192,11 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
|||||||
width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
|
width:30px;height:32px;font:700 16px system-ui;cursor:pointer}
|
||||||
.step:hover{border-color:var(--acc)}
|
.step:hover{border-color:var(--acc)}
|
||||||
#conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
|
#conc,#bw{width:3.4rem;height:32px;text-align:center;font:700 16px system-ui;background:#11151a;
|
||||||
color:var(--fg);border:1px solid var(--bd);border-radius:8px}
|
color:var(--fg);border:1px solid var(--bd);border-radius:8px;appearance:textfield;-moz-appearance:textfield}
|
||||||
|
/* The browser's own spin arrows, hidden: the − / + beside each field are the
|
||||||
|
control, styled like the rest of the page (operator, 2026-09-24). */
|
||||||
|
#conc::-webkit-inner-spin-button,#conc::-webkit-outer-spin-button,
|
||||||
|
#bw::-webkit-inner-spin-button,#bw::-webkit-outer-spin-button{-webkit-appearance:none;margin:0}
|
||||||
.unit{color:var(--mut);font-size:12px;font-weight:600}
|
.unit{color:var(--mut);font-size:12px;font-weight:600}
|
||||||
.hint{color:var(--mut);font-size:12px;margin-top:12px}
|
.hint{color:var(--mut);font-size:12px;margin-top:12px}
|
||||||
.tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
|
.tiles{display:grid;grid-template-columns:repeat(6,1fr);gap:8px;margin-bottom:16px}
|
||||||
@@ -227,6 +235,7 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
|||||||
<div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
|
<div id=verbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3">
|
||||||
a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
|
a newer agent version is running — reload this page (Ctrl+Shift+R) to update the controls
|
||||||
</div>
|
</div>
|
||||||
|
<div id=accelbanner class=banner style="display:none;background:#3a1212;border-color:#5a1717;color:#ffb3b3"></div>
|
||||||
<div id=banner class=banner style=display:none>
|
<div id=banner class=banner style=display:none>
|
||||||
curator unreachable — holding work + retrying, resumes on its own (no restart needed)
|
curator unreachable — holding work + retrying, resumes on its own (no restart needed)
|
||||||
</div>
|
</div>
|
||||||
@@ -244,7 +253,9 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
|||||||
<button class=step onclick=setc(1)>+</button>
|
<button class=step onclick=setc(1)>+</button>
|
||||||
</div>
|
</div>
|
||||||
<div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
|
<div class=stepper title="aggregate download cap, downloads + video streams combined — 0 = unlimited">
|
||||||
|
<button class=step onclick=stepbw(-1)>−</button>
|
||||||
<input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
|
<input id=bw type=number min=0 step=1 value=8 onchange="setbw(this.value)">
|
||||||
|
<button class=step onclick=stepbw(1)>+</button>
|
||||||
<span class=unit>MB/s</span>
|
<span class=unit>MB/s</span>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -312,6 +323,14 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
|||||||
await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
|
await fetch('/auto',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||||
body:JSON.stringify({value:on})});refresh()
|
body:JSON.stringify({value:on})});refresh()
|
||||||
}
|
}
|
||||||
|
function stepbw(d){ setbw((parseFloat(bw.value)||0)+d) }
|
||||||
|
// Runtimes that did NOT get the GPU, from the startup report. Both fall back
|
||||||
|
// to the CPU without raising, so this banner and the pill are the only place
|
||||||
|
// on this page a slow, CPU-bound agent announces itself.
|
||||||
|
function cpuOnly(s){
|
||||||
|
const a=s.accel||{}
|
||||||
|
return Object.keys(a).filter(k=>a[k] && a[k].device!=='cuda')
|
||||||
|
}
|
||||||
async function setbw(v){
|
async function setbw(v){
|
||||||
v=Math.max(0,parseFloat(v)||0); bw.value=v
|
v=Math.max(0,parseFloat(v)||0); bw.value=v
|
||||||
await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
|
await fetch('/bandwidth',{method:'POST',headers:{'Content-Type':'application/json'},
|
||||||
@@ -382,11 +401,17 @@ _PAGE = """<!doctype html><html><head><meta charset=utf-8>
|
|||||||
// unreachable curator; grey when stopped; red with no token.
|
// unreachable curator; grey when stopped; red with no token.
|
||||||
let dc='dot', lbl='stopped'
|
let dc='dot', lbl='stopped'
|
||||||
if(!ok){ dc='dot red'; lbl='no token' }
|
if(!ok){ dc='dot red'; lbl='no token' }
|
||||||
else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable' }
|
else if(st==='running'){ dc='dot '+(s.queue?'green':'amber'); lbl=s.queue?'running':'running · curator unreachable'
|
||||||
|
if(s.queue && cpuOnly(s).length){ dc='dot amber'; lbl='running · CPU only (degraded)' } }
|
||||||
else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
|
else if(st==='starting'){ dc='dot amber'; lbl='starting…' }
|
||||||
else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
|
else if(st==='stopping'){ dc='dot amber'; lbl='stopping…' }
|
||||||
dot.className=dc; connlbl.textContent=lbl
|
dot.className=dc; connlbl.textContent=lbl
|
||||||
banner.style.display=(st==='running' && !s.queue)?'block':'none'
|
banner.style.display=(st==='running' && !s.queue)?'block':'none'
|
||||||
|
const slow=cpuOnly(s)
|
||||||
|
accelbanner.style.display=slow.length?'block':'none'
|
||||||
|
accelbanner.textContent=slow.length?('degraded — '+slow.join(' + ')+' not on the GPU, so that work runs on the CPU: '
|
||||||
|
+slow.map(k=>k+': '+(s.accel[k].error||s.accel[k].device)).join(' · ')
|
||||||
|
+'. After a driver update, regenerate the CDI spec (agent README).'):''
|
||||||
queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
|
queue.textContent=s.queue?('queue · pending '+s.queue.pending+' · in flight '+s.queue.leased+' · done '+s.queue.done+' · errored '+s.queue.error):'queue · unreachable'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import requests
|
|||||||
from requests.adapters import HTTPAdapter
|
from requests.adapters import HTTPAdapter
|
||||||
from urllib3.util.retry import Retry
|
from urllib3.util.retry import Retry
|
||||||
|
|
||||||
|
from . import accel
|
||||||
|
|
||||||
|
|
||||||
class FcClient:
|
class FcClient:
|
||||||
def __init__(self, base_url: str, token: str, agent_id: str):
|
def __init__(self, base_url: str, token: str, agent_id: str):
|
||||||
@@ -72,7 +74,10 @@ class FcClient:
|
|||||||
def lease(self, batch_size: int) -> list[dict]:
|
def lease(self, batch_size: int) -> list[dict]:
|
||||||
r = self.s.post(
|
r = self.s.post(
|
||||||
f"{self.base}/api/gpu/jobs/lease",
|
f"{self.base}/api/gpu/jobs/lease",
|
||||||
json={"agent_id": self.agent_id, "batch_size": batch_size},
|
json={
|
||||||
|
"agent_id": self.agent_id, "batch_size": batch_size,
|
||||||
|
"accel": accel.summary(),
|
||||||
|
},
|
||||||
timeout=30,
|
timeout=30,
|
||||||
)
|
)
|
||||||
r.raise_for_status()
|
r.raise_for_status()
|
||||||
@@ -90,7 +95,9 @@ class FcClient:
|
|||||||
})
|
})
|
||||||
|
|
||||||
def heartbeat(self, job_ids: list[int]) -> None:
|
def heartbeat(self, job_ids: list[int]) -> None:
|
||||||
self._post_quiet("/api/gpu/jobs/heartbeat", {"job_ids": job_ids})
|
self._post_quiet(
|
||||||
|
"/api/gpu/jobs/heartbeat", {"job_ids": job_ids, "accel": accel.summary()},
|
||||||
|
)
|
||||||
|
|
||||||
def fail(self, job_id: int, error: str) -> None:
|
def fail(self, job_id: int, error: str) -> None:
|
||||||
self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
|
self._post_quiet("/api/gpu/jobs/fail", {"job_id": job_id, "error": error})
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
# CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace).
|
# CCIP + figure detection (ONNX models, auto-downloaded from HuggingFace).
|
||||||
dghs-imgutils>=0.4
|
dghs-imgutils>=0.4
|
||||||
# GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
|
# GPU inference for the ONNX models. Swap to onnxruntime (CPU) for a slow
|
||||||
# server-side fallback run.
|
# server-side fallback run. The extras declare the CUDA/cuDNN pip packages its
|
||||||
onnxruntime-gpu
|
# CUDA provider loads (fc_agent/accel.py preloads them) rather than relying on
|
||||||
# The crop EMBEDDER (concept bag). torch is installed separately in the
|
# torch happening to install the same ones.
|
||||||
# Dockerfile from the CUDA-12.4 wheel index so the GPU build is deterministic;
|
onnxruntime-gpu[cuda,cudnn]
|
||||||
|
# The crop EMBEDDER (concept bag). torch + torchvision are installed separately
|
||||||
|
# in the Dockerfile from the cu130 wheel index, so pip never swaps them out;
|
||||||
# transformers loads whatever SigLIP-family model the server announces.
|
# transformers loads whatever SigLIP-family model the server announces.
|
||||||
transformers>=4.45
|
transformers>=4.45
|
||||||
# Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic
|
# Crop PROPOSERS — small YOLO detectors (booru_yolo anatomy, COCO person, comic
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
"""Discord native ingester ledgers — seen and dead-letter, per source.
|
||||||
|
|
||||||
|
Milestone 428, #4415. Discord moves off gallery-dl onto the native core, which
|
||||||
|
keeps its memory of what a source has already fetched in these two tables
|
||||||
|
instead of gallery-dl's archive. Same shape as the SubscribeStar pair.
|
||||||
|
|
||||||
|
Revision ID: 0111
|
||||||
|
Revises: 0110
|
||||||
|
Create Date: 2026-09-24
|
||||||
|
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0111"
|
||||||
|
down_revision = "0110"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
op.create_table(
|
||||||
|
"discord_seen_media",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("post_id", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"seen_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"], ["source.id"],
|
||||||
|
name=op.f("fk_discord_seen_media_source_id_source"), ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_seen_media")),
|
||||||
|
sa.UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_discord_seen_media_source_id"), "discord_seen_media", ["source_id"],
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"discord_failed_media",
|
||||||
|
sa.Column("id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("source_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("filehash", sa.String(length=128), nullable=False),
|
||||||
|
sa.Column("attempts", sa.Integer(), server_default="1", nullable=False),
|
||||||
|
sa.Column("last_error", sa.Text(), nullable=True),
|
||||||
|
sa.Column(
|
||||||
|
"first_failed_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.Column(
|
||||||
|
"last_failed_at", sa.DateTime(timezone=True),
|
||||||
|
server_default=sa.text("now()"), nullable=False,
|
||||||
|
),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["source_id"], ["source.id"],
|
||||||
|
name=op.f("fk_discord_failed_media_source_id_source"), ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_discord_failed_media")),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"source_id", "filehash", name="uq_discord_failed_media_source_id",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
op.f("ix_discord_failed_media_source_id"), "discord_failed_media", ["source_id"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.drop_index(op.f("ix_discord_failed_media_source_id"), table_name="discord_failed_media")
|
||||||
|
op.drop_table("discord_failed_media")
|
||||||
|
op.drop_index(op.f("ix_discord_seen_media_source_id"), table_name="discord_seen_media")
|
||||||
|
op.drop_table("discord_seen_media")
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
"""Retire the sketch/doodle WIP title tier — its tags, its review flags, its toggle.
|
||||||
|
|
||||||
|
Milestone 430, #4428. The soft tier (#1474) tagged `wip` on any post titled
|
||||||
|
sketch / doodle / scribble. Measured on the operator's library it was 6,096 of
|
||||||
|
8,876 wip tags, and its conflict audit filled the Gallery's review strip with
|
||||||
|
2,086 cards, because most finished art scores >= 0.5 on some content head. A
|
||||||
|
"sketch" is usually finished work, so the operator retired it (2026-09-25).
|
||||||
|
|
||||||
|
Data:
|
||||||
|
|
||||||
|
* A soft tag the operator stood behind is kept and relabelled `manual`: one they
|
||||||
|
confirmed (tag_positive_confirmation), or one whose review flag they resolved
|
||||||
|
while leaving the tag on (the strip's "Keep tag").
|
||||||
|
* Every other `wip_title_soft` row is deleted.
|
||||||
|
* Unresolved review flags whose tag is no longer on the image are deleted: the
|
||||||
|
question they ask no longer applies. That is the audit's cards, and any older
|
||||||
|
orphan the same way.
|
||||||
|
|
||||||
|
Then `import_settings.wip_soft_title_tagging_enabled` is dropped. The downgrade
|
||||||
|
restores the column only; deleted tags are not recreated.
|
||||||
|
|
||||||
|
Revision ID: 0112
|
||||||
|
Revises: 0111
|
||||||
|
Create Date: 2026-09-25
|
||||||
|
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0112"
|
||||||
|
down_revision = "0111"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def retire_soft_wip_tags(conn) -> None:
|
||||||
|
"""The data half, on a plain connection, so a test can run it directly."""
|
||||||
|
conn.execute(sa.text("""
|
||||||
|
UPDATE image_tag it SET source = 'manual'
|
||||||
|
WHERE it.source = 'wip_title_soft'
|
||||||
|
AND (
|
||||||
|
EXISTS (
|
||||||
|
SELECT 1 FROM tag_positive_confirmation c
|
||||||
|
WHERE c.image_record_id = it.image_record_id AND c.tag_id = it.tag_id
|
||||||
|
)
|
||||||
|
OR EXISTS (
|
||||||
|
SELECT 1 FROM presentation_review pr
|
||||||
|
WHERE pr.image_record_id = it.image_record_id AND pr.tag_id = it.tag_id
|
||||||
|
AND pr.resolved_at IS NOT NULL
|
||||||
|
)
|
||||||
|
)
|
||||||
|
"""))
|
||||||
|
conn.execute(sa.text("DELETE FROM image_tag WHERE source = 'wip_title_soft'"))
|
||||||
|
conn.execute(sa.text("""
|
||||||
|
DELETE FROM presentation_review pr
|
||||||
|
WHERE pr.resolved_at IS NULL
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM image_tag it
|
||||||
|
WHERE it.image_record_id = pr.image_record_id AND it.tag_id = pr.tag_id
|
||||||
|
)
|
||||||
|
"""))
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
retire_soft_wip_tags(op.get_bind())
|
||||||
|
op.drop_column("import_settings", "wip_soft_title_tagging_enabled")
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
op.add_column(
|
||||||
|
"import_settings",
|
||||||
|
sa.Column(
|
||||||
|
"wip_soft_title_tagging_enabled",
|
||||||
|
sa.Boolean(),
|
||||||
|
nullable=False,
|
||||||
|
server_default=sa.text("false"),
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -0,0 +1,60 @@
|
|||||||
|
"""Re-date images whose post's date arrived after they were linked.
|
||||||
|
|
||||||
|
#4431. The native ingesters import a post's media before its record, and the
|
||||||
|
date travels in the record (`_post.json`). Every natively downloaded image was
|
||||||
|
therefore linked to an undated post and kept its download time in both gallery
|
||||||
|
date columns, while the post itself was dated correctly. The importer now
|
||||||
|
re-dates a post's images when its record lands; this repairs the images that
|
||||||
|
landed before that.
|
||||||
|
|
||||||
|
Both columns get back the rules the importer keeps:
|
||||||
|
|
||||||
|
* `effective_date` is the primary post's date (left alone when that post has
|
||||||
|
none, as the importer does);
|
||||||
|
* `earliest_post_date` is the earliest dated post the image is linked to.
|
||||||
|
|
||||||
|
Only rows that differ are written. The downgrade does nothing: the old values
|
||||||
|
were download times that no one chose.
|
||||||
|
|
||||||
|
Revision ID: 0113
|
||||||
|
Revises: 0112
|
||||||
|
Create Date: 2026-09-25
|
||||||
|
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0113"
|
||||||
|
down_revision = "0112"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def redate_images(conn) -> None:
|
||||||
|
"""The data step, on a plain connection, so a test can run it directly."""
|
||||||
|
conn.execute(sa.text("""
|
||||||
|
UPDATE image_record ir SET effective_date = p.post_date
|
||||||
|
FROM post p
|
||||||
|
WHERE p.id = ir.primary_post_id
|
||||||
|
AND p.post_date IS NOT NULL
|
||||||
|
AND ir.effective_date IS DISTINCT FROM p.post_date
|
||||||
|
"""))
|
||||||
|
conn.execute(sa.text("""
|
||||||
|
UPDATE image_record ir SET earliest_post_date = m.earliest
|
||||||
|
FROM (
|
||||||
|
SELECT ip.image_record_id, MIN(p.post_date) AS earliest
|
||||||
|
FROM image_provenance ip JOIN post p ON p.id = ip.post_id
|
||||||
|
WHERE p.post_date IS NOT NULL
|
||||||
|
GROUP BY ip.image_record_id
|
||||||
|
) m
|
||||||
|
WHERE m.image_record_id = ir.id
|
||||||
|
AND ir.earliest_post_date IS DISTINCT FROM m.earliest
|
||||||
|
"""))
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade():
|
||||||
|
redate_images(op.get_bind())
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade():
|
||||||
|
pass
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
"""Fold the undated shell posts a misfiled attachment created into the real post.
|
||||||
|
|
||||||
|
#4435. A non-image file (an archive, a pdf) downloaded for one of an artist's
|
||||||
|
Discord channels was filed under the artist's FIRST Discord source: the
|
||||||
|
attachment path looked the source up by (artist, platform), which takes the
|
||||||
|
lowest id. That created an undated, url-less post there holding only the
|
||||||
|
attachment, and the message's real post record then created the dated post
|
||||||
|
under the right source. The importer now uses the source it was downloading
|
||||||
|
for; this repairs the pairs it left.
|
||||||
|
|
||||||
|
A shell is folded only when all of this holds:
|
||||||
|
|
||||||
|
* it has no date and no url, and nothing synthesized it;
|
||||||
|
* another post of the same artist, platform and external id HAS a date;
|
||||||
|
* no image is linked to the shell (it held an attachment and nothing else).
|
||||||
|
|
||||||
|
Its attachments move to the dated post, dropping any the dated post already
|
||||||
|
has (same sha256, which the per-post unique forbids twice), and the shell is
|
||||||
|
deleted. A shell with no dated twin is left alone: which channel it belongs to
|
||||||
|
is not recorded anywhere but the file name. The downgrade does nothing.
|
||||||
|
|
||||||
|
Revision ID: 0114
|
||||||
|
Revises: 0113
|
||||||
|
Create Date: 2026-09-25
|
||||||
|
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0114"
|
||||||
|
down_revision = "0113"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
_PAIRS = """
|
||||||
|
SELECT DISTINCT ON (shell.id) shell.id AS shell_id, real.id AS real_id
|
||||||
|
FROM post shell
|
||||||
|
JOIN source ss ON ss.id = shell.source_id
|
||||||
|
JOIN post real
|
||||||
|
ON real.artist_id = shell.artist_id
|
||||||
|
AND real.external_post_id = shell.external_post_id
|
||||||
|
AND real.id <> shell.id
|
||||||
|
AND real.post_date IS NOT NULL
|
||||||
|
JOIN source rs ON rs.id = real.source_id AND rs.platform = ss.platform
|
||||||
|
WHERE shell.post_date IS NULL
|
||||||
|
AND shell.post_url IS NULL
|
||||||
|
AND shell.synthesized_by IS NULL
|
||||||
|
AND NOT EXISTS (SELECT 1 FROM image_provenance ip WHERE ip.post_id = shell.id)
|
||||||
|
ORDER BY shell.id, real.id
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def fold_misfiled_attachment_posts(conn) -> int:
|
||||||
|
"""The data step, on a plain connection, so a test can run it directly.
|
||||||
|
Returns how many shells were folded."""
|
||||||
|
pairs = conn.execute(sa.text(_PAIRS)).all()
|
||||||
|
for shell_id, real_id in pairs:
|
||||||
|
conn.execute(sa.text("""
|
||||||
|
DELETE FROM post_attachment pa
|
||||||
|
WHERE pa.post_id = :shell
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM post_attachment keep
|
||||||
|
WHERE keep.post_id = :real AND keep.sha256 = pa.sha256
|
||||||
|
)
|
||||||
|
"""), {"shell": shell_id, "real": real_id})
|
||||||
|
conn.execute(
|
||||||
|
sa.text("UPDATE post_attachment SET post_id = :real WHERE post_id = :shell"),
|
||||||
|
{"shell": shell_id, "real": real_id},
|
||||||
|
)
|
||||||
|
conn.execute(sa.text("DELETE FROM post WHERE id = :shell"), {"shell": shell_id})
|
||||||
|
return len(pairs)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
fold_misfiled_attachment_posts(op.get_bind())
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
pass
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
"""source.display_name — what a source is called on its platform.
|
||||||
|
|
||||||
|
#4481. A Discord source's URL is a server id and a channel id, so the
|
||||||
|
Subscriptions page could only show two numbers. The Discord ingester already
|
||||||
|
loads the server and channel names to walk them; it now keeps them here,
|
||||||
|
refreshed on every walk. NULL until a walk has read it.
|
||||||
|
|
||||||
|
Revision ID: 0115
|
||||||
|
Revises: 0114
|
||||||
|
Create Date: 2026-09-28
|
||||||
|
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0115"
|
||||||
|
down_revision = "0114"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.add_column("source", sa.Column("display_name", sa.Text(), nullable=True))
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_column("source", "display_name")
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""The single-color filter's default becomes near-total: 0.95 → 0.995.
|
||||||
|
|
||||||
|
#4483. At 0.95 the filter rejected line art: a pencil doodle on white is
|
||||||
|
mostly white, and five of Todding's Discord doodles were skipped on import as
|
||||||
|
"single color", leaving their posts with text and no image. The predicate now
|
||||||
|
samples real pixels instead of a blurred thumbnail, and the default only
|
||||||
|
calls an image blank when it essentially is one.
|
||||||
|
|
||||||
|
A stored 0.95 is the old default, so it moves with it. Any other value was
|
||||||
|
chosen by hand and is left alone.
|
||||||
|
|
||||||
|
Revision ID: 0116
|
||||||
|
Revises: 0115
|
||||||
|
Create Date: 2026-09-28
|
||||||
|
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "0116"
|
||||||
|
down_revision = "0115"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.alter_column("import_settings", "single_color_threshold", server_default="0.995")
|
||||||
|
op.execute(
|
||||||
|
"UPDATE import_settings SET single_color_threshold = 0.995 "
|
||||||
|
"WHERE single_color_threshold = 0.95"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.alter_column("import_settings", "single_color_threshold", server_default="0.95")
|
||||||
@@ -19,7 +19,10 @@ from ..models import AppSetting
|
|||||||
from ..services.extension_service import (
|
from ..services.extension_service import (
|
||||||
ExtensionService,
|
ExtensionService,
|
||||||
InvalidUrlError,
|
InvalidUrlError,
|
||||||
|
PosterLookupError,
|
||||||
|
UnknownArtistError,
|
||||||
UnknownPlatformError,
|
UnknownPlatformError,
|
||||||
|
UnknownSourceError,
|
||||||
)
|
)
|
||||||
from ..services.source_service import KNOWN_PLATFORMS
|
from ..services.source_service import KNOWN_PLATFORMS
|
||||||
from ._responses import error_response as _bad
|
from ._responses import error_response as _bad
|
||||||
@@ -87,10 +90,16 @@ async def probe_source():
|
|||||||
url = (request.args.get("url") or "").strip()
|
url = (request.args.get("url") or "").strip()
|
||||||
if not url:
|
if not url:
|
||||||
return _bad("invalid_body", detail="url query parameter is required")
|
return _bad("invalid_body", detail="url query parameter is required")
|
||||||
|
from .credentials import _get_crypto
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
if not await _ext_key_required(session):
|
if not await _ext_key_required(session):
|
||||||
return _bad("unauthorized", status=401)
|
return _bad("unauthorized", status=401)
|
||||||
result = await ExtensionService(session).probe(url)
|
# crypto lets a Discord probe name the server and channel with the
|
||||||
|
# stored token; every other platform ignores it.
|
||||||
|
result = await ExtensionService(session, _get_crypto()).probe(
|
||||||
|
url, names=request.args.get("names") in ("1", "true"),
|
||||||
|
)
|
||||||
return jsonify(result)
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
@@ -102,6 +111,22 @@ async def quick_add_source():
|
|||||||
url = body.get("url")
|
url = body.get("url")
|
||||||
if not isinstance(url, str) or not url.strip():
|
if not isinstance(url, str) or not url.strip():
|
||||||
return _bad("invalid_body", detail="url is required")
|
return _bad("invalid_body", detail="url is required")
|
||||||
|
# Optional: connect the new source to an existing artist (artist_id) or to
|
||||||
|
# the artist of that name (artist_name). A Discord channel names no
|
||||||
|
# creator, so the extension's Add panel always sends one of them.
|
||||||
|
artist_id = body.get("artist_id")
|
||||||
|
if artist_id is not None and (isinstance(artist_id, bool) or not isinstance(artist_id, int)):
|
||||||
|
return _bad("invalid_body", detail="artist_id must be an integer")
|
||||||
|
artist_name = body.get("artist_name")
|
||||||
|
if artist_name is not None and not isinstance(artist_name, str):
|
||||||
|
return _bad("invalid_body", detail="artist_name must be a string")
|
||||||
|
# Patreon is canon: adding a Patreon source to an existing artist can take
|
||||||
|
# the creator's Patreon display name (name only; the slug never moves).
|
||||||
|
use_platform_name = body.get("use_platform_name") is True
|
||||||
|
# #4491: a Discord add can carry the posters picked in the panel.
|
||||||
|
discord_authors = body.get("discord_authors")
|
||||||
|
if discord_authors is not None and not _valid_authors(discord_authors):
|
||||||
|
return _bad("invalid_body", detail="discord_authors must be a list of {id, name}")
|
||||||
|
|
||||||
from .credentials import _get_crypto
|
from .credentials import _get_crypto
|
||||||
|
|
||||||
@@ -111,7 +136,13 @@ async def quick_add_source():
|
|||||||
try:
|
try:
|
||||||
# crypto lets an add resolve the artist's display name via the
|
# crypto lets an add resolve the artist's display name via the
|
||||||
# stored credential (else it falls back to the URL handle). #130.
|
# stored credential (else it falls back to the URL handle). #130.
|
||||||
result = await ExtensionService(session, _get_crypto()).quick_add_source(url)
|
result = await ExtensionService(session, _get_crypto()).quick_add_source(
|
||||||
|
url, artist_id=artist_id, artist_name=artist_name,
|
||||||
|
use_platform_name=use_platform_name,
|
||||||
|
discord_authors=discord_authors,
|
||||||
|
)
|
||||||
|
except UnknownArtistError as exc:
|
||||||
|
return _bad("not_found", detail=str(exc), status=404)
|
||||||
except UnknownPlatformError as exc:
|
except UnknownPlatformError as exc:
|
||||||
return _bad(
|
return _bad(
|
||||||
"unknown_platform",
|
"unknown_platform",
|
||||||
@@ -123,6 +154,59 @@ async def quick_add_source():
|
|||||||
return jsonify(result), (201 if result["created_source"] else 200)
|
return jsonify(result), (201 if result["created_source"] else 200)
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_authors(value) -> bool:
|
||||||
|
"""[{id, name}]: the id is what the list keys on, the name only shows it."""
|
||||||
|
return isinstance(value, list) and all(
|
||||||
|
isinstance(a, dict) and isinstance(a.get("id"), str | int)
|
||||||
|
and not isinstance(a.get("id"), bool)
|
||||||
|
and (a.get("name") is None or isinstance(a.get("name"), str))
|
||||||
|
for a in value
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@extension_bp.route("/discord/posters", methods=["GET"])
|
||||||
|
async def discord_posters():
|
||||||
|
"""Who posts in the Discord channel `url` shows, lately, with the likely
|
||||||
|
creator marked (#4491). Reads the channel with the stored token; writes
|
||||||
|
nothing."""
|
||||||
|
url = (request.args.get("url") or "").strip()
|
||||||
|
if not url:
|
||||||
|
return _bad("invalid_body", detail="url query parameter is required")
|
||||||
|
from .credentials import _get_crypto
|
||||||
|
|
||||||
|
async with get_session() as session:
|
||||||
|
if not await _ext_key_required(session):
|
||||||
|
return _bad("unauthorized", status=401)
|
||||||
|
try:
|
||||||
|
result = await ExtensionService(session, _get_crypto()).discord_posters(url)
|
||||||
|
except (InvalidUrlError, UnknownPlatformError) as exc:
|
||||||
|
return _bad("invalid_url", detail=str(exc))
|
||||||
|
except PosterLookupError as exc:
|
||||||
|
return _bad("posters_unavailable", detail=str(exc), status=409)
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
|
@extension_bp.route("/discord/posters", methods=["POST"])
|
||||||
|
async def set_discord_posters():
|
||||||
|
"""Set the poster list of the source following `url`'s channel."""
|
||||||
|
body = await request.get_json(silent=True)
|
||||||
|
if not isinstance(body, dict) or not isinstance(body.get("url"), str):
|
||||||
|
return _bad("invalid_body", detail="url is required")
|
||||||
|
authors = body.get("authors")
|
||||||
|
if not _valid_authors(authors):
|
||||||
|
return _bad("invalid_body", detail="authors must be a list of {id, name}")
|
||||||
|
async with get_session() as session:
|
||||||
|
if not await _ext_key_required(session):
|
||||||
|
return _bad("unauthorized", status=401)
|
||||||
|
try:
|
||||||
|
result = await ExtensionService(session).set_discord_posters(body["url"], authors)
|
||||||
|
except (InvalidUrlError, UnknownPlatformError, PosterLookupError) as exc:
|
||||||
|
return _bad("invalid_url", detail=str(exc))
|
||||||
|
except UnknownSourceError as exc:
|
||||||
|
return _bad("not_found", detail=str(exc), status=404)
|
||||||
|
return jsonify(result)
|
||||||
|
|
||||||
|
|
||||||
def _read_manifest_sync() -> dict | None:
|
def _read_manifest_sync() -> dict | None:
|
||||||
"""All the filesystem-touching work for /api/extension/manifest,
|
"""All the filesystem-touching work for /api/extension/manifest,
|
||||||
in a sync helper so the async route can dispatch it via
|
in a sync helper so the async route can dispatch it via
|
||||||
|
|||||||
+31
-2
@@ -245,6 +245,29 @@ async def errors_recover(image_id: int):
|
|||||||
|
|
||||||
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
# --- Agent (bearer token): lease / submit / heartbeat / fail ------------
|
||||||
|
|
||||||
|
|
||||||
|
def _accel_detail(body: dict) -> dict:
|
||||||
|
"""The agent's own report of which runtime got the GPU, kept on its roster
|
||||||
|
row so the System view can call a CPU-bound agent degraded (#4410).
|
||||||
|
|
||||||
|
Only a dict of {runtime: {device, error?}} is kept, and each value is
|
||||||
|
reduced to those two short strings: this is written on every lease, by a
|
||||||
|
client the server does not control. An agent that sends nothing (an
|
||||||
|
older build) simply has no `accel`, which reads as not-yet-reported.
|
||||||
|
"""
|
||||||
|
raw = body.get("accel")
|
||||||
|
if not isinstance(raw, dict):
|
||||||
|
return {}
|
||||||
|
accel = {}
|
||||||
|
for name, entry in list(raw.items())[:4]:
|
||||||
|
if not isinstance(entry, dict):
|
||||||
|
continue
|
||||||
|
clean = {"device": str(entry.get("device") or "")[:16]}
|
||||||
|
if entry.get("error"):
|
||||||
|
clean["error"] = str(entry["error"])[:200]
|
||||||
|
accel[str(name)[:16]] = clean
|
||||||
|
return {"accel": accel} if accel else {}
|
||||||
|
|
||||||
@gpu_bp.route("/jobs/lease", methods=["POST"])
|
@gpu_bp.route("/jobs/lease", methods=["POST"])
|
||||||
async def lease():
|
async def lease():
|
||||||
body = await request.get_json(silent=True) or {}
|
body = await request.get_json(silent=True) or {}
|
||||||
@@ -267,7 +290,10 @@ async def lease():
|
|||||||
key=f"agent:{agent_id}",
|
key=f"agent:{agent_id}",
|
||||||
kind="agent",
|
kind="agent",
|
||||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||||
details={"agent_id": agent_id, "last_call": "lease", "leased": len(jobs)},
|
details={
|
||||||
|
"agent_id": agent_id, "last_call": "lease", "leased": len(jobs),
|
||||||
|
**_accel_detail(body),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
ml = await MLSettings.load(session)
|
ml = await MLSettings.load(session)
|
||||||
# image rows for url/mime in one shot
|
# image rows for url/mime in one shot
|
||||||
@@ -347,7 +373,10 @@ async def heartbeat():
|
|||||||
key=f"agent:{agent_id}",
|
key=f"agent:{agent_id}",
|
||||||
kind="agent",
|
kind="agent",
|
||||||
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
display_name="GPU agent" if agent_id == "agent" else f"GPU agent ({agent_id})",
|
||||||
details={"agent_id": agent_id, "last_call": "heartbeat", "extended": n},
|
details={
|
||||||
|
"agent_id": agent_id, "last_call": "heartbeat", "extended": n,
|
||||||
|
**_accel_detail(body),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
await session.commit()
|
await session.commit()
|
||||||
return jsonify({"extended": n})
|
return jsonify({"extended": n})
|
||||||
|
|||||||
@@ -57,7 +57,6 @@ _EDITABLE_FIELDS = (
|
|||||||
"translation_target_lang",
|
"translation_target_lang",
|
||||||
"translation_min_confidence",
|
"translation_min_confidence",
|
||||||
"wip_title_tagging_enabled",
|
"wip_title_tagging_enabled",
|
||||||
"wip_soft_title_tagging_enabled",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
# Per-host external-download toggles — all plain booleans, validated uniformly.
|
||||||
@@ -196,12 +195,6 @@ async def update_import_settings():
|
|||||||
return jsonify(
|
return jsonify(
|
||||||
{"error": "wip_title_tagging_enabled must be a boolean"}
|
{"error": "wip_title_tagging_enabled must be a boolean"}
|
||||||
), 400
|
), 400
|
||||||
if "wip_soft_title_tagging_enabled" in body and not isinstance(
|
|
||||||
body["wip_soft_title_tagging_enabled"], bool
|
|
||||||
):
|
|
||||||
return jsonify(
|
|
||||||
{"error": "wip_soft_title_tagging_enabled must be a boolean"}
|
|
||||||
), 400
|
|
||||||
|
|
||||||
async with get_session() as session:
|
async with get_session() as session:
|
||||||
row = await ImportSettings.load(session)
|
row = await ImportSettings.load(session)
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
"""FC-3a: CRUD over Source rows. FC-3c adds POST /<id>/check."""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
from quart import Blueprint, jsonify, request
|
from quart import Blueprint, jsonify, request
|
||||||
from sqlalchemy import func, select
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
from ..extensions import get_session
|
from ..extensions import get_session
|
||||||
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
from ..models import DownloadEvent, MembershipSync, PlatformMembership, Source
|
||||||
|
from ..services import discord_poster_cleanup as poster_cleanup
|
||||||
from ..services.artist_membership_service import ArtistMembershipService
|
from ..services.artist_membership_service import ArtistMembershipService
|
||||||
from ..services.artist_membership_service import rescan as membership_rescan
|
from ..services.artist_membership_service import rescan as membership_rescan
|
||||||
from ..services.artist_service import ArtistService
|
from ..services.artist_service import ArtistService
|
||||||
@@ -463,3 +466,49 @@ async def adopt_membership():
|
|||||||
return jsonify({"already_tracked": exc.existing_id})
|
return jsonify({"already_tracked": exc.existing_id})
|
||||||
artist_id = artist.id
|
artist_id = artist.id
|
||||||
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
return jsonify({"source_id": record.id, "artist_id": artist_id}), 201
|
||||||
|
|
||||||
|
|
||||||
|
# -- Discord: remove posts by people outside the poster list (#4486) --------------
|
||||||
|
|
||||||
|
_IMAGES_ROOT = Path("/images")
|
||||||
|
|
||||||
|
|
||||||
|
async def _poster_cleanup_preview(source_id: int):
|
||||||
|
async with get_session() as session:
|
||||||
|
return await session.run_sync(
|
||||||
|
lambda s: poster_cleanup.preview(s, source_id=source_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@sources_bp.route("/<int:source_id>/discord/other-posters", methods=["GET"])
|
||||||
|
async def other_posters_preview(source_id: int):
|
||||||
|
"""What removing other posters' posts would delete. Nothing is touched."""
|
||||||
|
try:
|
||||||
|
projection = await _poster_cleanup_preview(source_id)
|
||||||
|
except LookupError:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
except poster_cleanup.PosterCleanupError as exc:
|
||||||
|
return _bad("not_applicable", detail=str(exc), status=409)
|
||||||
|
projection["confirm_token"] = poster_cleanup.confirm_token(projection)
|
||||||
|
return jsonify(projection)
|
||||||
|
|
||||||
|
|
||||||
|
@sources_bp.route("/<int:source_id>/discord/other-posters/remove", methods=["POST"])
|
||||||
|
async def other_posters_remove(source_id: int):
|
||||||
|
"""Delete them. `confirm` must be the token of the preview the operator saw:
|
||||||
|
a poster list edited since then changes the set, and the token with it."""
|
||||||
|
body = await request.get_json(silent=True) or {}
|
||||||
|
try:
|
||||||
|
projection = await _poster_cleanup_preview(source_id)
|
||||||
|
except LookupError:
|
||||||
|
return _bad("not_found", status=404)
|
||||||
|
except poster_cleanup.PosterCleanupError as exc:
|
||||||
|
return _bad("not_applicable", detail=str(exc), status=409)
|
||||||
|
expected = poster_cleanup.confirm_token(projection)
|
||||||
|
if body.get("confirm") != expected:
|
||||||
|
return _bad("confirm_mismatch", expected=expected)
|
||||||
|
async with get_session() as session:
|
||||||
|
result = await session.run_sync(
|
||||||
|
lambda s: poster_cleanup.apply(s, source_id=source_id, images_root=_IMAGES_ROOT)
|
||||||
|
)
|
||||||
|
return jsonify(result)
|
||||||
|
|||||||
@@ -152,6 +152,8 @@ async def list_runs():
|
|||||||
queue=<name> filter to one queue
|
queue=<name> filter to one queue
|
||||||
status=<status> filter to one status (running/ok/error/timeout/retry)
|
status=<status> filter to one status (running/ok/error/timeout/retry)
|
||||||
task=<substr> case-insensitive substring match on task_name
|
task=<substr> case-insensitive substring match on task_name
|
||||||
|
celery_task_id=<id> exactly one run — how a page follows a job it
|
||||||
|
started without having to know its lane
|
||||||
limit=<int> default 50, max 200
|
limit=<int> default 50, max 200
|
||||||
before_id=<int> cursor for keyset pagination
|
before_id=<int> cursor for keyset pagination
|
||||||
|
|
||||||
@@ -167,6 +169,7 @@ async def list_runs():
|
|||||||
queue = request.args.get("queue")
|
queue = request.args.get("queue")
|
||||||
status = request.args.get("status")
|
status = request.args.get("status")
|
||||||
task = request.args.get("task")
|
task = request.args.get("task")
|
||||||
|
celery_task_id = request.args.get("celery_task_id")
|
||||||
before_id_raw = request.args.get("before_id")
|
before_id_raw = request.args.get("before_id")
|
||||||
before_id = int(before_id_raw) if before_id_raw else None
|
before_id = int(before_id_raw) if before_id_raw else None
|
||||||
|
|
||||||
@@ -176,6 +179,8 @@ async def list_runs():
|
|||||||
stmt = stmt.where(TaskRun.queue == queue)
|
stmt = stmt.where(TaskRun.queue == queue)
|
||||||
if status:
|
if status:
|
||||||
stmt = stmt.where(TaskRun.status == status)
|
stmt = stmt.where(TaskRun.status == status)
|
||||||
|
if celery_task_id:
|
||||||
|
stmt = stmt.where(TaskRun.celery_task_id == celery_task_id)
|
||||||
if task:
|
if task:
|
||||||
# Task names contain literal underscores (download_source,
|
# Task names contain literal underscores (download_source,
|
||||||
# vacuum_analyze) — escape LIKE wildcards so a search for
|
# vacuum_analyze) — escape LIKE wildcards so a search for
|
||||||
|
|||||||
@@ -72,9 +72,14 @@ assert STALE_AFTER_SECONDS >= SWEEP_PERIOD_SECONDS * _SWEEPS_BEFORE_STALE, (
|
|||||||
PROBE_TIMEOUT_SECONDS = 2.0
|
PROBE_TIMEOUT_SECONDS = 2.0
|
||||||
|
|
||||||
_OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
|
_OK, _STALE, _DOWN, _UNKNOWN = "ok", "stale", "down", "unknown"
|
||||||
|
# Checking in, but working at a fraction of its speed: a GPU agent whose
|
||||||
|
# runtimes fell back to the CPU (#4410). Below stale — a part that may have
|
||||||
|
# stopped is the more urgent question — and above unknown, because this one
|
||||||
|
# IS known to be wrong.
|
||||||
|
_DEGRADED = "degraded"
|
||||||
|
|
||||||
# Worst-first, so an overall verdict is just the max.
|
# Worst-first, so an overall verdict is just the max.
|
||||||
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _STALE: 2, _DOWN: 3}
|
_SEVERITY = {_OK: 0, _UNKNOWN: 1, _DEGRADED: 2, _STALE: 3, _DOWN: 4}
|
||||||
|
|
||||||
|
|
||||||
def _age_state(age_seconds: float) -> str:
|
def _age_state(age_seconds: float) -> str:
|
||||||
@@ -100,6 +105,39 @@ def _describe_learned(name: str, state: str, age: float, details: dict) -> str:
|
|||||||
return f"{name} has not checked in for {ago} — treat it as stopped"
|
return f"{name} has not checked in for {ago} — treat it as stopped"
|
||||||
|
|
||||||
|
|
||||||
|
def _cpu_runtimes(details: dict) -> list[str]:
|
||||||
|
"""The runtimes an agent reported as NOT on the GPU, with why.
|
||||||
|
|
||||||
|
Both torch and onnxruntime fall back to the CPU without raising, so an
|
||||||
|
agent in that state leases, works and checks in exactly like a healthy
|
||||||
|
one. On 2026-09-24 one had been doing so since a driver update left a
|
||||||
|
stale CDI spec; the only sign was a line in the agent's own log.
|
||||||
|
"""
|
||||||
|
accel = details.get("accel")
|
||||||
|
if not isinstance(accel, dict):
|
||||||
|
return []
|
||||||
|
out = []
|
||||||
|
for name, entry in sorted(accel.items()):
|
||||||
|
if not isinstance(entry, dict) or entry.get("device") == "cuda":
|
||||||
|
continue
|
||||||
|
why = entry.get("error") or entry.get("device") or "unknown"
|
||||||
|
out.append(f"{name} ({why})")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _learned_state(name: str, state: str, age: float, details: dict) -> tuple[str, str]:
|
||||||
|
"""A roster row's state and its sentence, degraded included."""
|
||||||
|
if state == _OK:
|
||||||
|
cpu = _cpu_runtimes(details)
|
||||||
|
if cpu:
|
||||||
|
return _DEGRADED, (
|
||||||
|
f"{name} is running on the CPU — not on the GPU: {'; '.join(cpu)}. "
|
||||||
|
"After a driver update, regenerate the agent host's CDI spec "
|
||||||
|
"(agent README)."
|
||||||
|
)
|
||||||
|
return state, _describe_learned(name, state, age, details)
|
||||||
|
|
||||||
|
|
||||||
async def _probe_postgres(session) -> dict:
|
async def _probe_postgres(session) -> dict:
|
||||||
started = time.monotonic()
|
started = time.monotonic()
|
||||||
try:
|
try:
|
||||||
@@ -175,13 +213,15 @@ async def system_health():
|
|||||||
).scalars().all()
|
).scalars().all()
|
||||||
for row in rows:
|
for row in rows:
|
||||||
age = (now - row.last_seen_at).total_seconds()
|
age = (now - row.last_seen_at).total_seconds()
|
||||||
state = _age_state(age)
|
state, detail = _learned_state(
|
||||||
|
row.display_name, _age_state(age), age, row.details or {},
|
||||||
|
)
|
||||||
parts.append({
|
parts.append({
|
||||||
"key": row.key,
|
"key": row.key,
|
||||||
"kind": row.kind,
|
"kind": row.kind,
|
||||||
"name": row.display_name,
|
"name": row.display_name,
|
||||||
"state": state,
|
"state": state,
|
||||||
"detail": _describe_learned(row.display_name, state, age, row.details or {}),
|
"detail": detail,
|
||||||
"last_seen_at": row.last_seen_at.isoformat(),
|
"last_seen_at": row.last_seen_at.isoformat(),
|
||||||
"first_seen_at": row.first_seen_at.isoformat(),
|
"first_seen_at": row.first_seen_at.isoformat(),
|
||||||
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
**{k: v for k, v in (row.details or {}).items() if k != "agent_id"},
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
"""Celery beat that remembers when each job last ran — from task_run, not a file.
|
||||||
|
|
||||||
|
Celery's default PersistentScheduler keeps its memory in a shelve file in the
|
||||||
|
working directory. Nothing mounts that directory, so every container recreate
|
||||||
|
forgets it, and a scheduler that remembers nothing seeds every entry with
|
||||||
|
`last_run_at = now`: each job waits a FULL interval after startup. A daily job
|
||||||
|
therefore needs 24 hours without a redeploy to fire. Since the one-container
|
||||||
|
image (172e33d) every redeploy restarts beat, and on 2026-09-24 no daily or
|
||||||
|
weekly job had run since the 21st (#4408).
|
||||||
|
|
||||||
|
task_run already records every task that starts (celery_signals), indexed on
|
||||||
|
(task_name, started_at DESC), and prune_task_runs keeps the newest row of each
|
||||||
|
task however old it is. So on startup each entry takes its last_run_at from
|
||||||
|
there:
|
||||||
|
- a job that is overdue runs at once;
|
||||||
|
- a job that is not due waits only the remainder of its interval;
|
||||||
|
- a job that has never run is due now.
|
||||||
|
|
||||||
|
Beat keeps last_run_at in memory from then on, as the default scheduler does;
|
||||||
|
only the startup seed changes. If the database cannot be read, the entries keep
|
||||||
|
Celery's own default rather than beat failing to start.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from celery.beat import Scheduler
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Seed for a job with no recorded run: far enough back that any interval or
|
||||||
|
# crontab reads as due.
|
||||||
|
NEVER = datetime(2000, 1, 1, tzinfo=UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def last_runs(session, task_names: list[str]) -> dict[str, datetime]:
|
||||||
|
"""The latest recorded start of each task, by task name."""
|
||||||
|
from .models import TaskRun
|
||||||
|
|
||||||
|
if not task_names:
|
||||||
|
return {}
|
||||||
|
rows = session.execute(
|
||||||
|
select(TaskRun.task_name, func.max(TaskRun.started_at))
|
||||||
|
.where(TaskRun.task_name.in_(sorted(set(task_names))))
|
||||||
|
.group_by(TaskRun.task_name)
|
||||||
|
).all()
|
||||||
|
return dict(rows)
|
||||||
|
|
||||||
|
|
||||||
|
def seed(entries, last: dict[str, datetime]) -> None:
|
||||||
|
"""Set each entry's last_run_at from `last`; a task with none is due now."""
|
||||||
|
for entry in entries:
|
||||||
|
entry.last_run_at = last.get(entry.task, NEVER)
|
||||||
|
|
||||||
|
|
||||||
|
class TaskRunScheduler(Scheduler):
|
||||||
|
"""An in-memory beat seeded from task_run history at startup."""
|
||||||
|
|
||||||
|
def setup_schedule(self):
|
||||||
|
super().setup_schedule()
|
||||||
|
try:
|
||||||
|
from .tasks._sync_engine import sync_session_factory
|
||||||
|
|
||||||
|
with sync_session_factory()() as session:
|
||||||
|
last = last_runs(session, [e.task for e in self.schedule.values()])
|
||||||
|
except Exception:
|
||||||
|
log.exception("beat: could not read task_run; every job waits a full interval")
|
||||||
|
return
|
||||||
|
seed(self.schedule.values(), last)
|
||||||
|
due = sum(1 for e in self.schedule.values() if e.is_due()[0])
|
||||||
|
log.info(
|
||||||
|
"beat: seeded %d job(s) from task_run, %d due now",
|
||||||
|
len(self.schedule), due,
|
||||||
|
)
|
||||||
@@ -62,6 +62,15 @@ def make_celery() -> Celery:
|
|||||||
# can never starve the quick self-healing sweeps (operator-flagged
|
# can never starve the quick self-healing sweeps (operator-flagged
|
||||||
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
# 2026-06-07: a 2h audit blocked vacuum/backup/normalize for hours).
|
||||||
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
"backend.app.tasks.maintenance.*": {"queue": "maintenance"},
|
||||||
|
# The one long job in maintenance.py: a whole-library phash
|
||||||
|
# recompute (35 min hard limit; the library was cleared for
|
||||||
|
# re-hashing by migration 0098). On the quick lane it held a
|
||||||
|
# scheduler process for its whole run, and the minute ticks queued
|
||||||
|
# up behind it (2026-09-24: 7 waiting, "all workers busy for 18
|
||||||
|
# minutes"). An exact name wins over the glob above.
|
||||||
|
"backend.app.tasks.maintenance.backfill_phash": {"queue": "maintenance_long"},
|
||||||
|
# Walks a folder tree per artist with undated posts (#4436).
|
||||||
|
"backend.app.tasks.maintenance.date_posts_from_records": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.backup.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.admin.*": {"queue": "maintenance_long"},
|
||||||
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
"backend.app.tasks.library_audit.*": {"queue": "maintenance_long"},
|
||||||
@@ -149,6 +158,11 @@ def make_celery() -> Celery:
|
|||||||
"schedule": 86400.0, # daily — sweep .part/.partial left by a
|
"schedule": 86400.0, # daily — sweep .part/.partial left by a
|
||||||
# download/import killed mid-write (graceful-shutdown fallout)
|
# download/import killed mid-write (graceful-shutdown fallout)
|
||||||
},
|
},
|
||||||
|
"date-posts-from-records-hourly": {
|
||||||
|
"task": "backend.app.tasks.maintenance.date_posts_from_records",
|
||||||
|
"schedule": 3600.0, # an empty query once every native post is
|
||||||
|
# dated; otherwise upserts the _post.json a killed walk left (#4436)
|
||||||
|
},
|
||||||
"backfill-phash-daily": {
|
"backfill-phash-daily": {
|
||||||
"task": "backend.app.tasks.maintenance.backfill_phash",
|
"task": "backend.app.tasks.maintenance.backfill_phash",
|
||||||
"schedule": 86400.0, # daily — NULL-only, so a no-op once the
|
"schedule": 86400.0, # daily — NULL-only, so a no-op once the
|
||||||
@@ -216,11 +230,6 @@ def make_celery() -> Celery:
|
|||||||
"schedule": 86400.0, # auto-tag wip/editor process art (#1464);
|
"schedule": 86400.0, # auto-tag wip/editor process art (#1464);
|
||||||
# no-op unless process_auto_apply_enabled (opt-in)
|
# no-op unless process_auto_apply_enabled (opt-in)
|
||||||
},
|
},
|
||||||
"soft-wip-conflict-audit-daily": {
|
|
||||||
"task": "backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
|
||||||
"schedule": 86400.0, # flag ring-loud soft-WIP (sketch/doodle) tags
|
|
||||||
# for review (#1474); no-op with no content heads
|
|
||||||
},
|
|
||||||
"prune-presentation-reviews-daily": {
|
"prune-presentation-reviews-daily": {
|
||||||
"task": "backend.app.tasks.ml.prune_presentation_reviews",
|
"task": "backend.app.tasks.ml.prune_presentation_reviews",
|
||||||
"schedule": 86400.0, # retention: drop resolved review flags >30d
|
"schedule": 86400.0, # retention: drop resolved review flags >30d
|
||||||
@@ -353,6 +362,9 @@ def make_celery() -> Celery:
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
timezone="UTC",
|
timezone="UTC",
|
||||||
|
# Beat's memory of when each job last ran comes from task_run, not a
|
||||||
|
# shelve file nothing persists — see beat_scheduler (#4408).
|
||||||
|
beat_scheduler="backend.app.beat_scheduler:TaskRunScheduler",
|
||||||
)
|
)
|
||||||
# FC-3i: register task_run signal handlers (side-effect import).
|
# FC-3i: register task_run signal handlers (side-effect import).
|
||||||
from . import celery_signals # noqa: F401
|
from . import celery_signals # noqa: F401
|
||||||
|
|||||||
@@ -18,11 +18,18 @@ dark for that interval. Monitoring NEVER breaks the thing it's
|
|||||||
monitoring.
|
monitoring.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
import functools
|
||||||
import logging
|
import logging
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
from celery.exceptions import SoftTimeLimitExceeded
|
from celery.exceptions import SoftTimeLimitExceeded
|
||||||
from celery.signals import task_failure, task_postrun, task_prerun, task_retry
|
from celery.signals import (
|
||||||
|
task_failure,
|
||||||
|
task_postrun,
|
||||||
|
task_prerun,
|
||||||
|
task_retry,
|
||||||
|
worker_ready,
|
||||||
|
)
|
||||||
|
|
||||||
from .models import TaskRun
|
from .models import TaskRun
|
||||||
from .tasks._sync_engine import sync_session_factory
|
from .tasks._sync_engine import sync_session_factory
|
||||||
@@ -53,42 +60,29 @@ _INT32_MIN = -2_147_483_648
|
|||||||
|
|
||||||
|
|
||||||
def _queue_for(task) -> str:
|
def _queue_for(task) -> str:
|
||||||
"""Reverse the task→queue routing from celery_app.task_routes.
|
"""The queue Celery routes this task to — asked of the router itself.
|
||||||
Keep in sync if task_routes is reordered.
|
|
||||||
|
|
||||||
Audit 2026-06-02: backup/admin/library_audit prefixes were
|
This was a hand-kept copy of `celery_app.task_routes`, and it drifted
|
||||||
missing here even though task_routes sent all three to
|
twice (the 2026-06-02 audit, then #4432). Long-lane jobs were recorded as
|
||||||
'maintenance'. The TaskRun.queue column then lied for those
|
`maintenance`, and translation and gpu_queue runs as `default`, where the
|
||||||
rows (claimed 'default') so per-queue dashboard filters and
|
5-minute stall sweep failed healthy 35-minute translation runs. The router
|
||||||
per-queue threshold overrides silently missed them.
|
answers from the same table the broker uses, so the two cannot disagree.
|
||||||
"""
|
"""
|
||||||
name = getattr(task, "name", "") or ""
|
name = getattr(task, "name", "") or ""
|
||||||
if name.startswith("backend.app.tasks.import_file."):
|
app = getattr(task, "app", None)
|
||||||
return "import"
|
if app is None:
|
||||||
if name.startswith("backend.app.tasks.ml."):
|
from .celery_app import celery as app
|
||||||
return "ml"
|
return _routed_queue(app, name)
|
||||||
if name.startswith("backend.app.tasks.thumbnail."):
|
|
||||||
return "thumbnail"
|
|
||||||
if name.startswith((
|
@functools.lru_cache(maxsize=1024)
|
||||||
"backend.app.tasks.download.",
|
def _routed_queue(app, name: str) -> str:
|
||||||
# External file-host fetches share the download lane (celery_app
|
try:
|
||||||
# routes external.* → download). Mirror it here or TaskRun.queue
|
queue = app.amqp.router.route({}, name).get("queue")
|
||||||
# lies 'default' for them, so per-queue dashboard filters and the
|
except Exception: # noqa: BLE001 — monitoring never breaks the task
|
||||||
# per-queue threshold override miss them — the same gap the
|
log.warning("task_run: could not resolve the queue for %s", name)
|
||||||
# 2026-06-02 audit fixed for backup/admin/library_audit.
|
return "default"
|
||||||
"backend.app.tasks.external.",
|
return getattr(queue, "name", None) or (queue if isinstance(queue, str) else "default")
|
||||||
)):
|
|
||||||
return "download"
|
|
||||||
if name.startswith("backend.app.tasks.scan."):
|
|
||||||
return "scan"
|
|
||||||
if name.startswith((
|
|
||||||
"backend.app.tasks.maintenance.",
|
|
||||||
"backend.app.tasks.backup.",
|
|
||||||
"backend.app.tasks.admin.",
|
|
||||||
"backend.app.tasks.library_audit.",
|
|
||||||
)):
|
|
||||||
return "maintenance"
|
|
||||||
return "default"
|
|
||||||
|
|
||||||
|
|
||||||
def _target_id_from_args(args) -> int | None:
|
def _target_id_from_args(args) -> int | None:
|
||||||
@@ -225,3 +219,43 @@ def _on_retry(sender=None, request=None, reason=None, einfo=None, **_):
|
|||||||
error_message=str(reason) if reason else None,
|
error_message=str(reason) if reason else None,
|
||||||
retry_count=getattr(request, "retries", 0),
|
retry_count=getattr(request, "retries", 0),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _consumed_queues(consumer) -> set[str]:
|
||||||
|
"""The queue names this worker process consumes (its `-Q`), or empty when
|
||||||
|
they can't be read — which makes the boot hook below a no-op, not a guess."""
|
||||||
|
try:
|
||||||
|
return {q.name for q in consumer.task_consumer.queues}
|
||||||
|
except Exception: # noqa: BLE001 — shape varies across celery versions
|
||||||
|
return set()
|
||||||
|
|
||||||
|
|
||||||
|
@worker_ready.connect
|
||||||
|
def _on_worker_ready(sender=None, **_):
|
||||||
|
"""The download lane clears what its previous process left behind (#4433).
|
||||||
|
|
||||||
|
A restart SIGKILLs any walk that outlives the stop grace, so its event is
|
||||||
|
never finalized and its platform lock is never released. Both used to wait
|
||||||
|
out timers — a 30-min stall sweep that then blamed the source, and a 27-min
|
||||||
|
lock TTL that stalled every other source on the platform. Only the process
|
||||||
|
consuming `download` does this; the other lanes booting beside it must not.
|
||||||
|
Best-effort: a failure here is logged and the worker still starts.
|
||||||
|
"""
|
||||||
|
if "download" not in _consumed_queues(sender):
|
||||||
|
return
|
||||||
|
booted_at = datetime.now(UTC)
|
||||||
|
try:
|
||||||
|
from .services.download_recovery import interrupt_orphaned_download_events
|
||||||
|
from .services.platform_lock import release_all_platform_locks
|
||||||
|
|
||||||
|
with sync_session_factory()() as session:
|
||||||
|
closed = interrupt_orphaned_download_events(session, booted_at=booted_at)
|
||||||
|
session.commit()
|
||||||
|
released = release_all_platform_locks()
|
||||||
|
if closed or released:
|
||||||
|
log.info(
|
||||||
|
"download lane boot: closed %d orphaned download event(s) as "
|
||||||
|
"interrupted, released %d platform lock(s)", closed, released,
|
||||||
|
)
|
||||||
|
except Exception: # noqa: BLE001 — never block the worker from starting
|
||||||
|
log.exception("download lane boot recovery failed")
|
||||||
|
|||||||
+20
-2
@@ -46,6 +46,14 @@ async def serve_extension(filename: str):
|
|||||||
|
|
||||||
The application/x-xpinstall MIME tells Firefox to show its native
|
The application/x-xpinstall MIME tells Firefox to show its native
|
||||||
install prompt instead of downloading the file as a blob.
|
install prompt instead of downloading the file as a blob.
|
||||||
|
|
||||||
|
Caching differs by name, and has to. A versioned name is one build's bytes
|
||||||
|
forever, so it can be cached for good. `fabledcurator-latest.xpi` is ONE
|
||||||
|
URL whose bytes change on every release, and Quart's default for a file
|
||||||
|
is `public, max-age=43200`: a browser that fetched it once reused those
|
||||||
|
bytes for 12 hours, so "install the latest" quietly reinstalled the
|
||||||
|
previous build (operator-flagged 2026-09-25). It is `no-cache` — the ETag
|
||||||
|
still makes an unchanged file a cheap 304.
|
||||||
"""
|
"""
|
||||||
if not _XPI_NAME_RE.fullmatch(filename):
|
if not _XPI_NAME_RE.fullmatch(filename):
|
||||||
abort(404)
|
abort(404)
|
||||||
@@ -56,10 +64,11 @@ async def serve_extension(filename: str):
|
|||||||
if not xpis:
|
if not xpis:
|
||||||
abort(404)
|
abort(404)
|
||||||
latest = xpis[-1]
|
latest = xpis[-1]
|
||||||
return await send_file(
|
resp = await send_file(
|
||||||
latest, mimetype="application/x-xpinstall",
|
latest, mimetype="application/x-xpinstall",
|
||||||
attachment_filename=latest.name,
|
attachment_filename=latest.name,
|
||||||
)
|
)
|
||||||
|
return _cache(resp, "no-cache")
|
||||||
target = (XPI_DIR / filename).resolve()
|
target = (XPI_DIR / filename).resolve()
|
||||||
try:
|
try:
|
||||||
target.relative_to(XPI_DIR)
|
target.relative_to(XPI_DIR)
|
||||||
@@ -67,10 +76,19 @@ async def serve_extension(filename: str):
|
|||||||
abort(404)
|
abort(404)
|
||||||
if not target.is_file():
|
if not target.is_file():
|
||||||
abort(404)
|
abort(404)
|
||||||
return await send_file(
|
resp = await send_file(
|
||||||
target, mimetype="application/x-xpinstall",
|
target, mimetype="application/x-xpinstall",
|
||||||
attachment_filename=filename,
|
attachment_filename=filename,
|
||||||
)
|
)
|
||||||
|
return _cache(resp, "public, max-age=31536000, immutable")
|
||||||
|
|
||||||
|
|
||||||
|
def _cache(resp, policy: str):
|
||||||
|
"""Set the XPI's Cache-Control, dropping the Expires send_file adds so the
|
||||||
|
two can never disagree."""
|
||||||
|
resp.headers["Cache-Control"] = policy
|
||||||
|
resp.headers.pop("Expires", None)
|
||||||
|
return resp
|
||||||
|
|
||||||
|
|
||||||
@frontend_bp.route("/")
|
@frontend_bp.route("/")
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ from .backup_run import BackupRun
|
|||||||
from .base import Base
|
from .base import Base
|
||||||
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
from .character_prototype import CcipPrototypeState, CharacterPrototype
|
||||||
from .credential import Credential
|
from .credential import Credential
|
||||||
|
from .discord_failed_media import DiscordFailedMedia
|
||||||
|
from .discord_seen_media import DiscordSeenMedia
|
||||||
from .download_event import DownloadEvent
|
from .download_event import DownloadEvent
|
||||||
from .external_link import ExternalLink
|
from .external_link import ExternalLink
|
||||||
from .gpu_job import GpuJob
|
from .gpu_job import GpuJob
|
||||||
@@ -56,6 +58,8 @@ __all__ = [
|
|||||||
"BackupRun",
|
"BackupRun",
|
||||||
"Source",
|
"Source",
|
||||||
"Credential",
|
"Credential",
|
||||||
|
"DiscordFailedMedia",
|
||||||
|
"DiscordSeenMedia",
|
||||||
"PatreonFailedMedia",
|
"PatreonFailedMedia",
|
||||||
"PatreonSeenMedia",
|
"PatreonSeenMedia",
|
||||||
"SubscribeStarFailedMedia",
|
"SubscribeStarFailedMedia",
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
"""DiscordFailedMedia — per-source dead-letter ledger of Discord files that
|
||||||
|
keep failing to download or validate.
|
||||||
|
|
||||||
|
Mirror of SubscribeStarFailedMedia. After `attempts` reaches the dead-letter
|
||||||
|
threshold a routine walk skips the file (recovery still retries it); a later
|
||||||
|
clean download clears the row. `filehash` is the seen-ledger's key.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import DateTime
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordFailedMedia(Base):
|
||||||
|
__tablename__ = "discord_failed_media"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_id", "filehash", name="uq_discord_failed_media_source_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default="1")
|
||||||
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
first_failed_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
|
last_failed_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
"""DiscordSeenMedia — per-source ledger of Discord files already downloaded.
|
||||||
|
|
||||||
|
Mirror of SubscribeStarSeenMedia for the native Discord ingester (milestone
|
||||||
|
428). `filehash` holds the ingester's per-file key, `<message_id>:<media_id>`:
|
||||||
|
the attachment id, or for an embed a hash of its URL path. Not the file's
|
||||||
|
position in the message — an edit that removes a file renumbers the rest
|
||||||
|
(see `discord_client.MediaItem`). The message record's own gate is the
|
||||||
|
synthetic `message:<id>` key in the same column.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from sqlalchemy import ForeignKey, Integer, String, UniqueConstraint, func
|
||||||
|
from sqlalchemy.orm import Mapped, mapped_column
|
||||||
|
from sqlalchemy.types import DateTime
|
||||||
|
|
||||||
|
from .base import Base
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordSeenMedia(Base):
|
||||||
|
__tablename__ = "discord_seen_media"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_id", "filehash", name="uq_discord_seen_media_source_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||||
|
source_id: Mapped[int] = mapped_column(
|
||||||
|
ForeignKey("source.id", ondelete="CASCADE"), nullable=False, index=True
|
||||||
|
)
|
||||||
|
filehash: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||||
|
post_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
seen_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True), nullable=False, server_default=func.now()
|
||||||
|
)
|
||||||
@@ -39,7 +39,7 @@ class ImportSettings(Base):
|
|||||||
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
|
transparency_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.9, server_default="0.9")
|
||||||
|
|
||||||
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
skip_single_color: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="false")
|
||||||
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.95, server_default="0.95")
|
single_color_threshold: Mapped[float] = mapped_column(Float, nullable=False, default=0.995, server_default="0.995")
|
||||||
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
|
single_color_tolerance: Mapped[int] = mapped_column(Integer, nullable=False, default=30, server_default="30")
|
||||||
|
|
||||||
# Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The
|
# Hamming distance over a 256-bit pHash (utils.phash, hash_size=16). The
|
||||||
@@ -238,13 +238,6 @@ class ImportSettings(Base):
|
|||||||
wip_title_tagging_enabled: Mapped[bool] = mapped_column(
|
wip_title_tagging_enabled: Mapped[bool] = mapped_column(
|
||||||
Boolean, nullable=False, default=True, server_default="true",
|
Boolean, nullable=False, default=True, server_default="true",
|
||||||
)
|
)
|
||||||
# Soft WIP title tier (#1474): also tag sketch/doodle/scribble titles, but with
|
|
||||||
# a PROVISIONAL source (`wip_title_soft`) that never trains the head, since these
|
|
||||||
# are lower-precision (a finished "sketch" isn't WIP). OFF by default — a lower-
|
|
||||||
# precision tier is opt-in (the ring-loud audit surfaces false positives).
|
|
||||||
wip_soft_title_tagging_enabled: Mapped[bool] = mapped_column(
|
|
||||||
Boolean, nullable=False, default=False, server_default="false",
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
async def load(cls, session) -> ImportSettings:
|
async def load(cls, session) -> ImportSettings:
|
||||||
|
|||||||
@@ -46,6 +46,11 @@ class Source(Base):
|
|||||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True, server_default="true")
|
||||||
|
|
||||||
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
config_overrides: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
# alembic 0115: what the source is called on its platform, where the URL
|
||||||
|
# alone doesn't say — a Discord link is two numbers. The ingester that
|
||||||
|
# walks it refreshes it every walk, so a rename reads as the new name.
|
||||||
|
# NULL until a walk has read it, and on platforms that don't write it.
|
||||||
|
display_name: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|
||||||
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
last_checked_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
|||||||
@@ -316,17 +316,32 @@ class ArtistService:
|
|||||||
cleaned = (prefix or "").strip()
|
cleaned = (prefix or "").strip()
|
||||||
if not cleaned:
|
if not cleaned:
|
||||||
return []
|
return []
|
||||||
like = f"%{cleaned.lower()}%"
|
low = cleaned.lower()
|
||||||
prefix_like = f"{cleaned.lower()}%"
|
like = f"%{low}%"
|
||||||
# Rank: exact (0) < prefix (1) < substring (2).
|
prefix_like = f"{low}%"
|
||||||
|
# Spacing- and punctuation-insensitive too, so "Tamada Heijun" finds
|
||||||
|
# "TamadaHeijun" and "sabu_art" finds "Sabu Art" — the same creator is
|
||||||
|
# spelled differently on every platform, and the browser extension's
|
||||||
|
# Add panel matches a Discord server name against these (milestone 429).
|
||||||
|
# [[:alnum:]] keeps non-Latin letters; a query with none (all
|
||||||
|
# punctuation) skips this arm rather than matching every artist.
|
||||||
|
squashed = "".join(ch for ch in low if ch.isalnum())
|
||||||
|
name_squashed = func.regexp_replace(func.lower(Artist.name), "[^[:alnum:]]", "", "g")
|
||||||
|
matches = [func.lower(Artist.name).like(like)]
|
||||||
|
if squashed:
|
||||||
|
matches.append(name_squashed.like(f"%{squashed}%"))
|
||||||
|
# Rank: exact (0) < exact ignoring spacing (1) < prefix (2) <
|
||||||
|
# substring (3) < substring ignoring spacing (4).
|
||||||
rank = case(
|
rank = case(
|
||||||
(func.lower(Artist.name) == cleaned.lower(), 0),
|
(func.lower(Artist.name) == low, 0),
|
||||||
(func.lower(Artist.name).like(prefix_like), 1),
|
(name_squashed == squashed, 1),
|
||||||
else_=2,
|
(func.lower(Artist.name).like(prefix_like), 2),
|
||||||
|
(func.lower(Artist.name).like(like), 3),
|
||||||
|
else_=4,
|
||||||
).label("rank")
|
).label("rank")
|
||||||
rows = (await self.session.execute(
|
rows = (await self.session.execute(
|
||||||
select(Artist, rank)
|
select(Artist, rank)
|
||||||
.where(func.lower(Artist.name).like(like))
|
.where(or_(*matches))
|
||||||
.order_by(rank, Artist.name.asc())
|
.order_by(rank, Artist.name.asc())
|
||||||
.limit(limit)
|
.limit(limit)
|
||||||
)).all()
|
)).all()
|
||||||
|
|||||||
@@ -4,11 +4,19 @@ predicate for BOTH surfaces: FC-Cleanup's retroactive audit and — since
|
|||||||
2026-07-02 — the import-side filter (Importer._single_color_hit /
|
2026-07-02 — the import-side filter (Importer._single_color_hit /
|
||||||
SkipReason.single_color), so what the audit flags and what the import
|
SkipReason.single_color), so what the audit flags and what the import
|
||||||
skips can never disagree.
|
skips can never disagree.
|
||||||
|
|
||||||
|
It is meant to catch the blank: a placeholder, an error tile, a solid fill.
|
||||||
|
Line art is the case it must not catch (#4483): a pencil doodle on white can be
|
||||||
|
well over 95% white even at full size, and a smoothing downsample blends its strokes
|
||||||
|
into the paper until it measures as blank. So the sample is taken by NEAREST
|
||||||
|
(each sampled pixel is a real pixel, strokes keep their contrast), at 256px
|
||||||
|
so thin strokes are still hit, and the default threshold is near-total
|
||||||
|
(0.995) — a few percent of ink is a drawing, not an empty image.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
_THUMB_SIZE = (64, 64)
|
_THUMB_SIZE = (256, 256)
|
||||||
|
|
||||||
|
|
||||||
def evaluate(
|
def evaluate(
|
||||||
@@ -20,7 +28,8 @@ def evaluate(
|
|||||||
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
"""True iff the fraction of pixels within `tolerance` (Euclidean RGB
|
||||||
distance) of the dominant color exceeds `threshold`.
|
distance) of the dominant color exceeds `threshold`.
|
||||||
|
|
||||||
Downsamples to 64x64 for speed (~4ms regardless of source size).
|
Samples 256x256 by NEAREST (see the module docstring for why not a
|
||||||
|
smoothing resample).
|
||||||
Alpha channels are stripped; only RGB is considered. Animated images
|
Alpha channels are stripped; only RGB is considered. Animated images
|
||||||
use frame 0 (PIL's default after Image.open without seek).
|
use frame 0 (PIL's default after Image.open without seek).
|
||||||
"""
|
"""
|
||||||
@@ -30,7 +39,7 @@ def evaluate(
|
|||||||
elif im.mode not in ("RGB", "L"):
|
elif im.mode not in ("RGB", "L"):
|
||||||
im = im.convert("RGB")
|
im = im.convert("RGB")
|
||||||
if im.size != _THUMB_SIZE:
|
if im.size != _THUMB_SIZE:
|
||||||
im = im.resize(_THUMB_SIZE, Image.Resampling.BILINEAR)
|
im = im.resize(_THUMB_SIZE, Image.Resampling.NEAREST)
|
||||||
pixels = list(im.getdata())
|
pixels = list(im.getdata())
|
||||||
if not pixels:
|
if not pixels:
|
||||||
return False
|
return False
|
||||||
|
|||||||
@@ -0,0 +1,752 @@
|
|||||||
|
"""Native Discord read client — the Discord counterpart to subscribestar_client.
|
||||||
|
|
||||||
|
Mirrors gallery-dl 1.32.13's `extractor/discord.py` (rule 130: gallery-dl is the
|
||||||
|
known-working base), adapted to the native core's client contract
|
||||||
|
(`ingest_core` module docstring): `iter_posts` / `extract_media`, plus the
|
||||||
|
post-first `post_record_key` and the `post_meta` date the revisit window reads.
|
||||||
|
|
||||||
|
What is mirrored exactly, because drift in any of it changes what we fetch or
|
||||||
|
where it lands on disk:
|
||||||
|
|
||||||
|
- API v10, `Authorization: <user token>` (a USER token, not a bot token).
|
||||||
|
- gallery-dl's request profile: its date-derived Firefox User-Agent,
|
||||||
|
`Accept: */*`, `Accept-Language`, `Referer: https://discord.com/`.
|
||||||
|
- `GET /channels/{id}/messages?limit=100&before=<last id>`, newest first,
|
||||||
|
stopping on a short page. Message types {0, 19, 21} only.
|
||||||
|
- The walk: a text/news channel's own messages then its threads, a forum's
|
||||||
|
threads, a category's children, a server's text/news/forum channels.
|
||||||
|
- Files: attachments, then embeds of type image/gifv/video (FC configures
|
||||||
|
`embeds: all`, which for files is the same three plus rich/link embeds that
|
||||||
|
carry an image), then forwarded `message_snapshots`, numbered from 1 across
|
||||||
|
the lot — the `num` in `{date}_{message_id}_{num}_{filename}`.
|
||||||
|
- Text: `content`, rich-embed author/title/description/fields/footer, poll.
|
||||||
|
|
||||||
|
Two deliberate departures, both about the walk order, neither about content:
|
||||||
|
|
||||||
|
- Threads are walked newest-CREATED first (by id), not by last-message time.
|
||||||
|
A backfill resumes from a checkpointed channel; last-message order shifts
|
||||||
|
between chunks whenever someone posts, which can move an unwalked thread
|
||||||
|
above the resume point and skip it. Creation order only ever grows at the
|
||||||
|
front, where the next tick finds it.
|
||||||
|
- A 403 on a thread or a nested channel skips that feed instead of failing
|
||||||
|
the walk. gallery-dl skips only nested channels; one private thread the
|
||||||
|
token cannot read would otherwise stop every channel after it.
|
||||||
|
|
||||||
|
And one about content: a message is taken only if it has an image or video
|
||||||
|
ATTACHMENT (`has_visual_attachment`). Discord is where creators chat as well
|
||||||
|
as post, and the operator wants the art, not the conversation (2026-09-28): a
|
||||||
|
text line, a lone archive or PSD, a pasted link's preview or a Tenor GIF is
|
||||||
|
chat. A message that passes keeps every file gallery-dl would take, numbered
|
||||||
|
as gallery-dl numbers them, so the on-disk names still match.
|
||||||
|
|
||||||
|
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import date
|
||||||
|
from urllib.parse import unquote
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from .native_ingest_common import (
|
||||||
|
NativeAuthError,
|
||||||
|
NativeDriftError,
|
||||||
|
NativeIngestError,
|
||||||
|
retry_after_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
API_ROOT = "https://discord.com/api/v10"
|
||||||
|
_ROOT = "https://discord.com"
|
||||||
|
|
||||||
|
_TIMEOUT_SECONDS = 60.0
|
||||||
|
_MESSAGES_BATCH = 100
|
||||||
|
_THREADS_BATCH = 25
|
||||||
|
# gallery-dl retries a 429 up to its default 4 retries, waiting
|
||||||
|
# `request_interval_429` (60s) between them. Discord's Retry-After is exact, so
|
||||||
|
# it is honoured when present; 60s is the fallback and the cap.
|
||||||
|
_MAX_429_RETRIES = 4
|
||||||
|
_429_WAIT_SECONDS = 60.0
|
||||||
|
# How far back the poster picker looks: two pages. Enough to see who posts in
|
||||||
|
# a channel, few enough to answer while the operator waits (#4491).
|
||||||
|
_POSTER_SCAN = 200
|
||||||
|
|
||||||
|
# https://discord.com/developers/docs/resources/message#message-object-message-types
|
||||||
|
# DEFAULT, REPLY, CHAT_INPUT_COMMAND — the ones that carry user content.
|
||||||
|
MESSAGE_TYPES = frozenset({0, 19, 21})
|
||||||
|
# https://discord.com/developers/docs/resources/channel#channel-object-channel-types
|
||||||
|
_TEXT = frozenset({0, 5}) # text, announcement: messages + threads
|
||||||
|
_DIRECT = frozenset({1, 3, 10, 11, 12}) # DMs and threads: messages only
|
||||||
|
_FORUM = frozenset({15, 16}) # forum, media: threads only
|
||||||
|
_CATEGORY = 4
|
||||||
|
_SERVER_WALK = _TEXT | _FORUM
|
||||||
|
_EMBED_TYPES = frozenset({"image", "gifv", "video"})
|
||||||
|
# What makes a message art rather than chat: an attached file FC imports as an
|
||||||
|
# image or video. Mirrors importer.ALL_EXTS (not imported: that module is the
|
||||||
|
# import pipeline, and this one is a network client).
|
||||||
|
_VISUAL_EXTS = frozenset({
|
||||||
|
"png", "jpg", "jpeg", "gif", "webp", "bmp", "tif", "tiff",
|
||||||
|
"mp4", "mov", "avi", "mkv", "webm", "m4v", "wmv", "flv",
|
||||||
|
})
|
||||||
|
|
||||||
|
_URL_RE = re.compile(
|
||||||
|
r"^(?:https?://)?(?:www\.|ptb\.|canary\.)?discord(?:app)?\.com/channels/"
|
||||||
|
r"(?P<server>@me|\d+)(?:/(?:\d+/threads/)?(?P<channel>\d+))?(?P<rest>/.*)?/?$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordAPIError(NativeIngestError):
|
||||||
|
"""Base for native Discord client failures."""
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordAuthError(DiscordAPIError, NativeAuthError):
|
||||||
|
"""401 (the token is invalid or expired) or a 403 on the channel the
|
||||||
|
source names. The fix is a new token, not a new client."""
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordDriftError(DiscordAPIError, NativeDriftError):
|
||||||
|
"""A response did not have the shape the walk depends on."""
|
||||||
|
|
||||||
|
|
||||||
|
def firefox_user_agent(today: date | None = None) -> str:
|
||||||
|
"""gallery-dl's default User-Agent: a Firefox whose version advances every
|
||||||
|
four weeks (`util._ff_ver`, "147 on 2026-01-13"). Computed the same way so
|
||||||
|
the profile keeps matching the gallery-dl this replaced."""
|
||||||
|
ver = ((today or date.today()).toordinal() - 735_513) // 28
|
||||||
|
return (
|
||||||
|
f"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:{ver}.0) "
|
||||||
|
f"Gecko/20100101 Firefox/{ver}.0"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def nameext_from_url(url: str) -> tuple[str, str]:
|
||||||
|
"""gallery-dl's `text.nameext_from_url`: the URL's last path segment,
|
||||||
|
unquoted, split at the last dot when the extension is at most 16 chars
|
||||||
|
(lowercased); otherwise the whole name and no extension."""
|
||||||
|
filename = unquote(url.partition("?")[0].rpartition("/")[2])
|
||||||
|
name, _, ext = filename.rpartition(".")
|
||||||
|
if name and len(ext) <= 16:
|
||||||
|
return name, ext.lower()
|
||||||
|
return filename, ""
|
||||||
|
|
||||||
|
|
||||||
|
def parse_source_url(url: str) -> tuple[str | None, str | None]:
|
||||||
|
"""`(server_id, channel_id)` from a Discord channel/server URL. `server_id`
|
||||||
|
is None for a DM (`@me`); `channel_id` is None for a whole server. Raises
|
||||||
|
DiscordAPIError for anything else, including a link to a single message —
|
||||||
|
a message is not something a source can subscribe to."""
|
||||||
|
m = _URL_RE.match((url or "").strip())
|
||||||
|
if not m or (m.group("rest") or "").strip("/"):
|
||||||
|
raise DiscordAPIError(
|
||||||
|
f"Not a Discord channel or server link: {url!r} "
|
||||||
|
"(expected https://discord.com/channels/<server>[/<channel>])"
|
||||||
|
)
|
||||||
|
server = m.group("server")
|
||||||
|
channel = m.group("channel")
|
||||||
|
if server == "@me":
|
||||||
|
if not channel:
|
||||||
|
raise DiscordAPIError(f"A DM link needs a channel id: {url!r}")
|
||||||
|
return None, channel
|
||||||
|
return server, channel
|
||||||
|
|
||||||
|
|
||||||
|
def message_text(message: dict) -> str:
|
||||||
|
"""gallery-dl's `extract_message_text`: the body plus the text of rich
|
||||||
|
embeds and polls, newline-joined, empties dropped."""
|
||||||
|
parts = [message.get("content") or ""]
|
||||||
|
for embed in message.get("embeds") or []:
|
||||||
|
if embed.get("type") != "rich":
|
||||||
|
continue
|
||||||
|
parts.append((embed.get("author") or {}).get("name") or "")
|
||||||
|
parts.append(embed.get("title") or "")
|
||||||
|
parts.append(embed.get("description") or "")
|
||||||
|
for fld in embed.get("fields") or []:
|
||||||
|
parts.append(fld.get("name") or "")
|
||||||
|
parts.append(fld.get("value") or "")
|
||||||
|
parts.append((embed.get("footer") or {}).get("text") or "")
|
||||||
|
poll = message.get("poll")
|
||||||
|
if poll:
|
||||||
|
parts.append(((poll.get("question") or {}).get("text")) or "")
|
||||||
|
for answer in poll.get("answers") or []:
|
||||||
|
parts.append(((answer.get("poll_media") or {}).get("text")) or "")
|
||||||
|
return "\n".join(p for p in parts if p)
|
||||||
|
|
||||||
|
|
||||||
|
def _message_and_snapshots(message: dict) -> list[dict]:
|
||||||
|
"""The message itself, then each forwarded message it carries."""
|
||||||
|
return [message] + [
|
||||||
|
(s or {}).get("message") or {}
|
||||||
|
for s in message.get("message_snapshots") or []
|
||||||
|
if ((s or {}).get("message") or {}).get("type", 0) in MESSAGE_TYPES
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _is_visual(attachment: dict) -> bool:
|
||||||
|
ctype = (attachment.get("content_type") or "").lower()
|
||||||
|
if ctype.startswith(("image/", "video/")):
|
||||||
|
return True
|
||||||
|
name = attachment.get("filename") or attachment.get("url") or ""
|
||||||
|
return nameext_from_url(name)[1] in _VISUAL_EXTS
|
||||||
|
|
||||||
|
|
||||||
|
def has_visual_attachment(message: dict) -> bool:
|
||||||
|
"""Does this message, or a message it forwards, have an image or video
|
||||||
|
attached? The one test for "art, not chat" — see the module docstring."""
|
||||||
|
return any(
|
||||||
|
att.get("url") and _is_visual(att)
|
||||||
|
for snap in _message_and_snapshots(message)
|
||||||
|
for att in snap.get("attachments") or []
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_SQUASH = re.compile(r"[\W_]+", re.UNICODE)
|
||||||
|
# Words a server name wraps around its creator's: "Todding's Server",
|
||||||
|
# "The Official Todding Discord".
|
||||||
|
_SERVER_WORDS = frozenset({"server", "discord", "the", "official", "community", "s"})
|
||||||
|
|
||||||
|
|
||||||
|
def _squash(text: str | None) -> str:
|
||||||
|
return _SQUASH.sub("", (text or "").lower())
|
||||||
|
|
||||||
|
|
||||||
|
def _server_tokens(server_name: str | None) -> list[str]:
|
||||||
|
name = (server_name or "").lower().replace("'s", " ").replace("\u2019s", " ")
|
||||||
|
words = [w for w in _SQUASH.split(name) if w and w not in _SERVER_WORDS]
|
||||||
|
tokens = [w for w in words if len(w) >= 3]
|
||||||
|
joined = "".join(words)
|
||||||
|
if len(joined) >= 3 and joined not in tokens:
|
||||||
|
tokens.append(joined)
|
||||||
|
return tokens
|
||||||
|
|
||||||
|
|
||||||
|
def rank_posters(server_name: str | None, owner_id, posters: list[dict]) -> list[dict]:
|
||||||
|
"""Mark who is probably the creator a server is named for, and order the
|
||||||
|
list for picking: suggested first, then by images posted, then messages.
|
||||||
|
|
||||||
|
Two signals, each a reason the picker shows beside the name:
|
||||||
|
- they own the server;
|
||||||
|
- their username or display name matches the server's name, once
|
||||||
|
"'s Server", "Official", "Discord" and punctuation are taken off
|
||||||
|
("Todding's Server" and "todding").
|
||||||
|
Neither signal alone is proof, which is why this suggests and never
|
||||||
|
decides: the operator ticks the list. Bots are never suggested."""
|
||||||
|
tokens = _server_tokens(server_name)
|
||||||
|
owner = str(owner_id) if owner_id else None
|
||||||
|
ranked = []
|
||||||
|
for p in posters:
|
||||||
|
reasons = []
|
||||||
|
if owner and p.get("id") == owner:
|
||||||
|
reasons.append("owns the server")
|
||||||
|
names = [n for n in (_squash(p.get("username")), _squash(p.get("global_name"))) if len(n) >= 3]
|
||||||
|
if any(t in n or n in t for t in tokens for n in names):
|
||||||
|
reasons.append("name matches the server")
|
||||||
|
ranked.append({**p, "reasons": reasons, "suggested": False})
|
||||||
|
best = max((len(r["reasons"]) for r in ranked if not r.get("bot")), default=0)
|
||||||
|
for r in ranked:
|
||||||
|
r["suggested"] = best > 0 and not r.get("bot") and len(r["reasons"]) == best
|
||||||
|
ranked.sort(key=lambda r: (not r["suggested"], -r.get("images", 0), -r.get("messages", 0)))
|
||||||
|
return ranked
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class MediaItem:
|
||||||
|
"""One file of a Discord message. `filename`/`extension` are gallery-dl's
|
||||||
|
split of the URL; `num` is its 1-based position across the message's files,
|
||||||
|
which is what names it on disk.
|
||||||
|
|
||||||
|
`media_id` is what the seen-ledger keys on, and it is deliberately NOT
|
||||||
|
`num`: an edit that removes a file renumbers the ones after it, and a
|
||||||
|
positional key would then call a different file seen. It is the
|
||||||
|
attachment's id, or for an embed (which has none) a hash of its URL path —
|
||||||
|
the query string is a signature that changes on every fetch. `filehash` is
|
||||||
|
always None; nothing in a signed CDN URL is a content hash."""
|
||||||
|
|
||||||
|
url: str
|
||||||
|
filename: str
|
||||||
|
extension: str
|
||||||
|
kind: str
|
||||||
|
post_id: str
|
||||||
|
num: int
|
||||||
|
media_id: str
|
||||||
|
filehash: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordClient:
|
||||||
|
"""Synchronous Discord API v10 read client for one user token."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
token: str | None,
|
||||||
|
*,
|
||||||
|
request_sleep: float = 0.0,
|
||||||
|
max_retries: int = _MAX_429_RETRIES,
|
||||||
|
session: requests.Session | None = None,
|
||||||
|
):
|
||||||
|
self._session = session or requests.Session()
|
||||||
|
self._session.headers.update({
|
||||||
|
"User-Agent": firefox_user_agent(),
|
||||||
|
"Accept": "*/*",
|
||||||
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
|
"Referer": _ROOT + "/",
|
||||||
|
})
|
||||||
|
if token:
|
||||||
|
self._session.headers["Authorization"] = token
|
||||||
|
self._token = token
|
||||||
|
self._request_sleep = request_sleep or 0.0
|
||||||
|
self._max_retries = max_retries
|
||||||
|
self._server: dict = {}
|
||||||
|
self._channels: dict[str, dict] = {}
|
||||||
|
self._skip_feed = False
|
||||||
|
# Whose messages the walk takes (`only_from`); empty takes everyone's.
|
||||||
|
self._authors: frozenset[str] = frozenset()
|
||||||
|
|
||||||
|
# -- request -----------------------------------------------------------
|
||||||
|
|
||||||
|
def _get(self, endpoint: str, params: dict | None = None):
|
||||||
|
if not self._token:
|
||||||
|
raise DiscordAuthError("No Discord token is configured for this source")
|
||||||
|
if self._request_sleep > 0:
|
||||||
|
time.sleep(self._request_sleep)
|
||||||
|
url = API_ROOT + endpoint
|
||||||
|
attempt = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
resp = self._session.get(url, params=params, timeout=_TIMEOUT_SECONDS)
|
||||||
|
except requests.RequestException as exc:
|
||||||
|
raise DiscordAPIError(f"Discord request failed ({endpoint}): {exc}") from exc
|
||||||
|
if resp.status_code == 429 and attempt < self._max_retries:
|
||||||
|
attempt += 1
|
||||||
|
delay = retry_after_seconds(
|
||||||
|
resp, attempt, base=_429_WAIT_SECONDS, cap=_429_WAIT_SECONDS,
|
||||||
|
)
|
||||||
|
log.warning(
|
||||||
|
"Discord 429 (%s) — waiting %.1fs (retry %d/%d)",
|
||||||
|
endpoint, delay, attempt, self._max_retries,
|
||||||
|
)
|
||||||
|
time.sleep(delay)
|
||||||
|
continue
|
||||||
|
break
|
||||||
|
if resp.status_code == 401:
|
||||||
|
raise DiscordAuthError(
|
||||||
|
"Discord rejected the token (HTTP 401) — it is invalid or has "
|
||||||
|
"expired; copy a fresh one from the browser",
|
||||||
|
status_code=401,
|
||||||
|
)
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise DiscordAPIError(
|
||||||
|
f"Discord returned HTTP {resp.status_code} ({endpoint})",
|
||||||
|
status_code=resp.status_code,
|
||||||
|
retry_after=_retry_after(resp),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return resp.json()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise DiscordDriftError(
|
||||||
|
f"Discord returned non-JSON for {endpoint} ({len(resp.content)} bytes)"
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
# -- metadata (gallery-dl parse_server / parse_channel) -----------------
|
||||||
|
|
||||||
|
def _load_server(self, server_id: str) -> None:
|
||||||
|
server = self._get(f"/guilds/{server_id}")
|
||||||
|
if not isinstance(server, dict) or "id" not in server:
|
||||||
|
raise DiscordDriftError(f"Discord server {server_id} came back without an id")
|
||||||
|
self._server = {
|
||||||
|
"server": server.get("name") or "",
|
||||||
|
"server_id": str(server["id"]),
|
||||||
|
"owner_id": server.get("owner_id"),
|
||||||
|
}
|
||||||
|
channels = self._get(f"/guilds/{server_id}/channels")
|
||||||
|
if not isinstance(channels, list):
|
||||||
|
raise DiscordDriftError(f"Discord server {server_id} channel list is not a list")
|
||||||
|
# Categories first, so every child can name its parent.
|
||||||
|
for channel in sorted(channels, key=lambda ch: ch.get("type") != _CATEGORY):
|
||||||
|
self._parse_channel(channel)
|
||||||
|
|
||||||
|
def _parse_channel(self, channel: dict) -> dict:
|
||||||
|
parent_id = channel.get("parent_id")
|
||||||
|
meta = {
|
||||||
|
"channel": channel.get("name") or "",
|
||||||
|
"channel_id": str(channel.get("id")),
|
||||||
|
"channel_type": channel.get("type"),
|
||||||
|
"channel_topic": channel.get("topic") or "",
|
||||||
|
"parent_id": parent_id,
|
||||||
|
"is_thread": "thread_metadata" in channel,
|
||||||
|
}
|
||||||
|
parent = self._channels.get(parent_id) if parent_id else None
|
||||||
|
if parent:
|
||||||
|
meta["parent"] = parent["channel"]
|
||||||
|
meta["parent_type"] = parent["channel_type"]
|
||||||
|
if meta["channel_type"] in {1, 3}:
|
||||||
|
recipients = channel.get("recipients") or []
|
||||||
|
meta["channel"] = "DMs"
|
||||||
|
meta["recipients"] = [u.get("username") for u in recipients]
|
||||||
|
meta["recipients_id"] = [u.get("id") for u in recipients]
|
||||||
|
self._channels[meta["channel_id"]] = meta
|
||||||
|
return meta
|
||||||
|
|
||||||
|
def _channel_meta(self, channel_id: str) -> dict:
|
||||||
|
if channel_id not in self._channels:
|
||||||
|
self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||||||
|
return self._channels[channel_id]
|
||||||
|
|
||||||
|
def _threads(self, channel_id: str) -> list[dict]:
|
||||||
|
"""Every thread of a channel or forum, newest-created first (see the
|
||||||
|
module docstring for why not last-message order)."""
|
||||||
|
threads: list[dict] = []
|
||||||
|
offset = 0
|
||||||
|
while True:
|
||||||
|
data = self._get(f"/channels/{channel_id}/threads/search", {
|
||||||
|
"sort_by": "last_message_time",
|
||||||
|
"sort_order": "desc",
|
||||||
|
"limit": _THREADS_BATCH,
|
||||||
|
"offset": offset,
|
||||||
|
})
|
||||||
|
batch = (data.get("threads") or []) if isinstance(data, dict) else []
|
||||||
|
threads.extend(batch)
|
||||||
|
if len(batch) < _THREADS_BATCH:
|
||||||
|
break
|
||||||
|
offset += len(batch)
|
||||||
|
threads.sort(key=lambda t: int(t.get("id") or 0), reverse=True)
|
||||||
|
return threads
|
||||||
|
|
||||||
|
# -- the walk ------------------------------------------------------------
|
||||||
|
|
||||||
|
def _feeds(self, channel_id: str, *, safe: bool) -> Iterator[tuple[str, bool]]:
|
||||||
|
"""`(channel_id, safe)` for every message feed under `channel_id`, in
|
||||||
|
gallery-dl's order. `safe` feeds are skipped on a 403."""
|
||||||
|
try:
|
||||||
|
ctype = self._channel_meta(channel_id)["channel_type"]
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code != 403:
|
||||||
|
raise
|
||||||
|
if not safe:
|
||||||
|
raise DiscordAuthError(
|
||||||
|
f"The Discord token cannot see channel {channel_id} (HTTP 403)",
|
||||||
|
status_code=403,
|
||||||
|
) from exc
|
||||||
|
log.info("Discord: no access to channel %s — skipped", channel_id)
|
||||||
|
return
|
||||||
|
if ctype in _TEXT or ctype in _DIRECT:
|
||||||
|
yield channel_id, safe
|
||||||
|
if ctype in _TEXT or ctype in _FORUM:
|
||||||
|
try:
|
||||||
|
threads = self._threads(channel_id)
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code != 403:
|
||||||
|
raise
|
||||||
|
log.info("Discord: cannot list threads of %s — skipped", channel_id)
|
||||||
|
threads = []
|
||||||
|
for thread in threads:
|
||||||
|
yield self._parse_channel(thread)["channel_id"], True
|
||||||
|
elif ctype == _CATEGORY:
|
||||||
|
for child in list(self._channels.values()):
|
||||||
|
if child.get("parent_id") == channel_id:
|
||||||
|
yield from self._feeds(child["channel_id"], safe=True)
|
||||||
|
elif ctype not in _DIRECT and not safe:
|
||||||
|
raise DiscordAPIError(
|
||||||
|
f"Discord channel {channel_id} is of type {ctype}, which has no messages"
|
||||||
|
)
|
||||||
|
|
||||||
|
def _source_feeds(self, url: str) -> Iterator[tuple[str, bool]]:
|
||||||
|
server_id, channel_id = parse_source_url(url)
|
||||||
|
self._server, self._channels = {}, {}
|
||||||
|
if server_id is not None:
|
||||||
|
self._load_server(server_id)
|
||||||
|
if channel_id is not None:
|
||||||
|
yield from self._feeds(channel_id, safe=False)
|
||||||
|
return
|
||||||
|
for meta in list(self._channels.values()):
|
||||||
|
if meta["channel_type"] in _SERVER_WALK:
|
||||||
|
yield from self._feeds(meta["channel_id"], safe=True)
|
||||||
|
|
||||||
|
def only_from(self, authors) -> None:
|
||||||
|
"""Take only messages posted by these people: user ids, usernames or
|
||||||
|
display names, any case. A creator's server is full of other members
|
||||||
|
posting their own pictures; the source is subscribed to the creator.
|
||||||
|
Empty or None takes everyone's, as before."""
|
||||||
|
self._authors = frozenset(
|
||||||
|
str(a).strip().lower() for a in authors or () if str(a).strip()
|
||||||
|
)
|
||||||
|
|
||||||
|
def _author_wanted(self, message: dict) -> bool:
|
||||||
|
if not self._authors:
|
||||||
|
return True
|
||||||
|
author = message.get("author") or {}
|
||||||
|
names = (author.get("id"), author.get("username"), author.get("global_name"))
|
||||||
|
return any(str(n).lower() in self._authors for n in names if n)
|
||||||
|
|
||||||
|
def source_label(self, url: str) -> str | None:
|
||||||
|
"""What the source is called in Discord — `Server · #channel`, a
|
||||||
|
thread as `#parent › thread`, a whole server by its name — from the
|
||||||
|
metadata the walk loaded. None when the walk never got that far."""
|
||||||
|
try:
|
||||||
|
_, channel_id = parse_source_url(url)
|
||||||
|
except DiscordAPIError:
|
||||||
|
return None
|
||||||
|
server = self._server.get("server") or None
|
||||||
|
if channel_id is None:
|
||||||
|
return server
|
||||||
|
meta = self._channels.get(channel_id) or {}
|
||||||
|
name = meta.get("channel")
|
||||||
|
if not name:
|
||||||
|
return None
|
||||||
|
if name != "DMs":
|
||||||
|
name = f"#{name}"
|
||||||
|
if meta.get("is_thread") and meta.get("parent"):
|
||||||
|
name = f"#{meta['parent']} › {meta['channel']}"
|
||||||
|
return f"{server} · {name}" if server else name
|
||||||
|
|
||||||
|
def skip_feed(self) -> None:
|
||||||
|
"""Optional core seam (#4413): end the current channel and go on to the
|
||||||
|
next one. A tick's early-out means THIS channel has nothing new, not
|
||||||
|
that the server has nothing new."""
|
||||||
|
self._skip_feed = True
|
||||||
|
|
||||||
|
def iter_posts(
|
||||||
|
self, campaign_id: str, cursor: str | None = None
|
||||||
|
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||||
|
"""Yield `(message, channel_meta, page_cursor)` for every content
|
||||||
|
message the source reaches, channel by channel, each newest first.
|
||||||
|
|
||||||
|
`campaign_id` is the source URL. The cursor is `<channel_id>:<before>`
|
||||||
|
— the channel and the `before` id that fetched the page (empty for a
|
||||||
|
channel's first page) — so a backfill resumes inside the right channel
|
||||||
|
and re-fetches the page it was cut in. A cursor naming a channel the
|
||||||
|
walk no longer reaches (a deleted thread) restarts from the top rather
|
||||||
|
than walking nothing.
|
||||||
|
"""
|
||||||
|
resume_channel, _, resume_before = (cursor or "").partition(":")
|
||||||
|
resuming = bool(resume_channel)
|
||||||
|
feeds = list(self._source_feeds(campaign_id)) if resuming else None
|
||||||
|
if feeds is not None and resume_channel not in {cid for cid, _ in feeds}:
|
||||||
|
log.warning(
|
||||||
|
"Discord: resume channel %s is no longer in %s — restarting",
|
||||||
|
resume_channel, campaign_id,
|
||||||
|
)
|
||||||
|
resuming = False
|
||||||
|
for channel_id, safe in feeds if feeds is not None else self._source_feeds(campaign_id):
|
||||||
|
before = None
|
||||||
|
if resuming:
|
||||||
|
if channel_id != resume_channel:
|
||||||
|
continue
|
||||||
|
resuming = False
|
||||||
|
before = resume_before or None
|
||||||
|
yield from self._iter_channel(channel_id, before, safe=safe)
|
||||||
|
|
||||||
|
def _iter_channel(
|
||||||
|
self, channel_id: str, before: str | None, *, safe: bool
|
||||||
|
) -> Iterator[tuple[dict, dict, str | None]]:
|
||||||
|
self._skip_feed = False
|
||||||
|
meta = {**self._server, **self._channels.get(channel_id, {})}
|
||||||
|
while True:
|
||||||
|
page_cursor = f"{channel_id}:{before or ''}"
|
||||||
|
try:
|
||||||
|
messages = self._get(
|
||||||
|
f"/channels/{channel_id}/messages",
|
||||||
|
{"limit": _MESSAGES_BATCH, "before": before},
|
||||||
|
)
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code != 403:
|
||||||
|
raise
|
||||||
|
if not safe:
|
||||||
|
raise DiscordAuthError(
|
||||||
|
f"The Discord token cannot read channel {channel_id} (HTTP 403)",
|
||||||
|
status_code=403,
|
||||||
|
) from exc
|
||||||
|
log.info("Discord: no access to messages of %s — skipped", channel_id)
|
||||||
|
return
|
||||||
|
if not isinstance(messages, list):
|
||||||
|
raise DiscordDriftError(
|
||||||
|
f"Discord messages of {channel_id} came back as "
|
||||||
|
f"{type(messages).__name__}, not a list"
|
||||||
|
)
|
||||||
|
for message in messages:
|
||||||
|
if message.get("type") not in MESSAGE_TYPES:
|
||||||
|
continue
|
||||||
|
if not self._author_wanted(message):
|
||||||
|
continue
|
||||||
|
message["_meta"] = meta
|
||||||
|
yield message, meta, page_cursor
|
||||||
|
if self._skip_feed:
|
||||||
|
return
|
||||||
|
if len(messages) < _MESSAGES_BATCH:
|
||||||
|
return
|
||||||
|
before = str(messages[-1]["id"])
|
||||||
|
|
||||||
|
# -- per-message -------------------------------------------------------
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def extract_media(post: dict, included: dict | None = None) -> list[MediaItem]:
|
||||||
|
"""gallery-dl's file list for one message: attachments, then the first
|
||||||
|
of video/image/thumbnail `proxy_url` of each file-bearing embed, then
|
||||||
|
the same for every forwarded snapshot; numbered from 1 across them.
|
||||||
|
|
||||||
|
Empty for a message with no image or video attached: that is chat,
|
||||||
|
whatever else it carries (`has_visual_attachment`)."""
|
||||||
|
if not has_visual_attachment(post):
|
||||||
|
return []
|
||||||
|
mid = str(post.get("id") or "")
|
||||||
|
snapshots = _message_and_snapshots(post)
|
||||||
|
found: list[tuple[str, str, str | None]] = []
|
||||||
|
for snap in snapshots:
|
||||||
|
for att in snap.get("attachments") or []:
|
||||||
|
if att.get("url"):
|
||||||
|
aid = att.get("id")
|
||||||
|
found.append((att["url"], "attachment", str(aid) if aid else None))
|
||||||
|
for embed in snap.get("embeds") or []:
|
||||||
|
if embed.get("type") not in _EMBED_TYPES:
|
||||||
|
continue
|
||||||
|
for fld in ("video", "image", "thumbnail"):
|
||||||
|
url = (embed.get(fld) or {}).get("proxy_url")
|
||||||
|
if url:
|
||||||
|
found.append((url, "embed", None))
|
||||||
|
break
|
||||||
|
items = []
|
||||||
|
for num, (url, kind, fid) in enumerate(found, start=1):
|
||||||
|
name, ext = nameext_from_url(url)
|
||||||
|
if fid is None:
|
||||||
|
path = url.partition("?")[0].encode()
|
||||||
|
fid = "u" + hashlib.sha1(path, usedforsecurity=False).hexdigest()[:32]
|
||||||
|
items.append(MediaItem(
|
||||||
|
url=url, filename=name, extension=ext, kind=kind, post_id=mid,
|
||||||
|
num=num, media_id=fid,
|
||||||
|
))
|
||||||
|
return items
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def post_meta(post: dict) -> dict:
|
||||||
|
"""No title (Discord has none); `date` is the message timestamp, ISO
|
||||||
|
with an offset — what the core's revisit window reads."""
|
||||||
|
return {"title": None, "date": post.get("timestamp")}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def post_record_key(cls, post: dict) -> tuple[str, str] | None:
|
||||||
|
"""`(message:<id>, <id>)` — gates the message record through the seen
|
||||||
|
ledger, like `post:<id>` on the other platforms.
|
||||||
|
|
||||||
|
None for a message `extract_media` takes nothing from, i.e. one with
|
||||||
|
no image or video attached. The drop grouping (discord_grouping) is
|
||||||
|
built on that: a channel's chatter recorded as posts would bury the
|
||||||
|
drops it exists to surface."""
|
||||||
|
mid = post.get("id")
|
||||||
|
mid = str(mid) if mid is not None else ""
|
||||||
|
if not mid or not cls.extract_media(post):
|
||||||
|
return None
|
||||||
|
return (f"message:{mid}", mid)
|
||||||
|
|
||||||
|
# -- verify ------------------------------------------------------------
|
||||||
|
|
||||||
|
def describe(self, server_id: str | None, channel_id: str | None) -> dict:
|
||||||
|
"""The display names behind a server/channel pair, for the browser
|
||||||
|
extension's Add panel. Best-effort per name: one that can't be read
|
||||||
|
comes back None, and the other is still returned."""
|
||||||
|
out: dict = {"server": None, "channel": None, "parent": None}
|
||||||
|
if server_id:
|
||||||
|
try:
|
||||||
|
out["server"] = (self._get(f"/guilds/{server_id}") or {}).get("name") or None
|
||||||
|
except DiscordAPIError:
|
||||||
|
pass
|
||||||
|
if channel_id:
|
||||||
|
try:
|
||||||
|
meta = self._parse_channel(self._get(f"/channels/{channel_id}"))
|
||||||
|
out["channel"] = meta.get("channel") or None
|
||||||
|
except (DiscordAPIError, AttributeError):
|
||||||
|
pass
|
||||||
|
return out
|
||||||
|
|
||||||
|
def recent_posters(
|
||||||
|
self, server_id: str | None, channel_id: str, *, max_messages: int = _POSTER_SCAN,
|
||||||
|
) -> dict:
|
||||||
|
"""Who has posted lately in a channel, for picking a source's poster
|
||||||
|
list (#4491): the newest `max_messages` messages, tallied per author.
|
||||||
|
Deliberately shallow — it answers "who posts here", not "who ever did".
|
||||||
|
Every poster comes back with their stable id, which is what the list
|
||||||
|
stores: a username can change on a whim, an id can't."""
|
||||||
|
server: dict = {}
|
||||||
|
if server_id:
|
||||||
|
try:
|
||||||
|
server = self._get(f"/guilds/{server_id}") or {}
|
||||||
|
except DiscordAPIError:
|
||||||
|
server = {}
|
||||||
|
tally: dict[str, dict] = {}
|
||||||
|
before = None
|
||||||
|
seen = 0
|
||||||
|
while seen < max_messages:
|
||||||
|
page = self._get(
|
||||||
|
f"/channels/{channel_id}/messages",
|
||||||
|
{"limit": min(_MESSAGES_BATCH, max_messages - seen), "before": before},
|
||||||
|
)
|
||||||
|
if not isinstance(page, list) or not page:
|
||||||
|
break
|
||||||
|
for message in page:
|
||||||
|
seen += 1
|
||||||
|
if message.get("type") not in MESSAGE_TYPES:
|
||||||
|
continue
|
||||||
|
author = message.get("author") or {}
|
||||||
|
aid = str(author.get("id") or "")
|
||||||
|
if not aid:
|
||||||
|
continue
|
||||||
|
row = tally.setdefault(aid, {
|
||||||
|
"id": aid,
|
||||||
|
"username": author.get("username"),
|
||||||
|
"global_name": author.get("global_name"),
|
||||||
|
"bot": bool(author.get("bot")),
|
||||||
|
"messages": 0,
|
||||||
|
"images": 0,
|
||||||
|
})
|
||||||
|
row["messages"] += 1
|
||||||
|
if has_visual_attachment(message):
|
||||||
|
row["images"] += 1
|
||||||
|
if len(page) < _MESSAGES_BATCH:
|
||||||
|
break
|
||||||
|
before = str(page[-1]["id"])
|
||||||
|
return {
|
||||||
|
"server": server.get("name") or None,
|
||||||
|
"owner_id": str(server["owner_id"]) if server.get("owner_id") else None,
|
||||||
|
"scanned": seen,
|
||||||
|
"posters": rank_posters(
|
||||||
|
server.get("name"), server.get("owner_id"), list(tally.values()),
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
def verify_auth(self, url: str) -> tuple[bool | None, str]:
|
||||||
|
"""Is the token valid, and can it see what the source names?"""
|
||||||
|
try:
|
||||||
|
server_id, channel_id = parse_source_url(url)
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
return None, str(exc)
|
||||||
|
try:
|
||||||
|
me = self._get("/users/@me")
|
||||||
|
if channel_id is not None:
|
||||||
|
self._get(f"/channels/{channel_id}")
|
||||||
|
elif server_id is not None:
|
||||||
|
self._get(f"/guilds/{server_id}")
|
||||||
|
except DiscordAuthError as exc:
|
||||||
|
return False, f"Discord rejected the token — {exc}"
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
if exc.status_code in (403, 404):
|
||||||
|
return False, (
|
||||||
|
"The token is valid, but its account cannot see "
|
||||||
|
f"{'this channel' if channel_id else 'this server'} "
|
||||||
|
f"(HTTP {exc.status_code})"
|
||||||
|
)
|
||||||
|
return None, f"Couldn't verify (network/HTTP issue): {exc}"
|
||||||
|
who = (me or {}).get("username") if isinstance(me, dict) else None
|
||||||
|
return True, f"Token valid{f' ({who})' if who else ''} — the source is readable."
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_after(resp: requests.Response) -> float | None:
|
||||||
|
hdr = resp.headers.get("Retry-After")
|
||||||
|
try:
|
||||||
|
return float(hdr) if hdr else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
"""Native Discord media downloader — the Discord counterpart to
|
||||||
|
subscribestar_downloader.
|
||||||
|
|
||||||
|
Writes files exactly where gallery-dl wrote them, so a cutover finds every
|
||||||
|
existing file on disk (`skipped_disk`) instead of fetching it again:
|
||||||
|
|
||||||
|
<images>/<artist>/discord/<channel>/<YYYYMMDD>_<message_id>_<NN>_<name>.<ext>
|
||||||
|
|
||||||
|
That is what FC's gallery-dl config produced (directory `{channel}`, filename
|
||||||
|
`{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}`, under the
|
||||||
|
per-source base directory `<images>/<artist>/<platform>`), retired from that
|
||||||
|
config once Discord moved here; tests/test_discord_naming.py pins the match
|
||||||
|
against a real gallery-dl sidecar. The name is cleaned the way gallery-dl cleans it
|
||||||
|
on Linux — `/` becomes `_` and control characters are removed, nothing else
|
||||||
|
(`path-restrict: auto`, `path-remove` defaults). It is NOT `sanitize_segment`,
|
||||||
|
whose Windows set would turn a `:` in a channel or file name into `_` and miss
|
||||||
|
the file gallery-dl wrote.
|
||||||
|
|
||||||
|
Post-first (rule 120): each file gets a minimal sidecar named like it minus the
|
||||||
|
extension (what `find_sidecar` pairs first), and the message itself gets one
|
||||||
|
record, `<YYYYMMDD>_<message_id>_post.json`, carrying gallery-dl's metadata keys
|
||||||
|
— `message_id` for the post id, `server_id`/`channel_id` for the permalink,
|
||||||
|
`message` for the body, `date` — so `parse_sidecar` reads it exactly as it read
|
||||||
|
the gallery-dl sidecars. Neither file carries an `id` or `post_id` key: both
|
||||||
|
outrank `message_id` in the post-id chain (`platforms.base`).
|
||||||
|
|
||||||
|
PURE: no DB; the seen-skip is an injected predicate.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from .discord_client import firefox_user_agent, message_text
|
||||||
|
from .native_ingest_common import (
|
||||||
|
BaseNativeDownloader,
|
||||||
|
MediaOutcome,
|
||||||
|
PostRecordOutcome,
|
||||||
|
make_session,
|
||||||
|
)
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
PLATFORM = "discord"
|
||||||
|
_CONTROL = re.compile("[\x00-\x1f\x7f]")
|
||||||
|
# gallery-dl falls back to the response's type for a URL with no extension;
|
||||||
|
# we never see the response before naming, and such URLs do not occur for
|
||||||
|
# Discord attachments or embed proxies in practice.
|
||||||
|
_NO_EXTENSION = "bin"
|
||||||
|
|
||||||
|
|
||||||
|
def gdl_clean(segment: str) -> str:
|
||||||
|
"""One path segment as gallery-dl writes it on Linux."""
|
||||||
|
return _CONTROL.sub("", segment.replace("/", "_"))
|
||||||
|
|
||||||
|
|
||||||
|
def message_date(post: dict) -> datetime | None:
|
||||||
|
raw = post.get("timestamp")
|
||||||
|
if not isinstance(raw, str) or not raw:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return (dt if dt.tzinfo else dt.replace(tzinfo=UTC)).astimezone(UTC)
|
||||||
|
|
||||||
|
|
||||||
|
def channel_dir(images_root: Path, artist_slug: str, post: dict) -> Path:
|
||||||
|
"""gallery-dl's `{channel}` directory; an empty name adds no segment."""
|
||||||
|
base = Path(images_root) / artist_slug / PLATFORM
|
||||||
|
channel = gdl_clean(((post.get("_meta") or {}).get("channel") or "").strip())
|
||||||
|
return base / channel if channel else base
|
||||||
|
|
||||||
|
|
||||||
|
def media_stem(post: dict, media) -> str:
|
||||||
|
"""`<YYYYMMDD>_<message_id>_<NN>_<name>` — the file's name minus `.<ext>`."""
|
||||||
|
when = message_date(post)
|
||||||
|
day = f"{when:%Y%m%d}" if when else "None"
|
||||||
|
return gdl_clean(f"{day}_{post.get('id')}_{media.num:>02}_{media.filename}")
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordDownloader(BaseNativeDownloader):
|
||||||
|
"""Download a message's files to gallery-dl's layout. The CDN gets
|
||||||
|
gallery-dl's browser profile and no token — gallery-dl sends the token only
|
||||||
|
to the API, and the CDN URLs are pre-signed."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
images_root: Path,
|
||||||
|
cookies_path: str | None = None,
|
||||||
|
*,
|
||||||
|
validate: bool = True,
|
||||||
|
rate_limit: float = 0.0,
|
||||||
|
session: requests.Session | None = None,
|
||||||
|
):
|
||||||
|
super().__init__(
|
||||||
|
images_root, None, platform=PLATFORM,
|
||||||
|
validate=validate, rate_limit=rate_limit,
|
||||||
|
session=session if session is not None else make_session(None, extra_headers={
|
||||||
|
"User-Agent": firefox_user_agent(),
|
||||||
|
"Accept-Language": "en-US,en;q=0.5",
|
||||||
|
"Referer": "https://discord.com/",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
|
||||||
|
def download_post(
|
||||||
|
self,
|
||||||
|
post: dict,
|
||||||
|
media_items: list,
|
||||||
|
artist_slug: str,
|
||||||
|
*,
|
||||||
|
is_seen: Callable[[object], bool] = lambda m: False,
|
||||||
|
should_stop: Callable[[], bool] = lambda: False,
|
||||||
|
recapture: bool = False,
|
||||||
|
) -> list[MediaOutcome]:
|
||||||
|
"""Every file of one message; per-file outcomes, one failure isolated."""
|
||||||
|
folder = channel_dir(self.images_root, artist_slug, post)
|
||||||
|
outcomes: list[MediaOutcome] = []
|
||||||
|
for media in media_items:
|
||||||
|
if should_stop():
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
outcomes.append(self._download_one(
|
||||||
|
post, media, folder, artist_slug, is_seen, recapture=recapture,
|
||||||
|
))
|
||||||
|
except Exception as exc: # resilient: isolate one item's failure
|
||||||
|
log.warning(
|
||||||
|
"Discord media failed (message %s, file %d): %s",
|
||||||
|
post.get("id"), media.num, exc,
|
||||||
|
)
|
||||||
|
outcomes.append(
|
||||||
|
MediaOutcome(media=media, status="error", path=None, error=str(exc))
|
||||||
|
)
|
||||||
|
return outcomes
|
||||||
|
|
||||||
|
def _download_one(
|
||||||
|
self,
|
||||||
|
post: dict,
|
||||||
|
media,
|
||||||
|
folder: Path,
|
||||||
|
artist_slug: str,
|
||||||
|
is_seen: Callable[[object], bool],
|
||||||
|
*,
|
||||||
|
recapture: bool = False,
|
||||||
|
) -> MediaOutcome:
|
||||||
|
seen = is_seen(media)
|
||||||
|
if seen and not recapture:
|
||||||
|
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||||
|
stem = media_stem(post, media)
|
||||||
|
path = folder / f"{stem}.{media.extension or _NO_EXTENSION}"
|
||||||
|
if path.exists(): # tier-2: gallery-dl (or an earlier walk) wrote it
|
||||||
|
return MediaOutcome(media=media, status="skipped_disk", path=path, error=None)
|
||||||
|
if seen: # recapture never re-fetches a seen file that is gone
|
||||||
|
return MediaOutcome(media=media, status="skipped_seen", path=None, error=None)
|
||||||
|
|
||||||
|
folder.mkdir(parents=True, exist_ok=True)
|
||||||
|
if self._rate_limit > 0:
|
||||||
|
time.sleep(self._rate_limit)
|
||||||
|
out = self._fetch_get(media.url, path)
|
||||||
|
reason, quarantined = self._validate_path(out, artist_slug, media.url)
|
||||||
|
if reason is not None:
|
||||||
|
return MediaOutcome(media=media, status="quarantined", path=quarantined, error=reason)
|
||||||
|
sidecar = {"category": PLATFORM, "message_id": str(post.get("id") or "")}
|
||||||
|
sidecar["source_url"] = media.url
|
||||||
|
(folder / f"{stem}.json").write_text(json.dumps(sidecar, indent=2))
|
||||||
|
return MediaOutcome(media=media, status="downloaded", path=out, error=None)
|
||||||
|
|
||||||
|
def write_post_record(
|
||||||
|
self, post: dict, artist_slug: str, *, revisit: bool = False,
|
||||||
|
) -> PostRecordOutcome:
|
||||||
|
"""The message record — the one writer of a Discord post's body, date
|
||||||
|
and permalink ids. `revisit` re-reads a message already captured (an
|
||||||
|
edit); an empty re-read writes nothing, so it never blanks a body."""
|
||||||
|
mid = str(post.get("id") or "")
|
||||||
|
body = message_text(post)
|
||||||
|
if not mid or (revisit and not body.strip()):
|
||||||
|
return PostRecordOutcome(path=None, post_type=None, title=None, body_chars=0)
|
||||||
|
meta = post.get("_meta") or {}
|
||||||
|
author = post.get("author") or {}
|
||||||
|
record = {
|
||||||
|
"category": PLATFORM,
|
||||||
|
"message_id": mid,
|
||||||
|
"server": meta.get("server"),
|
||||||
|
"server_id": meta.get("server_id"),
|
||||||
|
"channel": meta.get("channel"),
|
||||||
|
"channel_id": meta.get("channel_id") or post.get("channel_id"),
|
||||||
|
"parent": meta.get("parent"),
|
||||||
|
"is_thread": meta.get("is_thread"),
|
||||||
|
"author": author.get("username"),
|
||||||
|
"author_name": author.get("global_name"),
|
||||||
|
"author_id": author.get("id"),
|
||||||
|
"message": body,
|
||||||
|
"date": post.get("timestamp"),
|
||||||
|
}
|
||||||
|
folder = channel_dir(self.images_root, artist_slug, post)
|
||||||
|
folder.mkdir(parents=True, exist_ok=True)
|
||||||
|
when = message_date(post)
|
||||||
|
day = f"{when:%Y%m%d}" if when else "None"
|
||||||
|
path = folder / f"{day}_{mid}_post.json"
|
||||||
|
path.write_text(json.dumps(
|
||||||
|
{k: v for k, v in record.items() if v is not None}, indent=2,
|
||||||
|
))
|
||||||
|
return PostRecordOutcome(
|
||||||
|
path=path, post_type=None, title=None, body_chars=len(body),
|
||||||
|
)
|
||||||
@@ -63,7 +63,7 @@ from collections import Counter
|
|||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import UTC, datetime, timedelta
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
from sqlalchemy import Select, delete, func, select, update
|
from sqlalchemy import Select, delete, func, select, union, update
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
@@ -126,6 +126,31 @@ def cosine_distance(a, b) -> float:
|
|||||||
return 1.0 - (dot / (na * nb))
|
return 1.0 - (dot / (na * nb))
|
||||||
|
|
||||||
|
|
||||||
|
def _message_images(posts):
|
||||||
|
"""(post_id, image_id) for every image a message carries — owned AND re-posted.
|
||||||
|
|
||||||
|
A message owns an image through `primary_post_id`, but only the FIRST
|
||||||
|
message imported with a given file does. The same file posted again is a
|
||||||
|
provenance link, and the backfill runs newest-first, so it is usually the
|
||||||
|
ORIGINAL message that ends up owning nothing. Reading ownership alone left
|
||||||
|
101 of Yellowroom's messages (2018–2020 mostly) ungroupable: every image
|
||||||
|
they carried also sat in another message.
|
||||||
|
|
||||||
|
`posts` is a list of ids or a select of them; filtering both branches by it
|
||||||
|
keeps the union to the messages in hand rather than the whole library.
|
||||||
|
Callers pass MESSAGE posts only — a drop's own provenance rows would read
|
||||||
|
as images it carries.
|
||||||
|
"""
|
||||||
|
owned = select(
|
||||||
|
ImageRecord.primary_post_id.label("post_id"), ImageRecord.id.label("image_id"),
|
||||||
|
).where(ImageRecord.primary_post_id.in_(posts))
|
||||||
|
reposted = select(
|
||||||
|
ImageProvenance.post_id.label("post_id"),
|
||||||
|
ImageProvenance.image_record_id.label("image_id"),
|
||||||
|
).where(ImageProvenance.post_id.in_(posts))
|
||||||
|
return union(owned, reposted).subquery()
|
||||||
|
|
||||||
|
|
||||||
def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
||||||
"""Ungrouped Discord message-posts, one representative image each, OLDEST
|
"""Ungrouped Discord message-posts, one representative image each, OLDEST
|
||||||
FIRST — which is the order `build_groups` requires.
|
FIRST — which is the order `build_groups` requires.
|
||||||
@@ -143,13 +168,15 @@ def _candidate_stmt(source_id: int, *, not_after: datetime) -> Select:
|
|||||||
take the OLDEST candidates instead of the lowest-numbered ones.
|
take the OLDEST candidates instead of the lowest-numbered ones.
|
||||||
"""
|
"""
|
||||||
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||||
|
carried = _message_images(select(Post.id).where(Post.source_id == source_id))
|
||||||
inner = (
|
inner = (
|
||||||
select(
|
select(
|
||||||
Post.id.label("post_id"),
|
Post.id.label("post_id"),
|
||||||
sort_key.label("occurred_at"),
|
sort_key.label("occurred_at"),
|
||||||
ImageRecord.siglip_embedding.label("embedding"),
|
ImageRecord.siglip_embedding.label("embedding"),
|
||||||
)
|
)
|
||||||
.join(ImageRecord, ImageRecord.primary_post_id == Post.id)
|
.join(carried, carried.c.post_id == Post.id)
|
||||||
|
.join(ImageRecord, ImageRecord.id == carried.c.image_id)
|
||||||
.where(
|
.where(
|
||||||
Post.source_id == source_id,
|
Post.source_id == source_id,
|
||||||
# Never absorb a post FC wrote, and never re-absorb one already
|
# Never absorb a post FC wrote, and never re-absorb one already
|
||||||
@@ -295,9 +322,10 @@ async def _link_member_images(
|
|||||||
"""
|
"""
|
||||||
if not member_ids:
|
if not member_ids:
|
||||||
return 0
|
return 0
|
||||||
image_rows = (await session.execute(
|
carried = _message_images(member_ids)
|
||||||
select(ImageRecord.id).where(ImageRecord.primary_post_id.in_(member_ids))
|
image_rows = sorted(set((await session.execute(
|
||||||
)).scalars().all()
|
select(carried.c.image_id)
|
||||||
|
)).scalars().all()))
|
||||||
if not image_rows:
|
if not image_rows:
|
||||||
return 0
|
return 0
|
||||||
await session.execute(
|
await session.execute(
|
||||||
@@ -421,9 +449,11 @@ async def _group_seed(session: AsyncSession, post_id: int) -> list[float] | None
|
|||||||
free to disagree; this way there is one.
|
free to disagree; this way there is one.
|
||||||
"""
|
"""
|
||||||
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
sort_key = func.coalesce(Post.post_date, Post.downloaded_at)
|
||||||
|
carried = _message_images(select(Post.id).where(Post.absorbed_by_post_id == post_id))
|
||||||
return (await session.execute(
|
return (await session.execute(
|
||||||
select(ImageRecord.siglip_embedding)
|
select(ImageRecord.siglip_embedding)
|
||||||
.join(Post, ImageRecord.primary_post_id == Post.id)
|
.join(carried, carried.c.image_id == ImageRecord.id)
|
||||||
|
.join(Post, Post.id == carried.c.post_id)
|
||||||
.where(
|
.where(
|
||||||
Post.absorbed_by_post_id == post_id,
|
Post.absorbed_by_post_id == post_id,
|
||||||
ImageRecord.siglip_embedding.is_not(None),
|
ImageRecord.siglip_embedding.is_not(None),
|
||||||
@@ -725,12 +755,15 @@ async def _load_drops(session: AsyncSession, source: Source) -> list[_Drop]:
|
|||||||
names: dict[int, set[str]] = {pid: set() for pid in by_id}
|
names: dict[int, set[str]] = {pid: set() for pid in by_id}
|
||||||
images: dict[int, list[int]] = {pid: [] for pid in by_id}
|
images: dict[int, list[int]] = {pid: [] for pid in by_id}
|
||||||
if owner_of:
|
if owner_of:
|
||||||
for iid, primary, path in (await session.execute(
|
carried = _message_images(list(owner_of))
|
||||||
select(ImageRecord.id, ImageRecord.primary_post_id, ImageRecord.path)
|
for iid, message, path in (await session.execute(
|
||||||
.where(ImageRecord.primary_post_id.in_(list(owner_of)))
|
select(ImageRecord.id, carried.c.post_id, ImageRecord.path)
|
||||||
|
.join(carried, carried.c.image_id == ImageRecord.id)
|
||||||
.order_by(ImageRecord.id)
|
.order_by(ImageRecord.id)
|
||||||
)).all():
|
)).all():
|
||||||
drop = owner_of[primary]
|
drop = owner_of[message]
|
||||||
|
if iid in images[drop]:
|
||||||
|
continue # one file carried by two of the drop's messages
|
||||||
images[drop].append(iid)
|
images[drop].append(iid)
|
||||||
if (name := leading_name(path)) is not None:
|
if (name := leading_name(path)) is not None:
|
||||||
names[drop].add(name)
|
names[drop].add(name)
|
||||||
@@ -844,7 +877,12 @@ async def merge_trickles(
|
|||||||
if drop.first_at - earlier.last_at > gap:
|
if drop.first_at - earlier.last_at > gap:
|
||||||
continue
|
continue
|
||||||
shared = mine & gated(earlier.names)
|
shared = mine & gated(earlier.names)
|
||||||
if shared:
|
if set(drop.images) & set(earlier.images):
|
||||||
|
# The creator posted the very same file again — the strongest
|
||||||
|
# evidence there is, and one nearest-neighbour cannot see: it
|
||||||
|
# skips the image itself, which is the one they share.
|
||||||
|
targets[earlier.post.id] = (earlier, "same_image")
|
||||||
|
elif shared:
|
||||||
targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
|
targets[earlier.post.id] = (earlier, f"name:{min(shared)}")
|
||||||
elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
|
elif drop.nearest & earlier.members or (earlier.nearest or set()) & drop.members:
|
||||||
targets[earlier.post.id] = (earlier, "nearest")
|
targets[earlier.post.id] = (earlier, "nearest")
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
"""Native Discord ingester — the Discord ADAPTER over `ingest_core.Ingester`.
|
||||||
|
|
||||||
|
Thin counterpart to subscribestar_ingester (milestone 428). The walk's modes,
|
||||||
|
both ledgers, cursor checkpointing and the post-first capture live in the core;
|
||||||
|
this wires in the Discord client, downloader, ledger models and key.
|
||||||
|
|
||||||
|
Two things differ from the cookie platforms:
|
||||||
|
|
||||||
|
- Discord authenticates with a user TOKEN, so `auth_token` is the credential
|
||||||
|
here rather than an argument accepted and ignored.
|
||||||
|
- The body canary is off. It fails a walk whose first 30+ captured posts all
|
||||||
|
came back without text, on the theory that a creator nearly always writes
|
||||||
|
something; a Discord drop is routinely files and nothing else, so on
|
||||||
|
Discord that is an ordinary backfill, not a broken parser.
|
||||||
|
|
||||||
|
And two things only a Discord source has: `discord_authors` in its
|
||||||
|
config_overrides limits it to the creator's own messages (a server's other
|
||||||
|
members post pictures too), and it learns its own name — the server and
|
||||||
|
channel it walks — into `source.display_name`.
|
||||||
|
|
||||||
|
`campaign_id` is the source URL (a server, channel, thread or category link).
|
||||||
|
FC runs on a plain-HTTP homelab; nothing here uses a secure-context Web API.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
from collections.abc import Callable
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
|
from ..models import DiscordFailedMedia, DiscordSeenMedia, Source
|
||||||
|
from .discord_client import DiscordAPIError, DiscordClient, MediaItem
|
||||||
|
from .discord_downloader import DiscordDownloader
|
||||||
|
from .ingest_core import Ingester
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_LEDGER_KEY_MAX = 128
|
||||||
|
# `config_overrides` key: the people whose messages this source takes (user
|
||||||
|
# ids, usernames or display names). Absent or empty takes everyone's.
|
||||||
|
AUTHORS_KEY = "discord_authors"
|
||||||
|
|
||||||
|
|
||||||
|
def _ledger_key(media: MediaItem) -> str:
|
||||||
|
"""`<message_id>:<media_id>` — stable across edits (see MediaItem)."""
|
||||||
|
return f"{media.post_id}:{media.media_id}"[:_LEDGER_KEY_MAX]
|
||||||
|
|
||||||
|
|
||||||
|
class DiscordIngester(Ingester):
|
||||||
|
"""Walk a Discord source's channels, download unseen files, return a
|
||||||
|
`DownloadResult`. `client` / `downloader` are injectable for tests."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
images_root: Path,
|
||||||
|
cookies_path: str | None,
|
||||||
|
session_factory: Callable[[], object],
|
||||||
|
*,
|
||||||
|
validate: bool = True,
|
||||||
|
rate_limit: float = 0.0,
|
||||||
|
request_sleep: float = 0.0,
|
||||||
|
auth_token: str | None = None,
|
||||||
|
client: DiscordClient | None = None,
|
||||||
|
downloader: DiscordDownloader | None = None,
|
||||||
|
):
|
||||||
|
del cookies_path # Discord authenticates by token (uniform signature)
|
||||||
|
self.images_root = Path(images_root)
|
||||||
|
super().__init__(
|
||||||
|
client=client if client is not None else DiscordClient(
|
||||||
|
auth_token, request_sleep=request_sleep,
|
||||||
|
),
|
||||||
|
downloader=downloader if downloader is not None else DiscordDownloader(
|
||||||
|
self.images_root, validate=validate, rate_limit=rate_limit,
|
||||||
|
),
|
||||||
|
session_factory=session_factory,
|
||||||
|
seen_model=DiscordSeenMedia,
|
||||||
|
failed_model=DiscordFailedMedia,
|
||||||
|
seen_constraint="uq_discord_seen_media_source_id",
|
||||||
|
failed_constraint="uq_discord_failed_media_source_id",
|
||||||
|
ledger_key=_ledger_key,
|
||||||
|
platform="discord",
|
||||||
|
error_base=DiscordAPIError,
|
||||||
|
drift_label="Discord API",
|
||||||
|
body_canary=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def run(self, **kwargs):
|
||||||
|
"""The core walk, bracketed by the two things only a Discord source
|
||||||
|
has: whose messages it takes, read before the walk, and the name of
|
||||||
|
what it walks, written after it (the walk is what loads that name)."""
|
||||||
|
source_id = kwargs["source_id"]
|
||||||
|
self.client.only_from(self._source_authors(source_id))
|
||||||
|
try:
|
||||||
|
return super().run(**kwargs)
|
||||||
|
finally:
|
||||||
|
self._record_display_name(source_id, kwargs["campaign_id"])
|
||||||
|
|
||||||
|
def _source_authors(self, source_id: int) -> list:
|
||||||
|
if self.session_factory is None:
|
||||||
|
return []
|
||||||
|
with self.session_factory() as session:
|
||||||
|
overrides = session.execute(
|
||||||
|
select(Source.config_overrides).where(Source.id == source_id)
|
||||||
|
).scalar_one_or_none() or {}
|
||||||
|
authors = overrides.get(AUTHORS_KEY) or []
|
||||||
|
return authors if isinstance(authors, list) else []
|
||||||
|
|
||||||
|
def _record_display_name(self, source_id: int, url: str) -> None:
|
||||||
|
"""Refreshed on every walk, never written once: a renamed channel
|
||||||
|
should read as its new name. A name the walk couldn't read leaves the
|
||||||
|
stored one alone, and a failure here never fails the walk."""
|
||||||
|
label = self.client.source_label(url)
|
||||||
|
if not label or self.session_factory is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
with self.session_factory() as session:
|
||||||
|
session.execute(
|
||||||
|
update(Source)
|
||||||
|
.where(Source.id == source_id)
|
||||||
|
.where(Source.display_name.is_distinct_from(label))
|
||||||
|
.values(display_name=label)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
except Exception as exc: # a name is decoration — never fail the walk
|
||||||
|
log.warning("Discord: couldn't record source %s's name: %s", source_id, exc)
|
||||||
|
|
||||||
|
|
||||||
|
async def verify_discord_credential(url: str, auth_token: str | None) -> tuple[bool | None, str]:
|
||||||
|
"""The uniform `(ok, message)` probe: is the token valid, and can its
|
||||||
|
account see the channel or server the source names?"""
|
||||||
|
if not auth_token:
|
||||||
|
return False, "No Discord token is saved — add one under Credentials."
|
||||||
|
client = DiscordClient(auth_token)
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
return await loop.run_in_executor(None, client.verify_auth, url)
|
||||||
@@ -0,0 +1,243 @@
|
|||||||
|
"""Remove a Discord source's posts by people outside its poster list (#4486).
|
||||||
|
|
||||||
|
A creator's server is full of other members posting their own pictures. The
|
||||||
|
poster list (`discord_authors`, #4481) stops a walk taking them; this removes
|
||||||
|
the ones taken before the list existed. The operator sets the list first, then
|
||||||
|
previews what would go, then applies.
|
||||||
|
|
||||||
|
## What goes
|
||||||
|
|
||||||
|
A post of this source whose record names its poster (`author_id` / `author`,
|
||||||
|
written by `DiscordDownloader.write_post_record`) and names someone NOT on the
|
||||||
|
list. Two kinds of post are never touched:
|
||||||
|
|
||||||
|
* a post whose record names no poster — a message recorded before the native
|
||||||
|
ingester, whose poster FC cannot tell. Counted as `unknown`, never guessed;
|
||||||
|
* a synthetic drop post (`synthesized_by`) — FC authored it, and it is handled
|
||||||
|
below as a consequence, not matched as a poster's post.
|
||||||
|
|
||||||
|
An image goes when every REAL post it belongs to is going. A link to a
|
||||||
|
synthetic drop does not keep an image: the drop only references its members'
|
||||||
|
images. An image also on a kept post — the creator re-posting a piece someone
|
||||||
|
else shared — stays.
|
||||||
|
|
||||||
|
A drop that absorbed a removed message is deleted outright. Deleting a drop is
|
||||||
|
its undo (discord_grouping's honesty rule): its remaining members return to
|
||||||
|
the feed and the grouping sweep regroups them on its next run, so no half-
|
||||||
|
rebuilt drop keeps text or thumbnails from someone it no longer contains.
|
||||||
|
|
||||||
|
Preview and apply spread the same predicates (rule 93, snippet #3087).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import and_, delete, exists, func, or_, select
|
||||||
|
from sqlalchemy.orm import Session, aliased
|
||||||
|
|
||||||
|
from ..models import ImageProvenance, ImageRecord, Post, PostAttachment, Source
|
||||||
|
from .cleanup_service import delete_images
|
||||||
|
from .discord_ingester import AUTHORS_KEY
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
PLATFORM = "discord"
|
||||||
|
# The record keys that name a message's poster. `author_name` (the display
|
||||||
|
# name) is only on records written after 2026-09-28.
|
||||||
|
_POSTER_KEYS = ("author_id", "author", "author_name")
|
||||||
|
|
||||||
|
|
||||||
|
class PosterCleanupError(ValueError):
|
||||||
|
"""The source can't be cleaned this way (not Discord, or no list)."""
|
||||||
|
|
||||||
|
|
||||||
|
def source_authors(source: Source) -> list[str]:
|
||||||
|
"""The source's poster list, lowercased; empty means everyone's wanted."""
|
||||||
|
authors = (source.config_overrides or {}).get(AUTHORS_KEY) or []
|
||||||
|
if not isinstance(authors, list):
|
||||||
|
return []
|
||||||
|
return sorted({str(a).strip().lower() for a in authors if str(a).strip()})
|
||||||
|
|
||||||
|
|
||||||
|
def _poster(key: str):
|
||||||
|
return func.lower(Post.raw_metadata[key].as_string())
|
||||||
|
|
||||||
|
|
||||||
|
def _has_poster():
|
||||||
|
return or_(*(Post.raw_metadata[k].as_string().isnot(None) for k in ("author_id", "author")))
|
||||||
|
|
||||||
|
|
||||||
|
def _real_post_conditions(source_id: int) -> list:
|
||||||
|
return [Post.source_id == source_id, Post.synthesized_by.is_(None)]
|
||||||
|
|
||||||
|
|
||||||
|
def _named_poster_conditions(source_id: int) -> list:
|
||||||
|
"""A real post of this source whose record says who posted it."""
|
||||||
|
return [*_real_post_conditions(source_id), _has_poster()]
|
||||||
|
|
||||||
|
|
||||||
|
def _other_poster_post_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
"""The posts that go: a named poster none of whose names is on the list."""
|
||||||
|
return [
|
||||||
|
*_named_poster_conditions(source_id),
|
||||||
|
*(func.coalesce(_poster(k), "").notin_(authors) for k in _POSTER_KEYS),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _doomed_post_ids(source_id: int, authors: list[str]):
|
||||||
|
# correlate(None): used inside queries that are themselves over `post` (the
|
||||||
|
# drops, the delete), where auto-correlation would bind this to the outer
|
||||||
|
# row instead of scanning the table.
|
||||||
|
return (
|
||||||
|
select(Post.id)
|
||||||
|
.where(*_other_poster_post_conditions(source_id, authors))
|
||||||
|
.correlate(None)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _image_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
"""Images that belong to a removed post and to no kept real post."""
|
||||||
|
doomed = _doomed_post_ids(source_id, authors)
|
||||||
|
keeper = aliased(Post)
|
||||||
|
on_doomed = or_(
|
||||||
|
ImageRecord.primary_post_id.in_(doomed),
|
||||||
|
exists().where(
|
||||||
|
ImageProvenance.image_record_id == ImageRecord.id,
|
||||||
|
ImageProvenance.post_id.in_(doomed),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
on_kept = or_(
|
||||||
|
and_(
|
||||||
|
ImageRecord.primary_post_id.isnot(None),
|
||||||
|
ImageRecord.primary_post_id.notin_(doomed),
|
||||||
|
),
|
||||||
|
exists().where(
|
||||||
|
ImageProvenance.image_record_id == ImageRecord.id,
|
||||||
|
ImageProvenance.post_id == keeper.id,
|
||||||
|
keeper.synthesized_by.is_(None),
|
||||||
|
keeper.id.notin_(doomed),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return [on_doomed, ~on_kept]
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
"""The synthetic drops that absorbed a removed message."""
|
||||||
|
member = aliased(Post)
|
||||||
|
return [
|
||||||
|
Post.source_id == source_id,
|
||||||
|
Post.synthesized_by.isnot(None),
|
||||||
|
exists().where(
|
||||||
|
member.absorbed_by_post_id == Post.id,
|
||||||
|
member.id.in_(_doomed_post_ids(source_id, authors)),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _attachment_conditions(source_id: int, authors: list[str]) -> list:
|
||||||
|
return [PostAttachment.post_id.in_(_doomed_post_ids(source_id, authors))]
|
||||||
|
|
||||||
|
|
||||||
|
def _load(session: Session, source_id: int) -> tuple[Source, list[str]]:
|
||||||
|
source = session.get(Source, source_id)
|
||||||
|
if source is None:
|
||||||
|
raise LookupError(source_id)
|
||||||
|
if source.platform != PLATFORM:
|
||||||
|
raise PosterCleanupError("Only a Discord source has posters to filter by.")
|
||||||
|
authors = source_authors(source)
|
||||||
|
if not authors:
|
||||||
|
raise PosterCleanupError(
|
||||||
|
"Set this source's 'Only posts by' list first — with no list, "
|
||||||
|
"every poster is wanted and nothing would be removed."
|
||||||
|
)
|
||||||
|
return source, authors
|
||||||
|
|
||||||
|
|
||||||
|
def _count(session: Session, stmt) -> int:
|
||||||
|
return session.execute(stmt).scalar_one()
|
||||||
|
|
||||||
|
|
||||||
|
def preview(session: Session, *, source_id: int) -> dict:
|
||||||
|
"""What `apply` would remove, per poster, without touching anything."""
|
||||||
|
_, authors = _load(session, source_id)
|
||||||
|
who = func.coalesce(
|
||||||
|
Post.raw_metadata["author_name"].as_string(),
|
||||||
|
Post.raw_metadata["author"].as_string(),
|
||||||
|
Post.raw_metadata["author_id"].as_string(),
|
||||||
|
)
|
||||||
|
rows = session.execute(
|
||||||
|
select(who, func.count(Post.id))
|
||||||
|
.where(*_other_poster_post_conditions(source_id, authors))
|
||||||
|
.group_by(who)
|
||||||
|
.order_by(func.count(Post.id).desc())
|
||||||
|
).all()
|
||||||
|
posters = [{"poster": name, "posts": n} for name, n in rows]
|
||||||
|
unknown = _count(session, select(func.count(Post.id)).where(
|
||||||
|
*_real_post_conditions(source_id), ~_has_poster(),
|
||||||
|
))
|
||||||
|
return {
|
||||||
|
"authors": authors,
|
||||||
|
"posters": posters,
|
||||||
|
"posts": sum(p["posts"] for p in posters),
|
||||||
|
"images": _count(session, select(func.count(ImageRecord.id)).where(
|
||||||
|
*_image_conditions(source_id, authors))),
|
||||||
|
"attachments": _count(session, select(func.count(PostAttachment.id)).where(
|
||||||
|
*_attachment_conditions(source_id, authors))),
|
||||||
|
"drops": _count(session, select(func.count(Post.id)).where(
|
||||||
|
*_drop_conditions(source_id, authors))),
|
||||||
|
"unknown_posts": unknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def confirm_token(projection: dict) -> str:
|
||||||
|
"""What the operator's apply must echo: the preview it was shown, so a
|
||||||
|
list edited between preview and apply can't delete an unseen set."""
|
||||||
|
return f"remove-{projection['posts']}-posts-{projection['images']}-images"
|
||||||
|
|
||||||
|
|
||||||
|
def apply(session: Session, *, source_id: int, images_root: Path) -> dict:
|
||||||
|
"""Remove them. Counts are taken before the deletes they describe — once
|
||||||
|
the rows are gone there is nothing left to count (snippet #3087 note 2)."""
|
||||||
|
projection = preview(session, source_id=source_id)
|
||||||
|
_, authors = _load(session, source_id)
|
||||||
|
|
||||||
|
image_ids = session.execute(
|
||||||
|
select(ImageRecord.id).where(*_image_conditions(source_id, authors))
|
||||||
|
).scalars().all()
|
||||||
|
# Drops first-class: found BEFORE their members go, since the link that
|
||||||
|
# finds them is the member's absorbed_by_post_id.
|
||||||
|
drop_ids = session.execute(
|
||||||
|
select(Post.id).where(*_drop_conditions(source_id, authors))
|
||||||
|
).scalars().all()
|
||||||
|
|
||||||
|
deleted = delete_images(session, image_ids=list(image_ids), images_root=images_root)
|
||||||
|
# Attachments before posts: post_attachment.post_id is SET NULL, and a
|
||||||
|
# NULL-post attachment collides on its partial unique index (see
|
||||||
|
# cleanup_service.delete_artist_cascade).
|
||||||
|
attachments = session.execute(
|
||||||
|
delete(PostAttachment).where(*_attachment_conditions(source_id, authors))
|
||||||
|
).rowcount or 0
|
||||||
|
posts = session.execute(
|
||||||
|
Post.__table__.delete().where(Post.id.in_(_doomed_post_ids(source_id, authors)))
|
||||||
|
).rowcount or 0
|
||||||
|
drops = 0
|
||||||
|
if drop_ids:
|
||||||
|
drops = session.execute(
|
||||||
|
Post.__table__.delete().where(Post.id.in_(drop_ids))
|
||||||
|
).rowcount or 0
|
||||||
|
session.commit()
|
||||||
|
log.info(
|
||||||
|
"discord poster cleanup (source %s, keeping %s): %d post(s), %d image(s), "
|
||||||
|
"%d attachment(s), %d drop(s) removed",
|
||||||
|
source_id, authors, posts, deleted["images_deleted"], attachments, drops,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
**projection,
|
||||||
|
"posts_deleted": posts,
|
||||||
|
"images_deleted": deleted["images_deleted"],
|
||||||
|
"files_deleted": deleted["files_deleted"],
|
||||||
|
"attachments_deleted": attachments,
|
||||||
|
"drops_deleted": drops,
|
||||||
|
}
|
||||||
@@ -23,6 +23,7 @@ from __future__ import annotations
|
|||||||
import asyncio
|
import asyncio
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .discord_ingester import DiscordIngester
|
||||||
from .gallery_dl import DownloadResult, ErrorType
|
from .gallery_dl import DownloadResult, ErrorType
|
||||||
from .ingest_core import DEFAULT_REVISIT_DAYS
|
from .ingest_core import DEFAULT_REVISIT_DAYS
|
||||||
from .patreon_ingester import PatreonIngester
|
from .patreon_ingester import PatreonIngester
|
||||||
@@ -31,9 +32,13 @@ from .platforms import known_platform_keys
|
|||||||
from .subscribestar_ingester import SubscribeStarIngester
|
from .subscribestar_ingester import SubscribeStarIngester
|
||||||
|
|
||||||
# Platforms whose download + verify go through the native ingester rather than
|
# Platforms whose download + verify go through the native ingester rather than
|
||||||
# gallery-dl. gallery-dl still serves the rest (hentaifoundry, discord) until
|
# gallery-dl. gallery-dl still serves the rest (hentaifoundry) until it
|
||||||
# they migrate too.
|
# migrates too. Discord joined in milestone 428.
|
||||||
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
NATIVE_INGESTER_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||||
|
|
||||||
|
# Native platforms whose feed id IS the source URL, so there is nothing to
|
||||||
|
# resolve: SubscribeStar's creator page, Discord's server/channel link.
|
||||||
|
_URL_IS_FEED = frozenset({"subscribestar", "discord"})
|
||||||
|
|
||||||
|
|
||||||
def _unsupported_platform_message(platform: str) -> str | None:
|
def _unsupported_platform_message(platform: str) -> str | None:
|
||||||
@@ -67,6 +72,7 @@ def _native_ingester_cls(platform: str):
|
|||||||
return {
|
return {
|
||||||
"patreon": PatreonIngester,
|
"patreon": PatreonIngester,
|
||||||
"subscribestar": SubscribeStarIngester,
|
"subscribestar": SubscribeStarIngester,
|
||||||
|
"discord": DiscordIngester,
|
||||||
}[platform]
|
}[platform]
|
||||||
|
|
||||||
|
|
||||||
@@ -126,10 +132,10 @@ async def _resolve_native_campaign_id(
|
|||||||
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
platform: str, url: str, cookies_path: str | None, overrides: dict,
|
||||||
) -> tuple[str | None, str | None]:
|
) -> tuple[str | None, str | None]:
|
||||||
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
"""`(campaign_id, resolved_campaign_id)` for a native source. SubscribeStar's
|
||||||
feed id IS the creator URL (no lookup → resolved None). Patreon resolves the
|
and Discord's feed id IS the source URL (no lookup → resolved None). Patreon
|
||||||
campaign id from the vanity URL (resolved non-None when a lookup actually ran,
|
resolves the campaign id from the vanity URL (resolved non-None when a lookup
|
||||||
so phase 3 caches it)."""
|
actually ran, so phase 3 caches it)."""
|
||||||
if platform == "subscribestar":
|
if platform in _URL_IS_FEED:
|
||||||
return url, None
|
return url, None
|
||||||
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
return await resolve_campaign_id_for_source(url, cookies_path, overrides)
|
||||||
|
|
||||||
@@ -252,6 +258,10 @@ async def verify_source_credential(
|
|||||||
from .subscribestar_ingester import verify_subscribestar_credential
|
from .subscribestar_ingester import verify_subscribestar_credential
|
||||||
|
|
||||||
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
return await verify_subscribestar_credential(url, cookies_path, config_overrides)
|
||||||
|
if platform == "discord":
|
||||||
|
from .discord_ingester import verify_discord_credential
|
||||||
|
|
||||||
|
return await verify_discord_credential(url, auth_token)
|
||||||
from .patreon_ingester import verify_patreon_credential
|
from .patreon_ingester import verify_patreon_credential
|
||||||
|
|
||||||
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
return await verify_patreon_credential(url, cookies_path, config_overrides)
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""Closing the download runs a worker restart orphaned (#4433).
|
||||||
|
|
||||||
|
Its own module, importing nothing but the model, because its caller is the
|
||||||
|
worker boot hook in `celery_signals` — which every download task imports via
|
||||||
|
`celery_app`. Living in `tasks.maintenance` put the whole maintenance import
|
||||||
|
graph, the membership roster included, on the fetch path, which
|
||||||
|
`test_gated_reason` forbids.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import literal, update
|
||||||
|
from sqlalchemy.dialects.postgresql import JSONB
|
||||||
|
|
||||||
|
from ..models import DownloadEvent
|
||||||
|
|
||||||
|
DOWNLOAD_INTERRUPTED_MESSAGE = (
|
||||||
|
"interrupted by a worker restart — the next check picks it up where it left off"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def interrupt_orphaned_download_events(session, *, booted_at: datetime) -> int:
|
||||||
|
"""Close the download events a restart orphaned, without blaming the source.
|
||||||
|
|
||||||
|
Called when the download lane comes up (#4433). Anything still
|
||||||
|
pending/running from before this boot belongs to the previous process:
|
||||||
|
a walk that outlived the 90s stop grace was SIGKILLed, and a queued or
|
||||||
|
serialize-deferred task is held unacked until Redis redelivers it about an
|
||||||
|
hour later. Left alone, the 30-min stall sweep would error each one and
|
||||||
|
bump `consecutive_failures`, backing the source off as if the platform had
|
||||||
|
failed.
|
||||||
|
|
||||||
|
Instead they end as `skipped` (terminal, not a failure) and the source is
|
||||||
|
not touched: `last_checked_at` keeps its old value, so the next tick finds
|
||||||
|
it due and the walk resumes from its checkpoint. A redelivered message
|
||||||
|
that arrives later finds no pending event and opens a fresh one.
|
||||||
|
|
||||||
|
An event promoted to running after the boot has `started_at` reset to its
|
||||||
|
real start (download_service), so it is never caught here. Does NOT commit.
|
||||||
|
"""
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
result = session.execute(
|
||||||
|
update(DownloadEvent)
|
||||||
|
.where(DownloadEvent.status.in_(["pending", "running"]))
|
||||||
|
.where(DownloadEvent.started_at < booted_at)
|
||||||
|
.values(
|
||||||
|
status="skipped",
|
||||||
|
finished_at=now,
|
||||||
|
error=DOWNLOAD_INTERRUPTED_MESSAGE,
|
||||||
|
metadata_=DownloadEvent.metadata_.op("||")(
|
||||||
|
literal({"error_type": "interrupted"}, JSONB)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.returning(DownloadEvent.id)
|
||||||
|
)
|
||||||
|
return len(result.all())
|
||||||
@@ -13,6 +13,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import logging
|
import logging
|
||||||
|
import time
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -39,7 +40,7 @@ from .gallery_dl import (
|
|||||||
walk_completed,
|
walk_completed,
|
||||||
)
|
)
|
||||||
from .importer import Importer
|
from .importer import Importer
|
||||||
from .ingest_core import DEFAULT_REVISIT_DAYS
|
from .ingest_core import DEFAULT_REVISIT_DAYS, PHASE3_DEADLINE_SECONDS
|
||||||
from .platforms import auth_type_for
|
from .platforms import auth_type_for
|
||||||
from .scheduler_service import set_platform_cooldown
|
from .scheduler_service import set_platform_cooldown
|
||||||
|
|
||||||
@@ -82,6 +83,7 @@ class DownloadService:
|
|||||||
|
|
||||||
async def download_source(self, source_id: int) -> int:
|
async def download_source(self, source_id: int) -> int:
|
||||||
"""Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
|
"""Returns DownloadEvent.id. Idempotent: in-flight events are returned as-is."""
|
||||||
|
started = time.monotonic()
|
||||||
setup = await self._phase1_setup(source_id)
|
setup = await self._phase1_setup(source_id)
|
||||||
if setup["status"] in ("skipped", "in_flight"):
|
if setup["status"] in ("skipped", "in_flight"):
|
||||||
return setup["event_id"]
|
return setup["event_id"]
|
||||||
@@ -174,6 +176,7 @@ class DownloadService:
|
|||||||
|
|
||||||
return await self._phase3_persist(
|
return await self._phase3_persist(
|
||||||
ctx["event_id"], ctx, dl_result, resolved_campaign_id,
|
ctx["event_id"], ctx, dl_result, resolved_campaign_id,
|
||||||
|
started=started,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _run_download(
|
async def _run_download(
|
||||||
@@ -268,6 +271,8 @@ class DownloadService:
|
|||||||
ctx: dict,
|
ctx: dict,
|
||||||
dl_result,
|
dl_result,
|
||||||
resolved_campaign_id: str | None,
|
resolved_campaign_id: str | None,
|
||||||
|
*,
|
||||||
|
started: float | None = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
# Cache Patreon campaign id on the source.
|
# Cache Patreon campaign id on the source.
|
||||||
if resolved_campaign_id:
|
if resolved_campaign_id:
|
||||||
@@ -303,7 +308,23 @@ class DownloadService:
|
|||||||
|
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
|
|
||||||
for path_str in dl_result.written_paths or []:
|
# Phase 3 shares the task's soft limit with the walk, and an import's
|
||||||
|
# cost is only estimated there. Native platforms stop starting imports
|
||||||
|
# at the deadline: what is left stays on disk with no record, which the
|
||||||
|
# next chunk's walk imports. gallery-dl platforms have no such pickup,
|
||||||
|
# so they import everything.
|
||||||
|
deadline = (
|
||||||
|
started + PHASE3_DEADLINE_SECONDS
|
||||||
|
if started is not None and uses_native_ingester(ctx["platform"])
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
written_paths = list(dl_result.written_paths or [])
|
||||||
|
deferred_from: int | None = None
|
||||||
|
|
||||||
|
for idx, path_str in enumerate(written_paths):
|
||||||
|
if deadline is not None and time.monotonic() >= deadline:
|
||||||
|
deferred_from = idx
|
||||||
|
break
|
||||||
if path_str in (dl_result.quarantined_paths or []):
|
if path_str in (dl_result.quarantined_paths or []):
|
||||||
continue
|
continue
|
||||||
path = Path(path_str)
|
path = Path(path_str)
|
||||||
@@ -423,6 +444,21 @@ class DownloadService:
|
|||||||
|
|
||||||
await loop.run_in_executor(None, _upsert)
|
await loop.run_in_executor(None, _upsert)
|
||||||
|
|
||||||
|
# Only now is it safe to call this walk's media seen: every file above
|
||||||
|
# has been through the importer. Had the run died before here they stay
|
||||||
|
# unmarked, and the next walk imports them from disk (ingest_core).
|
||||||
|
mark_seen = getattr(dl_result, "mark_seen_after_import", None)
|
||||||
|
deferred = written_paths[deferred_from:] if deferred_from is not None else []
|
||||||
|
if mark_seen is not None:
|
||||||
|
if deferred:
|
||||||
|
await loop.run_in_executor(
|
||||||
|
None, lambda: mark_seen(skip_paths=frozenset(deferred)),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
await loop.run_in_executor(None, mark_seen)
|
||||||
|
if deferred:
|
||||||
|
self._defer_unimported(dl_result, deferred_from, len(deferred))
|
||||||
|
|
||||||
# #830 recapture: backfill source_filehash on EXISTING on-disk images so
|
# #830 recapture: backfill source_filehash on EXISTING on-disk images so
|
||||||
# their post-body inline <img src=CDN> remaps to the local copy. A
|
# their post-body inline <img src=CDN> remaps to the local copy. A
|
||||||
# SEPARATE non-deleting channel (NOT the import list — that would unlink
|
# SEPARATE non-deleting channel (NOT the import list — that would unlink
|
||||||
@@ -523,6 +559,33 @@ class DownloadService:
|
|||||||
await self.async_session.commit()
|
await self.async_session.commit()
|
||||||
return event_id
|
return event_id
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _defer_unimported(dl_result, first: int, count: int) -> None:
|
||||||
|
"""Turn this chunk into a chunk boundary at the first file phase 3 left.
|
||||||
|
|
||||||
|
The next chunk resumes on the page that file came from, so its walk
|
||||||
|
meets every left file again, on disk with no record, and imports it. A
|
||||||
|
walk that reached its end has not finished while files wait, so a clean
|
||||||
|
result becomes PARTIAL; a real failure keeps its own error.
|
||||||
|
"""
|
||||||
|
cursors = getattr(dl_result, "written_cursors", None) or []
|
||||||
|
if first < len(cursors):
|
||||||
|
dl_result.cursor = cursors[first]
|
||||||
|
dl_result.phase3_imported_any = first > 0
|
||||||
|
msg = (
|
||||||
|
f"Imports stopped at the chunk deadline: {count} file(s) left "
|
||||||
|
"for the next chunk"
|
||||||
|
)
|
||||||
|
dl_result.stdout = f"{dl_result.stdout}\n{msg}" if dl_result.stdout else msg
|
||||||
|
log.info("%s", msg)
|
||||||
|
if dl_result.error_type in (None, ErrorType.PARTIAL, ErrorType.TIMEOUT) or (
|
||||||
|
is_informational(dl_result.error_type)
|
||||||
|
):
|
||||||
|
dl_result.success = False
|
||||||
|
dl_result.return_code = -1
|
||||||
|
dl_result.error_type = ErrorType.PARTIAL
|
||||||
|
dl_result.error_message = msg
|
||||||
|
|
||||||
async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None:
|
async def _apply_backfill_lifecycle(self, ctx: dict, dl_result) -> None:
|
||||||
"""Backfill state machine (plan #693, building on the cursor of #689).
|
"""Backfill state machine (plan #693, building on the cursor of #689).
|
||||||
|
|
||||||
@@ -590,9 +653,12 @@ class DownloadService:
|
|||||||
new_cursor = (
|
new_cursor = (
|
||||||
dl_result.cursor if uses_native_ingester(ctx["platform"]) else None
|
dl_result.cursor if uses_native_ingester(ctx["platform"]) else None
|
||||||
)
|
)
|
||||||
|
# Imports phase 3 finished before its deadline are progress even when
|
||||||
|
# the cursor went back to the page holding the files it left.
|
||||||
advanced = bool(
|
advanced = bool(
|
||||||
(new_cursor and new_cursor != old_cursor)
|
(new_cursor and new_cursor != old_cursor)
|
||||||
or dl_result.files_downloaded > 0
|
or dl_result.files_downloaded > 0
|
||||||
|
or getattr(dl_result, "phase3_imported_any", False)
|
||||||
)
|
)
|
||||||
if advanced:
|
if advanced:
|
||||||
if new_cursor:
|
if new_cursor:
|
||||||
|
|||||||
@@ -21,6 +21,15 @@ from .source_service import BACKFILL_MAX_CHUNKS
|
|||||||
|
|
||||||
log = logging.getLogger(__name__)
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# The probe runs while the chip is drawing; names that take longer are skipped.
|
||||||
|
_NAME_LOOKUP_SECONDS = 6.0
|
||||||
|
# The poster picker waits on two pages of a channel's messages (#4491).
|
||||||
|
_POSTER_LOOKUP_SECONDS = 20.0
|
||||||
|
|
||||||
|
# A source's poster list (discord_ingester.AUTHORS_KEY) holds user ids; this
|
||||||
|
# keeps the name each id was picked under, for showing the list to a person.
|
||||||
|
AUTHOR_LABELS_KEY = "discord_author_labels"
|
||||||
|
|
||||||
|
|
||||||
class UnknownPlatformError(Exception):
|
class UnknownPlatformError(Exception):
|
||||||
"""URL didn't match any platform pattern."""
|
"""URL didn't match any platform pattern."""
|
||||||
@@ -30,6 +39,19 @@ class InvalidUrlError(Exception):
|
|||||||
"""URL was empty or missing a scheme."""
|
"""URL was empty or missing a scheme."""
|
||||||
|
|
||||||
|
|
||||||
|
class UnknownArtistError(Exception):
|
||||||
|
"""quick-add named an `artist_id` that does not exist."""
|
||||||
|
|
||||||
|
|
||||||
|
class PosterLookupError(Exception):
|
||||||
|
"""The Discord poster picker couldn't read the channel (no token, no
|
||||||
|
access, not a channel page, or Discord was slow)."""
|
||||||
|
|
||||||
|
|
||||||
|
class UnknownSourceError(Exception):
|
||||||
|
"""No source follows this Discord channel or its server."""
|
||||||
|
|
||||||
|
|
||||||
# Mirrored byte-for-byte from extension/lib/platforms.js
|
# Mirrored byte-for-byte from extension/lib/platforms.js
|
||||||
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
# PLATFORM_ARTIST_PATTERNS. Keep these two copies in sync by hand —
|
||||||
# reviewers catch drift.
|
# reviewers catch drift.
|
||||||
@@ -55,8 +77,39 @@ _PLATFORM_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|||||||
r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
|
r"^https?://(?:www\.)?hentai-foundry\.com/user/(?P<slug>[^/?#]+)",
|
||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)),
|
)),
|
||||||
|
# A Discord URL names a server or a channel, never a creator, so the slug is
|
||||||
|
# `<server>` or `<server>/<channel>` and the artist is chosen, not derived.
|
||||||
|
# A trailing message id (a jump link) still names its channel. DMs (`@me`)
|
||||||
|
# are not sources; thread links (`/threads/`) are left to the manual form.
|
||||||
|
("discord", re.compile(
|
||||||
|
r"^https?://(?:www\.|ptb\.|canary\.)?discord\.com/channels/"
|
||||||
|
r"(?P<slug>\d+(?:/\d+)?)(?:/\d+)?/?(?:[?#].*)?$",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
DISCORD = "discord"
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_source_url(platform: str, url: str, slug: str) -> str:
|
||||||
|
"""The URL a new source is stored under. Discord's is rebuilt from the ids
|
||||||
|
— the form the manual Add form and the ingester use — so a jump link, a
|
||||||
|
ptb/canary host or a trailing slash never makes a second source for the
|
||||||
|
same channel. Every other platform keeps the URL as given."""
|
||||||
|
if platform == DISCORD:
|
||||||
|
return f"https://discord.com/channels/{slug}"
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def _discord_ids(url: str) -> tuple[str | None, str | None] | None:
|
||||||
|
"""`(server_id, channel_id)` of a stored Discord source URL, None if it
|
||||||
|
does not parse (a DM or thread link, or an old malformed row)."""
|
||||||
|
from .discord_client import DiscordAPIError, parse_source_url
|
||||||
|
try:
|
||||||
|
return parse_source_url(url)
|
||||||
|
except DiscordAPIError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class ExtensionService:
|
class ExtensionService:
|
||||||
def __init__(self, session: AsyncSession, crypto=None) -> None:
|
def __init__(self, session: AsyncSession, crypto=None) -> None:
|
||||||
@@ -65,29 +118,179 @@ class ExtensionService:
|
|||||||
# add-time. None → skip resolution, fall back to the handle.
|
# add-time. None → skip resolution, fall back to the handle.
|
||||||
self._crypto = crypto
|
self._crypto = crypto
|
||||||
|
|
||||||
async def quick_add_source(self, url: str) -> dict:
|
async def quick_add_source(
|
||||||
|
self,
|
||||||
|
url: str,
|
||||||
|
*,
|
||||||
|
artist_id: int | None = None,
|
||||||
|
artist_name: str | None = None,
|
||||||
|
use_platform_name: bool = False,
|
||||||
|
discord_authors: list[dict] | None = None,
|
||||||
|
) -> dict:
|
||||||
|
"""Add `url` as a source. `artist_id` connects it to an existing
|
||||||
|
artist, `artist_name` to that artist (created if new); with neither,
|
||||||
|
the artist is resolved from the platform as before.
|
||||||
|
|
||||||
|
`use_platform_name` applies the operator's convention that the Patreon
|
||||||
|
name is canon: a Patreon source added to an existing artist renames
|
||||||
|
that artist to the creator's Patreon display name. Name only — the
|
||||||
|
slug, and every path keyed off it, never moves (#130). Ignored on every
|
||||||
|
other platform, and when the name can't be read.
|
||||||
|
|
||||||
|
`discord_authors` ([{id, name}], #4491) sets a Discord source's poster
|
||||||
|
list in the same step, on a new source or one already there."""
|
||||||
platform, raw_slug = self._derive(url)
|
platform, raw_slug = self._derive(url)
|
||||||
|
url = canonical_source_url(platform, url, raw_slug)
|
||||||
|
renamed_from = None
|
||||||
# Identity by SOURCE handle (#130): an existing (platform, url) source
|
# Identity by SOURCE handle (#130): an existing (platform, url) source
|
||||||
# keeps its artist on re-add — even if that artist was since renamed (its
|
# keeps its artist on re-add — even if that artist was since renamed (its
|
||||||
# frozen slug no longer matches the current name). Only a genuinely new
|
# frozen slug no longer matches the current name), and even when the
|
||||||
# source resolves/creates an artist.
|
# add named a different artist. Only a genuinely new source
|
||||||
existing = (await self.session.execute(
|
# resolves/creates an artist.
|
||||||
select(Source).where(Source.platform == platform, Source.url == url)
|
existing = await self._existing_source(platform, url)
|
||||||
)).scalar_one_or_none()
|
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
artist = (await self.session.execute(
|
artist = (await self.session.execute(
|
||||||
select(Artist).where(Artist.id == existing.artist_id)
|
select(Artist).where(Artist.id == existing.artist_id)
|
||||||
)).scalar_one()
|
)).scalar_one()
|
||||||
|
if platform == DISCORD and discord_authors is not None:
|
||||||
|
await self._apply_posters(existing, discord_authors)
|
||||||
return self._shape(existing, artist, created_source=False, created_artist=False)
|
return self._shape(existing, artist, created_source=False, created_artist=False)
|
||||||
|
|
||||||
# New source → name the artist properly by resolving the real display
|
if artist_id is not None:
|
||||||
# name from the platform (falls back to the URL handle).
|
artist = (await self.session.execute(
|
||||||
name = await self._resolve_artist_name(platform, raw_slug, url)
|
select(Artist).where(Artist.id == artist_id)
|
||||||
artist, created_artist = await self._find_or_create_artist(name)
|
)).scalar_one_or_none()
|
||||||
|
if artist is None:
|
||||||
|
raise UnknownArtistError(f"no artist with id {artist_id}")
|
||||||
|
created_artist = False
|
||||||
|
if use_platform_name and platform == "patreon":
|
||||||
|
renamed_from = await self._adopt_patreon_name(artist, raw_slug, url)
|
||||||
|
else:
|
||||||
|
name = (artist_name or "").strip()
|
||||||
|
if not name:
|
||||||
|
# Name the artist properly by resolving the real display name
|
||||||
|
# from the platform (falls back to the URL handle).
|
||||||
|
name = await self._resolve_artist_name(platform, raw_slug, url)
|
||||||
|
artist, created_artist = await self._find_or_create_artist(name)
|
||||||
source, created_source = await self._find_or_create_source(
|
source, created_source = await self._find_or_create_source(
|
||||||
artist_id=artist.id, platform=platform, url=url,
|
artist_id=artist.id, platform=platform, url=url,
|
||||||
)
|
)
|
||||||
return self._shape(source, artist, created_source, created_artist)
|
if platform == DISCORD and discord_authors is not None:
|
||||||
|
await self._apply_posters(source, discord_authors)
|
||||||
|
shaped = self._shape(source, artist, created_source, created_artist)
|
||||||
|
if renamed_from is not None:
|
||||||
|
shaped["renamed_from"] = renamed_from
|
||||||
|
return shaped
|
||||||
|
|
||||||
|
# -- Discord poster picker (#4491) ------------------------------------------
|
||||||
|
|
||||||
|
def _discord_page(self, url: str) -> tuple[str, str]:
|
||||||
|
platform, raw_slug = self._derive(url)
|
||||||
|
if platform != DISCORD:
|
||||||
|
raise InvalidUrlError("not a Discord page")
|
||||||
|
server_id, _, channel_id = raw_slug.partition("/")
|
||||||
|
if not channel_id:
|
||||||
|
raise PosterLookupError("Open a channel to see who posts in it.")
|
||||||
|
return server_id, channel_id
|
||||||
|
|
||||||
|
async def _source_for_page(self, server_id: str, channel_id: str) -> Source | None:
|
||||||
|
"""The source this channel's posts land on: the channel's own, else a
|
||||||
|
whole-server source that walks it."""
|
||||||
|
base = f"https://discord.com/channels/{server_id}"
|
||||||
|
return (
|
||||||
|
await self._existing_source(DISCORD, f"{base}/{channel_id}")
|
||||||
|
or await self._existing_source(DISCORD, base)
|
||||||
|
)
|
||||||
|
|
||||||
|
async def discord_posters(self, url: str) -> dict:
|
||||||
|
"""Who posts in the channel `url` shows, newest `_POSTER_SCAN`
|
||||||
|
messages deep, with the likely creator marked — and, when a source
|
||||||
|
already follows it, the poster list that source has now."""
|
||||||
|
server_id, channel_id = self._discord_page(url)
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from .credential_service import CredentialService
|
||||||
|
from .discord_client import DiscordAPIError, DiscordClient
|
||||||
|
token = None
|
||||||
|
if self._crypto is not None:
|
||||||
|
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||||
|
if not token:
|
||||||
|
raise PosterLookupError("No Discord token is saved in FabledCurator.")
|
||||||
|
client = DiscordClient(token, max_retries=0)
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
try:
|
||||||
|
found = await asyncio.wait_for(
|
||||||
|
loop.run_in_executor(None, client.recent_posters, server_id, channel_id),
|
||||||
|
timeout=_POSTER_LOOKUP_SECONDS,
|
||||||
|
)
|
||||||
|
except TimeoutError as exc:
|
||||||
|
raise PosterLookupError("Discord took too long to answer; try again.") from exc
|
||||||
|
except DiscordAPIError as exc:
|
||||||
|
raise PosterLookupError(f"Couldn't read this channel: {exc}") from exc
|
||||||
|
source = await self._source_for_page(server_id, channel_id)
|
||||||
|
co = (source.config_overrides or {}) if source is not None else {}
|
||||||
|
from .discord_ingester import AUTHORS_KEY
|
||||||
|
return {
|
||||||
|
**found,
|
||||||
|
"source_id": source.id if source is not None else None,
|
||||||
|
"selected": list(co.get(AUTHORS_KEY) or []),
|
||||||
|
# So someone on the list who hasn't posted lately still shows by
|
||||||
|
# name, and can be unticked.
|
||||||
|
"selected_labels": dict(co.get(AUTHOR_LABELS_KEY) or {}),
|
||||||
|
}
|
||||||
|
|
||||||
|
async def set_discord_posters(self, url: str, authors: list[dict]) -> dict:
|
||||||
|
"""Store the picked posters on the source that follows `url`'s channel."""
|
||||||
|
server_id, channel_id = self._discord_page(url)
|
||||||
|
source = await self._source_for_page(server_id, channel_id)
|
||||||
|
if source is None:
|
||||||
|
raise UnknownSourceError("No source follows this channel yet.")
|
||||||
|
return await self._apply_posters(source, authors)
|
||||||
|
|
||||||
|
async def _apply_posters(self, source: Source, authors: list[dict]) -> dict:
|
||||||
|
"""Ids are what the list keys on — a username can change on a whim,
|
||||||
|
an id can't — and the name each was picked under rides beside it for
|
||||||
|
showing the list to a person. An empty pick clears the list, which
|
||||||
|
takes everyone's posts again."""
|
||||||
|
from .discord_ingester import AUTHORS_KEY
|
||||||
|
picked = [a for a in authors if str(a.get("id") or "").strip()]
|
||||||
|
co = dict(source.config_overrides or {})
|
||||||
|
if picked:
|
||||||
|
co[AUTHORS_KEY] = [str(a["id"]).strip() for a in picked]
|
||||||
|
co[AUTHOR_LABELS_KEY] = {
|
||||||
|
str(a["id"]).strip(): str(a.get("name") or a["id"]) for a in picked
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
co.pop(AUTHORS_KEY, None)
|
||||||
|
co.pop(AUTHOR_LABELS_KEY, None)
|
||||||
|
source.config_overrides = co or None
|
||||||
|
await self.session.commit()
|
||||||
|
return {"source_id": source.id, "discord_authors": co.get(AUTHORS_KEY, [])}
|
||||||
|
|
||||||
|
async def _adopt_patreon_name(self, artist, raw_slug: str, url: str) -> str | None:
|
||||||
|
"""Rename `artist` to the Patreon display name; the old name when it
|
||||||
|
changed, else None. Unreadable name → no rename, never the handle."""
|
||||||
|
name = await self._platform_display_name("patreon", raw_slug, url)
|
||||||
|
if not name or name == artist.name:
|
||||||
|
return None
|
||||||
|
old = artist.name
|
||||||
|
artist.name = name
|
||||||
|
await self.session.commit()
|
||||||
|
return old
|
||||||
|
|
||||||
|
async def _existing_source(self, platform: str, url: str) -> Source | None:
|
||||||
|
"""The source this URL already is, whichever artist owns it. Discord
|
||||||
|
compares ids, not strings, so a row stored before canonicalisation (a
|
||||||
|
ptb host, a trailing slash) is still found."""
|
||||||
|
if platform != DISCORD:
|
||||||
|
return (await self.session.execute(
|
||||||
|
select(Source).where(Source.platform == platform, Source.url == url)
|
||||||
|
)).scalars().first()
|
||||||
|
want = _discord_ids(url)
|
||||||
|
rows = (await self.session.execute(
|
||||||
|
select(Source).where(Source.platform == DISCORD).order_by(Source.id)
|
||||||
|
)).scalars().all()
|
||||||
|
return next((s for s in rows if _discord_ids(s.url) == want), None)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
|
def _shape(source, artist, created_source: bool, created_artist: bool) -> dict:
|
||||||
@@ -117,8 +320,23 @@ class ExtensionService:
|
|||||||
platforms (and any failure — no credential, network error) fall back to
|
platforms (and any failure — no credential, network error) fall back to
|
||||||
the URL handle, which is already readable.
|
the URL handle, which is already readable.
|
||||||
The resolvers are sync, so they run in an executor."""
|
The resolvers are sync, so they run in an executor."""
|
||||||
|
if platform == DISCORD:
|
||||||
|
# The server's name: what the operator knows the community as.
|
||||||
|
server_id = raw_slug.split("/", 1)[0]
|
||||||
|
names = await self._discord_names(server_id, None)
|
||||||
|
return names.get("server") or f"Discord {server_id}"
|
||||||
|
return await self._platform_display_name(platform, raw_slug, url) or raw_slug
|
||||||
|
|
||||||
|
async def _platform_display_name(
|
||||||
|
self, platform: str, raw_slug: str, url: str
|
||||||
|
) -> str | None:
|
||||||
|
"""The creator's display name as Patreon or SubscribeStar shows it, read
|
||||||
|
with the stored cookies; None when it can't be read (no credential, a
|
||||||
|
network error, a slow answer, any other platform). None, not the handle,
|
||||||
|
so a caller can tell a real name from a fallback — a rename to the
|
||||||
|
Patreon name must never rename to a URL handle instead."""
|
||||||
if self._crypto is None or platform not in ("patreon", "subscribestar"):
|
if self._crypto is None or platform not in ("patreon", "subscribestar"):
|
||||||
return raw_slug
|
return None
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from .credential_service import CredentialService
|
from .credential_service import CredentialService
|
||||||
@@ -128,7 +346,7 @@ class ExtensionService:
|
|||||||
if platform == "patreon":
|
if platform == "patreon":
|
||||||
cookies = await cred.get_cookies_path("patreon")
|
cookies = await cred.get_cookies_path("patreon")
|
||||||
from .patreon_resolver import resolve_display_name
|
from .patreon_resolver import resolve_display_name
|
||||||
name = await loop.run_in_executor(
|
call = loop.run_in_executor(
|
||||||
None, resolve_display_name, raw_slug,
|
None, resolve_display_name, raw_slug,
|
||||||
str(cookies) if cookies else None,
|
str(cookies) if cookies else None,
|
||||||
)
|
)
|
||||||
@@ -136,15 +354,14 @@ class ExtensionService:
|
|||||||
cookies = await cred.get_cookies_path("subscribestar")
|
cookies = await cred.get_cookies_path("subscribestar")
|
||||||
from .subscribestar_client import SubscribeStarClient
|
from .subscribestar_client import SubscribeStarClient
|
||||||
client = SubscribeStarClient(str(cookies) if cookies else None)
|
client = SubscribeStarClient(str(cookies) if cookies else None)
|
||||||
name = await loop.run_in_executor(
|
call = loop.run_in_executor(None, client.resolve_display_name, url)
|
||||||
None, client.resolve_display_name, url
|
name = await asyncio.wait_for(call, timeout=_NAME_LOOKUP_SECONDS)
|
||||||
)
|
|
||||||
except Exception as exc: # resolution is best-effort — never block the add
|
except Exception as exc: # resolution is best-effort — never block the add
|
||||||
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
log.warning("artist display-name resolution failed (%s): %s", platform, exc)
|
||||||
return raw_slug
|
return None
|
||||||
return name or raw_slug
|
return (name or "").strip() or None
|
||||||
|
|
||||||
async def probe(self, url: str) -> dict:
|
async def probe(self, url: str, *, names: bool = False) -> dict:
|
||||||
"""Read-only resolution of a creator-page URL against the FC DB.
|
"""Read-only resolution of a creator-page URL against the FC DB.
|
||||||
Returns one of:
|
Returns one of:
|
||||||
- {state: 'unknown_platform'} — URL didn't match any
|
- {state: 'unknown_platform'} — URL didn't match any
|
||||||
@@ -160,21 +377,30 @@ class ExtensionService:
|
|||||||
— exact (artist, platform,
|
— exact (artist, platform,
|
||||||
url) Source already exists
|
url) Source already exists
|
||||||
|
|
||||||
Side-effect-free: two SELECTs at most.
|
`names` (the Add panel asks, the chip does not) adds `display_name`:
|
||||||
|
the creator's name as Patreon/SubscribeStar shows it, or None. It costs
|
||||||
|
a request to the platform, so a plain page view never pays it.
|
||||||
|
|
||||||
|
Side-effect-free: two SELECTs at most, plus that one lookup.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
platform, raw_slug = self._derive(url)
|
platform, raw_slug = self._derive(url)
|
||||||
except (UnknownPlatformError, InvalidUrlError):
|
except (UnknownPlatformError, InvalidUrlError):
|
||||||
return {"state": "unknown_platform"}
|
return {"state": "unknown_platform"}
|
||||||
|
if platform == DISCORD:
|
||||||
|
return await self._probe_discord(raw_slug)
|
||||||
|
|
||||||
slug = slugify(raw_slug)
|
slug = slugify(raw_slug)
|
||||||
|
result: dict = {"platform": platform, "slug": slug}
|
||||||
|
if names:
|
||||||
|
result["display_name"] = await self._platform_display_name(
|
||||||
|
platform, raw_slug, url,
|
||||||
|
)
|
||||||
artist = (await self.session.execute(
|
artist = (await self.session.execute(
|
||||||
select(Artist).where(Artist.slug == slug)
|
select(Artist).where(Artist.slug == slug)
|
||||||
)).scalar_one_or_none()
|
)).scalar_one_or_none()
|
||||||
if artist is None:
|
if artist is None:
|
||||||
return {"state": "new", "platform": platform, "slug": slug}
|
return {"state": "new", **result}
|
||||||
|
|
||||||
artist_payload = {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
|
||||||
|
|
||||||
source = (await self.session.execute(
|
source = (await self.session.execute(
|
||||||
select(Source).where(
|
select(Source).where(
|
||||||
@@ -184,26 +410,114 @@ class ExtensionService:
|
|||||||
)
|
)
|
||||||
)).scalar_one_or_none()
|
)).scalar_one_or_none()
|
||||||
if source is None:
|
if source is None:
|
||||||
return {
|
return {"state": "artist_match", **result, "artist": self._artist_payload(artist)}
|
||||||
"state": "artist_match",
|
|
||||||
"platform": platform,
|
|
||||||
"slug": slug,
|
|
||||||
"artist": artist_payload,
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"state": "source_match",
|
"state": "source_match",
|
||||||
"platform": platform,
|
**result,
|
||||||
"slug": slug,
|
"artist": self._artist_payload(artist),
|
||||||
"artist": artist_payload,
|
"source": self._source_payload(source),
|
||||||
"source": {
|
}
|
||||||
"id": source.id,
|
|
||||||
"artist_id": source.artist_id,
|
async def _probe_discord(self, raw_slug: str) -> dict:
|
||||||
"platform": source.platform,
|
"""probe for a Discord server or channel. The states mean what they
|
||||||
"url": source.url,
|
mean elsewhere, but the artist is never read off the URL:
|
||||||
"enabled": source.enabled,
|
|
||||||
|
- source_match: this channel is a source — or the whole server is
|
||||||
|
(`covered_by_server`), which already walks every channel;
|
||||||
|
- artist_match: another source on this server belongs to an artist,
|
||||||
|
the one this channel most likely belongs to too (a suggestion the
|
||||||
|
Add panel preselects, not a decision);
|
||||||
|
- new: nothing on this server yet.
|
||||||
|
|
||||||
|
`discord` carries the ids, both canonical URLs and the display names,
|
||||||
|
read with the stored token; a name that can't be read is None."""
|
||||||
|
server_id, _, channel_id = raw_slug.partition("/")
|
||||||
|
channel_id = channel_id or None
|
||||||
|
rows = (await self.session.execute(
|
||||||
|
select(Source, Artist)
|
||||||
|
.join(Artist, Artist.id == Source.artist_id)
|
||||||
|
.where(Source.platform == DISCORD)
|
||||||
|
.order_by(Source.id)
|
||||||
|
)).all()
|
||||||
|
exact = server_whole = on_server = None
|
||||||
|
for source, artist in rows:
|
||||||
|
ids = _discord_ids(source.url)
|
||||||
|
if ids is None or ids[0] != server_id:
|
||||||
|
continue
|
||||||
|
if ids[1] == channel_id and exact is None:
|
||||||
|
exact = (source, artist)
|
||||||
|
elif ids[1] is None and server_whole is None:
|
||||||
|
server_whole = (source, artist)
|
||||||
|
if on_server is None:
|
||||||
|
on_server = (source, artist)
|
||||||
|
|
||||||
|
names = await self._discord_names(server_id, channel_id)
|
||||||
|
base = f"https://discord.com/channels/{server_id}"
|
||||||
|
result: dict = {
|
||||||
|
"platform": DISCORD,
|
||||||
|
"slug": raw_slug,
|
||||||
|
"discord": {
|
||||||
|
"server_id": server_id,
|
||||||
|
"channel_id": channel_id,
|
||||||
|
"server_name": names.get("server"),
|
||||||
|
"channel_name": names.get("channel"),
|
||||||
|
"server_url": base,
|
||||||
|
"channel_url": f"{base}/{channel_id}" if channel_id else None,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
hit = exact or server_whole
|
||||||
|
if hit is not None:
|
||||||
|
source, artist = hit
|
||||||
|
result.update(
|
||||||
|
state="source_match",
|
||||||
|
artist=self._artist_payload(artist),
|
||||||
|
source=self._source_payload(source),
|
||||||
|
covered_by_server=exact is None,
|
||||||
|
)
|
||||||
|
elif on_server is not None:
|
||||||
|
result.update(state="artist_match", artist=self._artist_payload(on_server[1]))
|
||||||
|
else:
|
||||||
|
result["state"] = "new"
|
||||||
|
return result
|
||||||
|
|
||||||
|
async def _discord_names(self, server_id: str | None, channel_id: str | None) -> dict:
|
||||||
|
"""Server/channel display names via the stored Discord token. Never
|
||||||
|
raises and never waits out a rate limit: it runs while the operator
|
||||||
|
looks at a page, so a slow or missing answer just means no names."""
|
||||||
|
if self._crypto is None:
|
||||||
|
return {}
|
||||||
|
import asyncio
|
||||||
|
|
||||||
|
from .credential_service import CredentialService
|
||||||
|
from .discord_client import DiscordClient
|
||||||
|
try:
|
||||||
|
token = await CredentialService(self.session, self._crypto).get_token(DISCORD)
|
||||||
|
if not token:
|
||||||
|
return {}
|
||||||
|
client = DiscordClient(token, max_retries=0)
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
return await asyncio.wait_for(
|
||||||
|
loop.run_in_executor(None, client.describe, server_id, channel_id),
|
||||||
|
timeout=_NAME_LOOKUP_SECONDS,
|
||||||
|
)
|
||||||
|
except Exception as exc: # names are decoration — never fail the call
|
||||||
|
log.info("Discord name lookup failed: %s", exc)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _artist_payload(artist) -> dict:
|
||||||
|
return {"id": artist.id, "name": artist.name, "slug": artist.slug}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _source_payload(source) -> dict:
|
||||||
|
return {
|
||||||
|
"id": source.id,
|
||||||
|
"artist_id": source.artist_id,
|
||||||
|
"platform": source.platform,
|
||||||
|
"url": source.url,
|
||||||
|
"enabled": source.enabled,
|
||||||
|
}
|
||||||
|
|
||||||
def _derive(self, url: str) -> tuple[str, str]:
|
def _derive(self, url: str) -> tuple[str, str]:
|
||||||
if not isinstance(url, str) or not url.strip():
|
if not isinstance(url, str) or not url.strip():
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ import subprocess
|
|||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from enum import StrEnum
|
from enum import StrEnum
|
||||||
@@ -94,48 +95,6 @@ BACKFILL_CHUNK_SECONDS = 600
|
|||||||
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
_DEFAULT_GDL_TIMEOUT_SECONDS = 870
|
||||||
|
|
||||||
|
|
||||||
# --- Discord naming ---------------------------------------------------------
|
|
||||||
#
|
|
||||||
# Derived from a REAL sidecar (operator's instance, 2026-09-13), not from memory
|
|
||||||
# of gallery-dl's extractor. What gallery-dl's discord extractor actually emits
|
|
||||||
# for an attachment: `channel` is a plain STRING (the channel's name), the
|
|
||||||
# message is `message_id`, the attachment's position in it is `num`, and there
|
|
||||||
# is NO `id` key at all.
|
|
||||||
#
|
|
||||||
# The previous patterns asked for `{channel[name]}` and `{id}`. Both render as
|
|
||||||
# "None", so every Discord download since the platform was added landed in a
|
|
||||||
# directory called `None` as `<date>_None_<original name>`. Worse, the sidecar was
|
|
||||||
# named `{filename}.json` — the attachment's ORIGINAL name — which (a) `find_
|
|
||||||
# sidecar` can never pair with `<date>_None_<name>.png`, so no Discord file ever
|
|
||||||
# got a Post or a post date, and (b) collides: every `image.png` in a channel
|
|
||||||
# overwrote the same `image.json`, so the one sidecar that survived described
|
|
||||||
# whichever message happened to be written last.
|
|
||||||
#
|
|
||||||
# The fix names the sidecar EXACTLY like the media minus its extension, so
|
|
||||||
# `find_sidecar`'s first candidate (`media.with_suffix(".json")`) is the match
|
|
||||||
# and the name is unique per attachment. tests/test_gallery_dl_naming.py renders
|
|
||||||
# these patterns against a sanitized copy of the real sidecar, so a key that
|
|
||||||
# does not exist fails CI instead of silently becoming "None".
|
|
||||||
DISCORD_FILENAME = "{date:%Y%m%d}_{message_id}_{num:>02}_{filename}.{extension}"
|
|
||||||
DISCORD_DIRECTORY = ["{channel}"]
|
|
||||||
|
|
||||||
|
|
||||||
def sidecar_name_for(media_pattern: str) -> str | None:
|
|
||||||
"""The metadata filename pattern that names a sidecar exactly like its media.
|
|
||||||
|
|
||||||
Returns None for a pattern that does not end in `.{extension}`, since then
|
|
||||||
there is no media stem to mirror and the caller must fall back.
|
|
||||||
"""
|
|
||||||
suffix = ".{extension}"
|
|
||||||
if not media_pattern.endswith(suffix):
|
|
||||||
return None
|
|
||||||
return media_pattern[: -len(suffix)] + ".json"
|
|
||||||
|
|
||||||
|
|
||||||
def metadata_postprocessor(filename: str) -> dict:
|
|
||||||
return {"name": "metadata", "mode": "json", "directory": ".", "filename": filename}
|
|
||||||
|
|
||||||
|
|
||||||
def archive_path(images_root: Path) -> Path:
|
def archive_path(images_root: Path) -> Path:
|
||||||
"""gallery-dl's download archive: the record of what it has already fetched.
|
"""gallery-dl's download archive: the record of what it has already fetched.
|
||||||
|
|
||||||
@@ -210,6 +169,10 @@ class DownloadResult:
|
|||||||
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
# backfilled (inline-image localization) WITHOUT re-download or unlink. Empty
|
||||||
# on the gallery-dl path and outside recapture.
|
# on the gallery-dl path and outside recapture.
|
||||||
relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
|
relink_source_paths: list[tuple[str, str, str]] = field(default_factory=list)
|
||||||
|
# Native ingester only: one entry per `written_paths` entry, the backfill
|
||||||
|
# cursor of the page that file came from. Phase 3 resumes the next chunk
|
||||||
|
# there when it leaves files unimported at its deadline. Empty on gallery-dl.
|
||||||
|
written_cursors: list[str | None] = field(default_factory=list)
|
||||||
stdout: str = ""
|
stdout: str = ""
|
||||||
stderr: str = ""
|
stderr: str = ""
|
||||||
return_code: int = 0
|
return_code: int = 0
|
||||||
@@ -230,6 +193,15 @@ class DownloadResult:
|
|||||||
# the platform cooldown matches the hint instead of a flat default. None when
|
# the platform cooldown matches the hint instead of a flat default. None when
|
||||||
# unknown (no header, or not a rate-limit failure).
|
# unknown (no header, or not a rate-limit failure).
|
||||||
retry_after_seconds: float | None = None
|
retry_after_seconds: float | None = None
|
||||||
|
# Native ingester only: marks this walk's fetched media seen in its ledger.
|
||||||
|
# Phase 3 calls it AFTER the import loop, never before — a file marked seen
|
||||||
|
# but not yet imported is invisible to every later walk, so a run killed in
|
||||||
|
# between orphaned it for good (TamadaHeijun's 12PCG post lost 7 of 13
|
||||||
|
# images to a stranded run, 2026-09-24). Unmarked, the next walk finds the
|
||||||
|
# file on disk with no ImageRecord and imports it. Takes `skip_paths`: the
|
||||||
|
# written paths phase 3 did not import, which stay unmarked for the same
|
||||||
|
# reason. None on gallery-dl.
|
||||||
|
mark_seen_after_import: Callable[..., None] | None = None
|
||||||
|
|
||||||
|
|
||||||
def extract_errors_warnings(stderr: str) -> str:
|
def extract_errors_warnings(stderr: str) -> str:
|
||||||
@@ -417,27 +389,16 @@ class GalleryDLService:
|
|||||||
# (services/patreon_ingester.py), not gallery-dl.
|
# (services/patreon_ingester.py), not gallery-dl.
|
||||||
PLATFORM_DEFAULTS = {
|
PLATFORM_DEFAULTS = {
|
||||||
# subscribestar removed — native-ingester platform now (#71); pixiv
|
# subscribestar removed — native-ingester platform now (#71); pixiv
|
||||||
# removed likewise (#129); deviantart removed at #3069 as a dropped
|
# removed likewise (#129); discord likewise (milestone 428, whose
|
||||||
# platform, not a migrated one. The remaining entries are the
|
# downloader keeps this config's on-disk naming); deviantart removed at
|
||||||
# gallery-dl platforms not yet migrated.
|
# #3069 as a dropped platform, not a migrated one. HentaiFoundry is the
|
||||||
|
# one platform left here, by the operator's choice not to migrate it.
|
||||||
"hentaifoundry": {
|
"hentaifoundry": {
|
||||||
"content_types": ["all"],
|
"content_types": ["all"],
|
||||||
"directory": [],
|
"directory": [],
|
||||||
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
"filename": "{category}_{index:>03}_{title[:50]}.{extension}",
|
||||||
"include": "all",
|
"include": "all",
|
||||||
},
|
},
|
||||||
"discord": {
|
|
||||||
"content_types": ["all"],
|
|
||||||
"directory": DISCORD_DIRECTORY,
|
|
||||||
"filename": DISCORD_FILENAME,
|
|
||||||
# Overrides the global `{filename}.json` sidecar for this extractor
|
|
||||||
# only — see the Discord naming note above.
|
|
||||||
"postprocessors": [metadata_postprocessor(sidecar_name_for(DISCORD_FILENAME))],
|
|
||||||
"embeds": "all",
|
|
||||||
"stickers": True,
|
|
||||||
"reactions": False,
|
|
||||||
"threads": True,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -552,17 +513,6 @@ class GalleryDLService:
|
|||||||
if source_config.filename_pattern:
|
if source_config.filename_pattern:
|
||||||
platform_section["filename"] = source_config.filename_pattern
|
platform_section["filename"] = source_config.filename_pattern
|
||||||
|
|
||||||
# A platform that names its sidecar after its media must keep doing so
|
|
||||||
# under a per-source filename override, or the pairing breaks exactly the
|
|
||||||
# way Discord's did. No metadata wanted means no platform postprocessor
|
|
||||||
# either — the global list was already dropped above.
|
|
||||||
if "postprocessors" in platform_section:
|
|
||||||
mirrored = sidecar_name_for(platform_section.get("filename") or "")
|
|
||||||
if not source_config.save_metadata or mirrored is None:
|
|
||||||
platform_section.pop("postprocessors")
|
|
||||||
else:
|
|
||||||
platform_section["postprocessors"] = [metadata_postprocessor(mirrored)]
|
|
||||||
|
|
||||||
platform_section["metadata"] = source_config.save_metadata
|
platform_section["metadata"] = source_config.save_metadata
|
||||||
|
|
||||||
return config
|
return config
|
||||||
@@ -810,9 +760,6 @@ class GalleryDLService:
|
|||||||
|
|
||||||
if cookies_path:
|
if cookies_path:
|
||||||
config["extractor"]["cookies"] = cookies_path
|
config["extractor"]["cookies"] = cookies_path
|
||||||
if auth_token and platform == "discord":
|
|
||||||
config["extractor"].setdefault("discord", {})
|
|
||||||
config["extractor"]["discord"]["token"] = auth_token
|
|
||||||
|
|
||||||
with tempfile.NamedTemporaryFile(
|
with tempfile.NamedTemporaryFile(
|
||||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||||
@@ -996,8 +943,6 @@ class GalleryDLService:
|
|||||||
config = self._build_config_for_source(platform, source_config, artist_slug)
|
config = self._build_config_for_source(platform, source_config, artist_slug)
|
||||||
if cookies_path:
|
if cookies_path:
|
||||||
config["extractor"]["cookies"] = cookies_path
|
config["extractor"]["cookies"] = cookies_path
|
||||||
if auth_token and platform == "discord":
|
|
||||||
config["extractor"].setdefault("discord", {})["token"] = auth_token
|
|
||||||
|
|
||||||
with tempfile.NamedTemporaryFile(
|
with tempfile.NamedTemporaryFile(
|
||||||
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
mode="w", suffix=".json", delete=False, dir=str(self._config_dir),
|
||||||
|
|||||||
@@ -49,10 +49,8 @@ from .audits import single_color
|
|||||||
from .link_extract import extract_external_links
|
from .link_extract import extract_external_links
|
||||||
from .thumbnailer import Thumbnailer
|
from .thumbnailer import Thumbnailer
|
||||||
from .wip_title import (
|
from .wip_title import (
|
||||||
WIP_TITLE_SOFT_SOURCE,
|
|
||||||
WIP_TITLE_SOURCE,
|
WIP_TITLE_SOURCE,
|
||||||
apply_wip_image_tags,
|
apply_wip_image_tags,
|
||||||
matches_soft_wip_title,
|
|
||||||
matches_wip_title,
|
matches_wip_title,
|
||||||
resolve_wip_tag_id,
|
resolve_wip_tag_id,
|
||||||
)
|
)
|
||||||
@@ -428,11 +426,19 @@ class Importer:
|
|||||||
return self._upsert_artist(name) if name else None
|
return self._upsert_artist(name) if name else None
|
||||||
|
|
||||||
def _post_for_sidecar(
|
def _post_for_sidecar(
|
||||||
self, source: Path, artist: Artist | None
|
self, source: Path, artist: Artist | None,
|
||||||
|
*, source_row: Source | None = None,
|
||||||
) -> Post | None:
|
) -> Post | None:
|
||||||
"""If a sidecar sits next to `source`, ensure its Source+Post
|
"""If a sidecar sits next to `source`, ensure its Source+Post
|
||||||
exist (idempotent) and return the Post — so attachments can link
|
exist (idempotent) and return the Post — so attachments can link
|
||||||
to the same Post the per-member _apply_sidecar will reuse."""
|
to the same Post the per-member _apply_sidecar will reuse.
|
||||||
|
|
||||||
|
`source_row` is the subscription being downloaded, when there is one,
|
||||||
|
and wins over the (artist, platform) lookup — as it does in
|
||||||
|
`upsert_post_record`. The lookup takes the artist's FIRST source on the
|
||||||
|
platform, which is right only while an artist has one: a Discord artist
|
||||||
|
has one per channel, and every non-image file from a later channel was
|
||||||
|
filed under the first as an undated second post (#4435)."""
|
||||||
sc = find_sidecar(source)
|
sc = find_sidecar(source)
|
||||||
if sc is None or artist is None:
|
if sc is None or artist is None:
|
||||||
return None
|
return None
|
||||||
@@ -444,10 +450,13 @@ class Importer:
|
|||||||
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
log.warning("sidecar parse failed for %s: %s", sc, exc)
|
||||||
return None
|
return None
|
||||||
sd = parse_sidecar(data)
|
sd = parse_sidecar(data)
|
||||||
platform = sd.platform or "unknown"
|
if source_row is not None:
|
||||||
src = self._lookup_source_for_sidecar(
|
src = source_row
|
||||||
artist_id=artist.id, platform=platform,
|
else:
|
||||||
)
|
platform = sd.platform or "unknown"
|
||||||
|
src = self._lookup_source_for_sidecar(
|
||||||
|
artist_id=artist.id, platform=platform,
|
||||||
|
)
|
||||||
epid = sd.external_post_id or sc.stem
|
epid = sd.external_post_id or sc.stem
|
||||||
return self._find_or_create_post(
|
return self._find_or_create_post(
|
||||||
source_id=src.id if src else None,
|
source_id=src.id if src else None,
|
||||||
@@ -545,7 +554,7 @@ class Importer:
|
|||||||
# nothing silently vanishes, matching extract_archive's
|
# nothing silently vanishes, matching extract_archive's
|
||||||
# fail-soft contract.
|
# fail-soft contract.
|
||||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||||
post = self._post_for_sidecar(source, artist_use)
|
post = self._post_for_sidecar(source, artist_use, source_row=source_row)
|
||||||
self._capture_attachment(
|
self._capture_attachment(
|
||||||
source, post=post, artist=artist_use, resolved=True,
|
source, post=post, artist=artist_use, resolved=True,
|
||||||
)
|
)
|
||||||
@@ -554,7 +563,7 @@ class Importer:
|
|||||||
return ImportResult(status="attached", error=reason)
|
return ImportResult(status="attached", error=reason)
|
||||||
|
|
||||||
artist_use = artist if artist is not None else self._resolve_artist(source)
|
artist_use = artist if artist is not None else self._resolve_artist(source)
|
||||||
post = self._post_for_sidecar(source, artist_use)
|
post = self._post_for_sidecar(source, artist_use, source_row=source_row)
|
||||||
member_ids: list[int] = []
|
member_ids: list[int] = []
|
||||||
# Every member image touched (new + superseded + deduped), so the
|
# Every member image touched (new + superseded + deduped), so the
|
||||||
# from_attachment_id stamp below covers files that already existed in the
|
# from_attachment_id stamp below covers files that already existed in the
|
||||||
@@ -1040,9 +1049,7 @@ class Importer:
|
|||||||
removal sticks. The existing catalogue is covered separately by the
|
removal sticks. The existing catalogue is covered separately by the
|
||||||
operator-triggered backfill sweep. Gated by the settings toggle, and
|
operator-triggered backfill sweep. Gated by the settings toggle, and
|
||||||
best-effort: any failure is logged, never allowed to fail the import."""
|
best-effort: any failure is logged, never allowed to fail the import."""
|
||||||
hard_on = self.settings.wip_title_tagging_enabled
|
if not self.settings.wip_title_tagging_enabled:
|
||||||
soft_on = self.settings.wip_soft_title_tagging_enabled
|
|
||||||
if not (hard_on or soft_on):
|
|
||||||
return
|
return
|
||||||
if record.primary_post_id is None:
|
if record.primary_post_id is None:
|
||||||
return
|
return
|
||||||
@@ -1050,21 +1057,14 @@ class Importer:
|
|||||||
title = self.session.execute(
|
title = self.session.execute(
|
||||||
select(Post.post_title).where(Post.id == record.primary_post_id)
|
select(Post.post_title).where(Post.id == record.primary_post_id)
|
||||||
).scalar_one_or_none()
|
).scalar_one_or_none()
|
||||||
# HARD tier ("WIP"/"work in progress") wins — higher precision, and it
|
if not matches_wip_title(title):
|
||||||
# trains the head; SOFT (sketch/doodle, #1474) is the provisional fallback
|
|
||||||
# that never trains (source wip_title_soft).
|
|
||||||
if hard_on and matches_wip_title(title):
|
|
||||||
source = WIP_TITLE_SOURCE
|
|
||||||
elif soft_on and matches_soft_wip_title(title):
|
|
||||||
source = WIP_TITLE_SOFT_SOURCE
|
|
||||||
else:
|
|
||||||
return
|
return
|
||||||
if self._wip_tag_id is _UNSET:
|
if self._wip_tag_id is _UNSET:
|
||||||
self._wip_tag_id = resolve_wip_tag_id(self.session)
|
self._wip_tag_id = resolve_wip_tag_id(self.session)
|
||||||
if self._wip_tag_id is None:
|
if self._wip_tag_id is None:
|
||||||
return
|
return
|
||||||
apply_wip_image_tags(
|
apply_wip_image_tags(
|
||||||
self.session, [record.id], self._wip_tag_id, source=source
|
self.session, [record.id], self._wip_tag_id, source=WIP_TITLE_SOURCE
|
||||||
)
|
)
|
||||||
except Exception as exc: # noqa: BLE001 — a tag must never fail an import
|
except Exception as exc: # noqa: BLE001 — a tag must never fail an import
|
||||||
log.warning(
|
log.warning(
|
||||||
@@ -1141,9 +1141,51 @@ class Importer:
|
|||||||
if post.artist_id is None:
|
if post.artist_id is None:
|
||||||
post.artist_id = artist.id
|
post.artist_id = artist.id
|
||||||
self._apply_post_fields(post, sd)
|
self._apply_post_fields(post, sd)
|
||||||
|
self._redate_post_images(post)
|
||||||
self.session.commit()
|
self.session.commit()
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def _redate_post_images(self, post: Post) -> None:
|
||||||
|
"""Carry a post's date onto the images already linked to it (#4431).
|
||||||
|
|
||||||
|
The native ingesters import a post's media BEFORE its record: the
|
||||||
|
per-media sidecar holds only the image identity (post-first, #856), and
|
||||||
|
the date arrives with `_post.json`. So `_attach_provenance` links each
|
||||||
|
image to a post that has no date yet, and the image keeps its download
|
||||||
|
time. This runs when the record lands, and applies the same two rules
|
||||||
|
`_attach_provenance` applies: `effective_date` is the PRIMARY post's
|
||||||
|
date, and `earliest_post_date` is the earliest date across every post
|
||||||
|
the image is linked to. Only rows that differ are written."""
|
||||||
|
if post.post_date is None:
|
||||||
|
return
|
||||||
|
self.session.flush()
|
||||||
|
self.session.execute(
|
||||||
|
update(ImageRecord)
|
||||||
|
.where(ImageRecord.primary_post_id == post.id)
|
||||||
|
.where(ImageRecord.effective_date.is_distinct_from(post.post_date))
|
||||||
|
.values(effective_date=post.post_date)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
)
|
||||||
|
linked = select(ImageProvenance.image_record_id).where(
|
||||||
|
ImageProvenance.post_id == post.id
|
||||||
|
)
|
||||||
|
earliest = (
|
||||||
|
select(func.min(Post.post_date))
|
||||||
|
.select_from(ImageProvenance)
|
||||||
|
.join(Post, Post.id == ImageProvenance.post_id)
|
||||||
|
.where(ImageProvenance.image_record_id == ImageRecord.id)
|
||||||
|
.where(Post.post_date.is_not(None))
|
||||||
|
.correlate(ImageRecord)
|
||||||
|
.scalar_subquery()
|
||||||
|
)
|
||||||
|
self.session.execute(
|
||||||
|
update(ImageRecord)
|
||||||
|
.where(ImageRecord.id.in_(linked))
|
||||||
|
.where(ImageRecord.earliest_post_date.is_distinct_from(earliest))
|
||||||
|
.values(earliest_post_date=earliest)
|
||||||
|
.execution_options(synchronize_session=False)
|
||||||
|
)
|
||||||
|
|
||||||
def attach_in_place(
|
def attach_in_place(
|
||||||
self,
|
self,
|
||||||
path: Path,
|
path: Path,
|
||||||
@@ -1187,7 +1229,10 @@ class Importer:
|
|||||||
path, artist=artist, source_row=source,
|
path, artist=artist, source_row=source,
|
||||||
)
|
)
|
||||||
if not is_supported(path):
|
if not is_supported(path):
|
||||||
post = self._post_for_sidecar(path, artist) if artist else None
|
post = (
|
||||||
|
self._post_for_sidecar(path, artist, source_row=source)
|
||||||
|
if artist else None
|
||||||
|
)
|
||||||
return self._capture_attachment(
|
return self._capture_attachment(
|
||||||
path, post=post, artist=artist, resolved=True,
|
path, post=post, artist=artist, resolved=True,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ from datetime import UTC, datetime, timedelta
|
|||||||
from sqlalchemy import delete, func, select, text
|
from sqlalchemy import delete, func, select, text
|
||||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||||
|
|
||||||
|
from ..models import ImageRecord
|
||||||
from .gallery_dl import (
|
from .gallery_dl import (
|
||||||
DownloadResult,
|
DownloadResult,
|
||||||
ErrorType,
|
ErrorType,
|
||||||
@@ -105,6 +106,38 @@ _LIVE_PROGRESS_INTERVAL = 5.0
|
|||||||
# recapture (the operator's schema-test flow) reaches the sample.
|
# recapture (the operator's schema-test flow) reaches the sample.
|
||||||
_CANARY_MIN_SAMPLE = 30
|
_CANARY_MIN_SAMPLE = 30
|
||||||
|
|
||||||
|
# The walk's time budget covers only the walk, but phase 3 runs in the SAME
|
||||||
|
# Celery task, under the same soft limit (tasks/download.py: 1350s). A walk that
|
||||||
|
# finds a lot of work for phase 3 must stop early and leave it to the next chunk,
|
||||||
|
# or phase 3 is killed mid-import: TamadaHeijun's recapture, 2026-09-24, walked
|
||||||
|
# for ~2 min and then spent 20 min importing 431 orphans and relinking ~3000
|
||||||
|
# on-disk files, and died at the soft limit.
|
||||||
|
#
|
||||||
|
# So the walk also stops when its elapsed time PLUS phase 3's estimated cost
|
||||||
|
# would pass CHUNK_TOTAL_SECONDS. An import (hash, pHash, sidecar, provenance,
|
||||||
|
# thumbnail enqueue) runs 2-5s depending on the files; the estimate takes the
|
||||||
|
# slow end. A relink is a sha256 over NFS plus a lookup.
|
||||||
|
#
|
||||||
|
# The estimate is a plan, not a guard. PHASE3_DEADLINE_SECONDS is the guard:
|
||||||
|
# phase 3 starts no import past it (download_service), and leaves the rest on
|
||||||
|
# disk, unmarked, for the next chunk. It sits under the soft limit by enough to
|
||||||
|
# finish the import in hand, upsert post records and finalize the event.
|
||||||
|
# test_download_source_task pins both under the soft limit.
|
||||||
|
CHUNK_TOTAL_SECONDS = 1200.0
|
||||||
|
PHASE3_IMPORT_SECONDS = 5.0
|
||||||
|
PHASE3_RELINK_SECONDS = 0.25
|
||||||
|
PHASE3_DEADLINE_SECONDS = 1200.0
|
||||||
|
|
||||||
|
|
||||||
|
def _unskipped(
|
||||||
|
fetched: list[tuple[str, str]],
|
||||||
|
written_keys: dict[str, tuple[str, str]],
|
||||||
|
skip_paths,
|
||||||
|
) -> list[tuple[str, str]]:
|
||||||
|
"""`fetched` without the ledger keys of files phase 3 left unimported."""
|
||||||
|
skip = {written_keys[p] for p in skip_paths if p in written_keys}
|
||||||
|
return [kv for kv in fetched if kv not in skip] if skip else list(fetched)
|
||||||
|
|
||||||
|
|
||||||
def _parse_published(raw: object) -> datetime | None:
|
def _parse_published(raw: object) -> datetime | None:
|
||||||
"""An ISO-8601 post date from either native client, as aware UTC.
|
"""An ISO-8601 post date from either native client, as aware UTC.
|
||||||
@@ -247,6 +280,14 @@ class Ingester:
|
|||||||
# operator-driven "re-read every body" pass; a horizon there would be a
|
# operator-driven "re-read every body" pass; a horizon there would be a
|
||||||
# third overlapping answer to a question that has two.
|
# third overlapping answer to a question that has two.
|
||||||
post_meta = getattr(self.client, "post_meta", None)
|
post_meta = getattr(self.client, "post_meta", None)
|
||||||
|
# #4413: optional client seam for a source that is several feeds walked
|
||||||
|
# one after another (a Discord server: every channel and thread). The
|
||||||
|
# tick early-out means "this feed has nothing new", and without the
|
||||||
|
# seam it ends the WHOLE walk — so the first quiet channel would hide
|
||||||
|
# every channel after it. With it, the early-out asks the client to
|
||||||
|
# move on and the walk continues. Absent → the early-out ends the walk,
|
||||||
|
# exactly as before (Patreon and SubscribeStar are one feed each).
|
||||||
|
skip_feed = getattr(self.client, "skip_feed", None)
|
||||||
horizon: datetime | None = None
|
horizon: datetime | None = None
|
||||||
if mode == "tick" and revisit_days > 0 and post_meta is not None:
|
if mode == "tick" and revisit_days > 0 and post_meta is not None:
|
||||||
horizon = datetime.now(UTC) - timedelta(days=revisit_days)
|
horizon = datetime.now(UTC) - timedelta(days=revisit_days)
|
||||||
@@ -261,6 +302,20 @@ class Ingester:
|
|||||||
# source_filehash and (b) link the on-disk image to its Post (#1288) —
|
# source_filehash and (b) link the on-disk image to its Post (#1288) —
|
||||||
# WITHOUT re-downloading or unlinking the file. Empty outside recapture.
|
# WITHOUT re-downloading or unlinking the file. Empty outside recapture.
|
||||||
relink: list[tuple[str, str, str]] = []
|
relink: list[tuple[str, str, str]] = []
|
||||||
|
# Media handed to phase 3 for import. Marked seen by phase 3 once the
|
||||||
|
# import has run (`mark_seen_after_import`), not here — see there.
|
||||||
|
fetched: list[tuple[str, str]] = []
|
||||||
|
# Per entry of `written`: the cursor of the page it came from, and its
|
||||||
|
# ledger key. Phase 3 can stop importing at its deadline; the files it
|
||||||
|
# leaves must stay unmarked, and the next chunk has to resume on the
|
||||||
|
# page holding the first of them, or no walk sees them again.
|
||||||
|
written_cursors: list[str | None] = []
|
||||||
|
written_keys: dict[str, tuple[str, str]] = {}
|
||||||
|
# Post-record keys written this walk. Marked with the media, after
|
||||||
|
# phase 3 has upserted the records — marking them at write time left a
|
||||||
|
# walk killed before phase 3 with posts the ledger calls recorded that
|
||||||
|
# the database never dated, and no later tick walks back to them (#4436).
|
||||||
|
recorded: list[tuple[str, str]] = []
|
||||||
downloaded = 0
|
downloaded = 0
|
||||||
errors = 0
|
errors = 0
|
||||||
quarantined = 0
|
quarantined = 0
|
||||||
@@ -292,6 +347,7 @@ class Ingester:
|
|||||||
reached_bottom = False
|
reached_bottom = False
|
||||||
budget_hit = False
|
budget_hit = False
|
||||||
early_out = False
|
early_out = False
|
||||||
|
feeds_caught_up = 0 # #4413: feeds a tick left early via skip_feed
|
||||||
stopped = False # plan #708 B4: operator hit Stop mid-walk
|
stopped = False # plan #708 B4: operator hit Stop mid-walk
|
||||||
cancel_armed = False # latched once we observe a live "running" state
|
cancel_armed = False # latched once we observe a live "running" state
|
||||||
|
|
||||||
@@ -313,6 +369,11 @@ class Ingester:
|
|||||||
written_paths=written,
|
written_paths=written,
|
||||||
post_record_paths=list(post_records),
|
post_record_paths=list(post_records),
|
||||||
relink_source_paths=list(relink),
|
relink_source_paths=list(relink),
|
||||||
|
written_cursors=list(written_cursors),
|
||||||
|
mark_seen_after_import=lambda skip_paths=(): self._mark_seen(
|
||||||
|
source_id,
|
||||||
|
_unskipped(fetched, written_keys, skip_paths) + recorded,
|
||||||
|
),
|
||||||
stdout="\n".join(log_lines),
|
stdout="\n".join(log_lines),
|
||||||
stderr="",
|
stderr="",
|
||||||
return_code=return_code,
|
return_code=return_code,
|
||||||
@@ -388,7 +449,17 @@ class Ingester:
|
|||||||
|
|
||||||
# Time-box check at the post boundary (coarse, like a gallery-dl
|
# Time-box check at the post boundary (coarse, like a gallery-dl
|
||||||
# chunk). Backfill/recovery resume from emitted_cursor next chunk.
|
# chunk). Backfill/recovery resume from emitted_cursor next chunk.
|
||||||
if time.monotonic() - start >= time_budget_seconds:
|
# The second half is phase 3's share of the task — see
|
||||||
|
# CHUNK_TOTAL_SECONDS. A mid-page stop resumes the same page.
|
||||||
|
elapsed = time.monotonic() - start
|
||||||
|
phase3 = (
|
||||||
|
len(written) * PHASE3_IMPORT_SECONDS
|
||||||
|
+ len(relink) * PHASE3_RELINK_SECONDS
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
elapsed >= time_budget_seconds
|
||||||
|
or elapsed + phase3 >= CHUNK_TOTAL_SECONDS
|
||||||
|
):
|
||||||
budget_hit = True
|
budget_hit = True
|
||||||
break
|
break
|
||||||
|
|
||||||
@@ -471,7 +542,7 @@ class Ingester:
|
|||||||
posts_with_body += 1
|
posts_with_body += 1
|
||||||
if rec.path is not None:
|
if rec.path is not None:
|
||||||
post_records.append(str(rec.path))
|
post_records.append(str(rec.path))
|
||||||
self._mark_seen(source_id, [(pkey, ppid)])
|
recorded.append((pkey, ppid))
|
||||||
# Per-post handling line in the run stdout (the existing
|
# Per-post handling line in the run stdout (the existing
|
||||||
# "Raw stdout" panel) — the downloader already read the
|
# "Raw stdout" panel) — the downloader already read the
|
||||||
# post; we only format its outcome here. post_type beside
|
# post; we only format its outcome here. post_type beside
|
||||||
@@ -512,6 +583,13 @@ class Ingester:
|
|||||||
recapture=recapture,
|
recapture=recapture,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# An on-disk file is only "done" if something imported it. One
|
||||||
|
# with no ImageRecord at its path was written by a run that died
|
||||||
|
# before phase 3 — it goes to import, not to the ledger.
|
||||||
|
imported_paths = self._recorded_paths([
|
||||||
|
str(o.path) for o in outcomes
|
||||||
|
if o.status == "skipped_disk" and o.path is not None
|
||||||
|
])
|
||||||
to_mark: list[tuple[str, str]] = []
|
to_mark: list[tuple[str, str]] = []
|
||||||
to_clear: list[str] = [] # recovered → drop any dead-letter row
|
to_clear: list[str] = [] # recovered → drop any dead-letter row
|
||||||
to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error)
|
to_fail: list[tuple[str, str, str]] = [] # (key, post_id, error)
|
||||||
@@ -523,11 +601,33 @@ class Ingester:
|
|||||||
downloaded += 1
|
downloaded += 1
|
||||||
if outcome.path is not None:
|
if outcome.path is not None:
|
||||||
written.append(str(outcome.path))
|
written.append(str(outcome.path))
|
||||||
to_mark.append((key, media_item.post_id))
|
written_cursors.append(emitted_cursor)
|
||||||
|
written_keys[str(outcome.path)] = (key, media_item.post_id)
|
||||||
|
fetched.append((key, media_item.post_id))
|
||||||
to_clear.append(key)
|
to_clear.append(key)
|
||||||
consecutive_seen = 0
|
consecutive_seen = 0
|
||||||
|
elif (
|
||||||
|
outcome.status == "skipped_disk"
|
||||||
|
and outcome.path is not None
|
||||||
|
and str(outcome.path) not in imported_paths
|
||||||
|
):
|
||||||
|
# On disk, never imported: a prior run wrote it and died
|
||||||
|
# before phase 3. Import it now. Safe to feed to
|
||||||
|
# attach_in_place because no record owns this path —
|
||||||
|
# the unlink below is about a file that IS the record.
|
||||||
|
written.append(str(outcome.path))
|
||||||
|
written_cursors.append(emitted_cursor)
|
||||||
|
written_keys[str(outcome.path)] = (key, media_item.post_id)
|
||||||
|
fetched.append((key, media_item.post_id))
|
||||||
|
to_clear.append(key)
|
||||||
|
skipped_count += 1
|
||||||
|
consecutive_seen += 1
|
||||||
|
log_lines.append(
|
||||||
|
f" post {media_item.post_id} — on disk but never "
|
||||||
|
f"imported: {outcome.path.name}"
|
||||||
|
)
|
||||||
elif outcome.status == "skipped_disk":
|
elif outcome.status == "skipped_disk":
|
||||||
# Already on disk (a prior run). Reconcile the ledger so a
|
# Already on disk and imported. Reconcile the ledger so a
|
||||||
# later tick skips it at tier-1 without a disk stat, but
|
# later tick skips it at tier-1 without a disk stat, but
|
||||||
# do NOT re-feed it to phase 3 — attach_in_place would see
|
# do NOT re-feed it to phase 3 — attach_in_place would see
|
||||||
# the duplicate sha256 and unlink the on-disk copy.
|
# the duplicate sha256 and unlink the on-disk copy.
|
||||||
@@ -618,9 +718,15 @@ class Ingester:
|
|||||||
})
|
})
|
||||||
|
|
||||||
if early_out:
|
if early_out:
|
||||||
break
|
if skip_feed is None:
|
||||||
|
break
|
||||||
|
skip_feed()
|
||||||
|
feeds_caught_up += 1
|
||||||
|
early_out = False
|
||||||
|
consecutive_seen = 0
|
||||||
else:
|
else:
|
||||||
reached_bottom = True
|
# A walk that left feeds early did not read to their ends.
|
||||||
|
reached_bottom = not feeds_caught_up
|
||||||
except self._error_base as exc:
|
except self._error_base as exc:
|
||||||
# The platform's client-error base — _failure_result (adapter)
|
# The platform's client-error base — _failure_result (adapter)
|
||||||
# maps it to a typed error.
|
# maps it to a typed error.
|
||||||
@@ -668,6 +774,7 @@ class Ingester:
|
|||||||
f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
|
f", {revisited} post(s) updated ({revisit_downloads} new file(s))"
|
||||||
if revisited else ""
|
if revisited else ""
|
||||||
)
|
)
|
||||||
|
+ (f", {feeds_caught_up} feed(s) caught up" if feeds_caught_up else "")
|
||||||
+ (", reached end" if reached_bottom else "")
|
+ (", reached end" if reached_bottom else "")
|
||||||
+ (", time-boxed" if budget_hit else "")
|
+ (", time-boxed" if budget_hit else "")
|
||||||
)
|
)
|
||||||
@@ -681,7 +788,13 @@ class Ingester:
|
|||||||
# next chunk resumes from the emitted cursor. No progress → TIMEOUT,
|
# next chunk resumes from the emitted cursor. No progress → TIMEOUT,
|
||||||
# which feeds download_service's backfill stall-guard. rc<0 mirrors
|
# which feeds download_service's backfill stall-guard. rc<0 mirrors
|
||||||
# subprocess TimeoutExpired so completion detection stays false.
|
# subprocess TimeoutExpired so completion detection stays false.
|
||||||
made_progress = downloaded > 0 or emitted_cursor != resume_cursor
|
# Work handed to phase 3 is progress too: a recapture chunk that
|
||||||
|
# stopped for its imports downloaded nothing, and may not have left
|
||||||
|
# its first page.
|
||||||
|
made_progress = (
|
||||||
|
downloaded > 0 or bool(written) or bool(relink)
|
||||||
|
or emitted_cursor != resume_cursor
|
||||||
|
)
|
||||||
if made_progress:
|
if made_progress:
|
||||||
return _result(
|
return _result(
|
||||||
success=False, return_code=-1,
|
success=False, return_code=-1,
|
||||||
@@ -887,6 +1000,16 @@ class Ingester:
|
|||||||
)
|
)
|
||||||
session.commit()
|
session.commit()
|
||||||
|
|
||||||
|
def _recorded_paths(self, paths: list[str]) -> set[str]:
|
||||||
|
"""Which of `paths` an ImageRecord already points at."""
|
||||||
|
if not paths:
|
||||||
|
return set()
|
||||||
|
with self.session_factory() as session:
|
||||||
|
rows = session.execute(
|
||||||
|
select(ImageRecord.path).where(ImageRecord.path.in_(paths))
|
||||||
|
).scalars().all()
|
||||||
|
return set(rows)
|
||||||
|
|
||||||
def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
|
def _mark_seen(self, source_id: int, items: list[tuple[str, str]]) -> None:
|
||||||
"""Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
|
"""Idempotent upsert of (filehash, post_id) seen-ledger rows for a page.
|
||||||
|
|
||||||
|
|||||||
@@ -97,8 +97,8 @@ def _sigmoid(z, np):
|
|||||||
|
|
||||||
def _conflict_scores(Xn, Wc, bc, np):
|
def _conflict_scores(Xn, Wc, bc, np):
|
||||||
"""The presentation conflict signal (#141): per row, the MAX content-head
|
"""The presentation conflict signal (#141): per row, the MAX content-head
|
||||||
probability and WHICH head produced it. Shared by the system-tag sweep's guard-2
|
probability and WHICH head produced it — the system-tag sweep's guard-2 asks
|
||||||
and the soft-wip audit — both ask "does this ALSO look like real content?"."""
|
"does this ALSO look like real content?"."""
|
||||||
cprobs = _sigmoid(Xn @ Wc.T + bc, np)
|
cprobs = _sigmoid(Xn @ Wc.T + bc, np)
|
||||||
return cprobs.max(axis=1), cprobs.argmax(axis=1)
|
return cprobs.max(axis=1), cprobs.argmax(axis=1)
|
||||||
|
|
||||||
@@ -106,10 +106,9 @@ def _conflict_scores(Xn, Wc, bc, np):
|
|||||||
def _insert_presentation_review(
|
def _insert_presentation_review(
|
||||||
session, *, image_record_id, tag_id, conflict_tag_id, conflict_score, mode,
|
session, *, image_record_id, tag_id, conflict_tag_id, conflict_score, mode,
|
||||||
):
|
):
|
||||||
"""Single-source the ring-loud PresentationReview row shape so the two writers
|
"""Single-source the ring-loud PresentationReview row shape, so every writer of
|
||||||
(system-tag sweep guard-2 + soft-wip audit) can't drift on columns or `mode` —
|
the (image_record_id, tag_id) composite PK agrees on columns and `mode` — a
|
||||||
they share the (image_record_id, tag_id) composite PK, so a divergent `mode`
|
divergent `mode` would be a silent first-writer-wins bug."""
|
||||||
would be a silent first-writer-wins bug."""
|
|
||||||
session.execute(
|
session.execute(
|
||||||
pg_insert(PresentationReview)
|
pg_insert(PresentationReview)
|
||||||
.values(
|
.values(
|
||||||
@@ -963,68 +962,6 @@ def system_tag_auto_apply_sweep(
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def soft_wip_conflict_audit(session: Session, dry_run: bool = False) -> dict:
|
|
||||||
"""Ring-loud audit for the SOFT WIP-title cohort (#1474). Images auto-tagged
|
|
||||||
`wip` from a low-precision sketch/doodle title (source='wip_title_soft') that ALSO
|
|
||||||
score >= the process conflict threshold on a content head are probably FINISHED
|
|
||||||
art mis-tagged as process — flag them (PresentationReview, mode='process') so the
|
|
||||||
review strip surfaces them ("also looks like <X>", Keep tag / Remove tag). Does
|
|
||||||
NOT remove the tag; the operator decides. No-op when there are no content heads.
|
|
||||||
numpy-only. Returns {n_scanned, n_flagged}."""
|
|
||||||
import numpy as np
|
|
||||||
|
|
||||||
from ..wip_title import WIP_TITLE_SOFT_SOURCE, resolve_wip_tag_id
|
|
||||||
|
|
||||||
settings = _settings(session)
|
|
||||||
ver = settings.embedder_model_version
|
|
||||||
conflict_thr = float(settings.process_conflict_threshold)
|
|
||||||
conf = _conflict_heads(session, ver)
|
|
||||||
wip_id = resolve_wip_tag_id(session)
|
|
||||||
if not conf or wip_id is None:
|
|
||||||
return {"n_scanned": 0, "n_flagged": 0}
|
|
||||||
Wc = np.vstack([np.asarray(r.weights, dtype=np.float32) for r in conf])
|
|
||||||
bc = np.asarray([r.bias for r in conf], dtype=np.float32)
|
|
||||||
conf_tag_ids = [r.tag_id for r in conf]
|
|
||||||
|
|
||||||
soft_ids = [iid for (iid,) in session.execute(
|
|
||||||
select(image_tag.c.image_record_id)
|
|
||||||
.where(image_tag.c.tag_id == wip_id)
|
|
||||||
.where(image_tag.c.source == WIP_TITLE_SOFT_SOURCE)
|
|
||||||
)]
|
|
||||||
# Skip images already flagged for this tag (idempotent re-runs).
|
|
||||||
flagged = {iid for (iid,) in session.execute(
|
|
||||||
select(PresentationReview.image_record_id)
|
|
||||||
.where(PresentationReview.tag_id == wip_id)
|
|
||||||
)}
|
|
||||||
soft_ids = [i for i in soft_ids if i not in flagged]
|
|
||||||
|
|
||||||
n_flagged = 0
|
|
||||||
scanned = 0
|
|
||||||
for start in range(0, len(soft_ids), _AUTO_APPLY_CHUNK):
|
|
||||||
chunk = soft_ids[start:start + _AUTO_APPLY_CHUNK]
|
|
||||||
emb = _load_embeddings(session, chunk)
|
|
||||||
cids = [i for i in chunk if i in emb]
|
|
||||||
if not cids:
|
|
||||||
continue
|
|
||||||
scanned += len(cids)
|
|
||||||
Xn = _l2norm(np.vstack([emb[i] for i in cids]).astype(np.float32), np)
|
|
||||||
max_c, arg_c = _conflict_scores(Xn, Wc, bc, np)
|
|
||||||
for k in range(len(cids)):
|
|
||||||
if float(max_c[k]) >= conflict_thr:
|
|
||||||
n_flagged += 1
|
|
||||||
if not dry_run:
|
|
||||||
_insert_presentation_review(
|
|
||||||
session,
|
|
||||||
image_record_id=cids[k], tag_id=wip_id,
|
|
||||||
conflict_tag_id=conf_tag_ids[int(arg_c[k])],
|
|
||||||
conflict_score=float(max_c[k]),
|
|
||||||
mode="process",
|
|
||||||
)
|
|
||||||
if not dry_run:
|
|
||||||
session.commit()
|
|
||||||
return {"n_scanned": scanned, "n_flagged": n_flagged}
|
|
||||||
|
|
||||||
|
|
||||||
def retract_auto_applied_heads(session: Session) -> int:
|
def retract_auto_applied_heads(session: Session) -> int:
|
||||||
"""Soft auto-apply (milestone 139): re-score every standing source='head_auto'
|
"""Soft auto-apply (milestone 139): re-score every standing source='head_auto'
|
||||||
tag against its CURRENT head and REMOVE the ones now BELOW the head's
|
tag against its CURRENT head and REMOVE the ones now BELOW the head's
|
||||||
|
|||||||
@@ -32,11 +32,8 @@ from ...models.tag import image_tag
|
|||||||
# `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
|
# `process_auto` (#1464): wip/editor screenshot applied by the process sweep are
|
||||||
# ALSO provisional — the head must learn only from title (`wip_title`) + manual
|
# ALSO provisional — the head must learn only from title (`wip_title`) + manual
|
||||||
# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
|
# labels, never its own auto-applied output, or it would runaway (operator 2026-07-12).
|
||||||
# `wip_title_soft` (#1474): the soft title tier (sketch/doodle) is LOW-precision, so
|
|
||||||
# it's provisional too — a finished piece titled "sketch" must not train the wip head.
|
|
||||||
_AUTO_SOURCES = (
|
_AUTO_SOURCES = (
|
||||||
"head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
|
"head_auto", "ccip_auto", "ml_auto", "presentation_auto", "process_auto",
|
||||||
"wip_title_soft",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -23,8 +23,10 @@ log = logging.getLogger(__name__)
|
|||||||
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
# Platforms walked one-at-a-time. gallery-dl platforms are intentionally NOT
|
||||||
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
# here: each runs as a self-pacing subprocess and they're lower-volume. The
|
||||||
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
# native-ingester platforms are serialized (one paced scrape/API walk at a time).
|
||||||
# Add a platform here to cap it to a single concurrent walk.
|
# Add a platform here to cap it to a single concurrent walk. Discord most of
|
||||||
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar"})
|
# all: every source walks on the operator's ONE user token, and parallel walks
|
||||||
|
# on a user account are both how its rate limit trips and what gets it flagged.
|
||||||
|
SERIALIZED_PLATFORMS = frozenset({"patreon", "subscribestar", "discord"})
|
||||||
|
|
||||||
_LOCK_PREFIX = "fc:download_lock:"
|
_LOCK_PREFIX = "fc:download_lock:"
|
||||||
|
|
||||||
@@ -58,3 +60,22 @@ def platform_lock(platform: str, *, ttl_seconds: int):
|
|||||||
except redis.RedisError as exc: # pragma: no cover - broker outage
|
except redis.RedisError as exc: # pragma: no cover - broker outage
|
||||||
log.warning("platform_lock unavailable for %s: %s", platform, exc)
|
log.warning("platform_lock unavailable for %s: %s", platform, exc)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def release_all_platform_locks() -> int:
|
||||||
|
"""Drop every serialized platform's lock. Returns how many were held.
|
||||||
|
|
||||||
|
Only for the download lane's boot (#4433). A worker that restarts mid-walk
|
||||||
|
is SIGKILLed past its stop grace, so its `finally` never releases the lock,
|
||||||
|
and the TTL keeps every other source on that platform bouncing for up to
|
||||||
|
27 minutes after the new worker is ready. At boot no walk of ours can be
|
||||||
|
running, so a held lock names a dead one. Assumes one download consumer —
|
||||||
|
the only shape FC deploys; a second replica booting would free a live
|
||||||
|
walk's lock (not corrupt it: the walk runs on, a second walk may overlap it).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
client = _redis()
|
||||||
|
return int(client.delete(*(f"{_LOCK_PREFIX}{p}" for p in SERIALIZED_PLATFORMS)))
|
||||||
|
except redis.RedisError as exc: # pragma: no cover - broker outage
|
||||||
|
log.warning("could not release platform locks at boot: %s", exc)
|
||||||
|
return 0
|
||||||
|
|||||||
@@ -479,6 +479,10 @@ class PostFeedService:
|
|||||||
# the UI uses this to explain why a post it linked to is not in the
|
# the UI uses this to explain why a post it linked to is not in the
|
||||||
# stream.
|
# stream.
|
||||||
"absorbed_by_post_id": post.absorbed_by_post_id,
|
"absorbed_by_post_id": post.absorbed_by_post_id,
|
||||||
|
# #4481. The Discord channel a message was posted in: both the native
|
||||||
|
# post record and gallery-dl's sidecars carry it as `channel`. None
|
||||||
|
# on every other platform, and on a Discord post that never said.
|
||||||
|
"channel": _discord_channel(post, source),
|
||||||
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
"artist": {"id": artist.id, "name": artist.name, "slug": artist.slug},
|
||||||
"source": (
|
"source": (
|
||||||
{"id": source.id, "platform": source.platform}
|
{"id": source.id, "platform": source.platform}
|
||||||
@@ -488,3 +492,13 @@ class PostFeedService:
|
|||||||
"thumbnails_more": thumbs_entry["more"],
|
"thumbnails_more": thumbs_entry["more"],
|
||||||
"attachments": atts_map.get(post.id, []),
|
"attachments": atts_map.get(post.id, []),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _discord_channel(post: Post, source: Source | None) -> str | None:
|
||||||
|
if source is None or source.platform != "discord":
|
||||||
|
return None
|
||||||
|
raw = post.raw_metadata if isinstance(post.raw_metadata, dict) else {}
|
||||||
|
channel = raw.get("channel")
|
||||||
|
if not isinstance(channel, str):
|
||||||
|
return None
|
||||||
|
return channel.strip() or None
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
"""Date the posts whose record reached the disk but never the database (#4436).
|
||||||
|
|
||||||
|
A native walk writes each post's record (`_post.json`, or Discord's
|
||||||
|
`<day>_<message id>_post.json`) as it goes, and phase 3 upserts those records
|
||||||
|
after the walk — which is when the post, and through it its images, get their
|
||||||
|
date. Until #4436 the walk marked a record's post key seen at write time, so a
|
||||||
|
walk killed before phase 3 (a restart, a stall) left the post undated with the
|
||||||
|
ledger saying it was done, and no later tick walked back that far to fix it.
|
||||||
|
|
||||||
|
The record files are still on disk. This finds each undated native post's
|
||||||
|
record under its artist's folder and upserts it with the post's OWN source —
|
||||||
|
never the (artist, platform) lookup, which picks the artist's first source and
|
||||||
|
would misfile a Discord channel's post (#4435). Idempotent: a post that is
|
||||||
|
already dated is never looked at, and upserting a record twice changes nothing.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from ..models import Artist, Post, Source
|
||||||
|
from ..utils.sidecar import parse_sidecar
|
||||||
|
from .download_backends import NATIVE_INGESTER_PLATFORMS
|
||||||
|
|
||||||
|
log = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_record(path: Path) -> bool:
|
||||||
|
return path.name == "_post.json" or path.name.endswith("_post.json")
|
||||||
|
|
||||||
|
|
||||||
|
def _record_id(path: Path) -> str | None:
|
||||||
|
try:
|
||||||
|
data = json.loads(path.read_text("utf-8"))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
return None
|
||||||
|
if not isinstance(data, dict):
|
||||||
|
return None
|
||||||
|
return parse_sidecar(data).external_post_id
|
||||||
|
|
||||||
|
|
||||||
|
def date_posts_from_records(session: Session, importer, images_root: Path) -> dict:
|
||||||
|
"""Upsert the on-disk record of every undated native post. Returns counts."""
|
||||||
|
rows = session.execute(
|
||||||
|
select(Post.external_post_id, Post.source_id, Post.artist_id, Source.platform)
|
||||||
|
.join(Source, Source.id == Post.source_id)
|
||||||
|
.where(
|
||||||
|
Post.post_date.is_(None),
|
||||||
|
Post.synthesized_by.is_(None),
|
||||||
|
Source.platform.in_(NATIVE_INGESTER_PLATFORMS),
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
wanted: dict[tuple[int, str], dict[str, int]] = defaultdict(dict)
|
||||||
|
for epid, source_id, artist_id, platform in rows:
|
||||||
|
wanted[(artist_id, platform)][epid] = source_id
|
||||||
|
|
||||||
|
summary = {"undated": len(rows), "dated": 0, "no_record": 0}
|
||||||
|
for (artist_id, platform), by_epid in wanted.items():
|
||||||
|
artist = session.get(Artist, artist_id)
|
||||||
|
root = Path(images_root) / artist.slug / platform if artist else None
|
||||||
|
if root is None or not root.is_dir():
|
||||||
|
summary["no_record"] += len(by_epid)
|
||||||
|
continue
|
||||||
|
found = 0
|
||||||
|
for record in root.rglob("*post.json"):
|
||||||
|
if not _is_record(record):
|
||||||
|
continue
|
||||||
|
epid = _record_id(record)
|
||||||
|
source_id = by_epid.pop(epid, None) if epid else None
|
||||||
|
if source_id is None:
|
||||||
|
continue
|
||||||
|
source = session.get(Source, source_id)
|
||||||
|
if importer.upsert_post_record(record, artist=artist, source=source):
|
||||||
|
found += 1
|
||||||
|
if not by_epid:
|
||||||
|
break
|
||||||
|
summary["dated"] += found
|
||||||
|
summary["no_record"] += len(by_epid)
|
||||||
|
if summary["undated"]:
|
||||||
|
log.info("date_posts_from_records: %s", summary)
|
||||||
|
return summary
|
||||||
@@ -115,6 +115,32 @@ async def active_platform_cooldowns(session: AsyncSession) -> dict[str, datetime
|
|||||||
return active
|
return active
|
||||||
|
|
||||||
|
|
||||||
|
def backfill_ready(source: Source) -> bool:
|
||||||
|
"""A deep walk the operator started, with budget left and no failure
|
||||||
|
backing it off — due NOW rather than at its next scheduled check.
|
||||||
|
|
||||||
|
A backfill runs one time-boxed chunk per download (plan #693), and nothing
|
||||||
|
queued the next chunk: each waited for the source's regular interval. At
|
||||||
|
the 8-hour default a freshly armed backfill sat untouched until the next
|
||||||
|
check (the operator armed one on 2026-09-25 and saw nothing happen) and a
|
||||||
|
five-chunk walk took most of two days. The tick's in-flight guard keeps
|
||||||
|
one chunk at a time per source and the platform lock one walk per
|
||||||
|
platform, so "due every tick" means "next chunk as soon as the last one
|
||||||
|
ends".
|
||||||
|
|
||||||
|
The failure gate is what keeps a broken source from retrying every
|
||||||
|
minute: any failed chunk raises `consecutive_failures`, which drops the
|
||||||
|
source back onto its backed-off interval. A chunk that fails to progress
|
||||||
|
twice marks the walk stalled (download_service), which ends it here too.
|
||||||
|
"""
|
||||||
|
co = source.config_overrides or {}
|
||||||
|
return (
|
||||||
|
co.get("_backfill_state") == "running"
|
||||||
|
and (source.backfill_runs_remaining or 0) > 0
|
||||||
|
and not (source.consecutive_failures or 0)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def select_due_sources(session: AsyncSession) -> list[Source]:
|
async def select_due_sources(session: AsyncSession) -> list[Source]:
|
||||||
"""Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
|
"""Sources where (enabled, artist.auto_check) and now >= last_checked_at + effective_interval.
|
||||||
|
|
||||||
@@ -123,6 +149,9 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
|||||||
cooldown is the preventive half of the burst-prevention pair (per-source
|
cooldown is the preventive half of the burst-prevention pair (per-source
|
||||||
consecutive_failures backoff handles the offending source itself).
|
consecutive_failures backoff handles the offending source itself).
|
||||||
|
|
||||||
|
A running backfill (`backfill_ready`) is due on every tick, and whether
|
||||||
|
or not its artist is on auto-check — the operator started it by hand.
|
||||||
|
|
||||||
Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked
|
Ordering: last_checked_at ASC NULLS FIRST, then id. Never-checked
|
||||||
sources go first, then the longest-since-checked, so the most overdue
|
sources go first, then the longest-since-checked, so the most overdue
|
||||||
sources hit Celery's FIFO download queue first. Anti-starvation: if
|
sources hit Celery's FIFO download queue first. Anti-starvation: if
|
||||||
@@ -135,7 +164,6 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
|||||||
.options(selectinload(Source.artist))
|
.options(selectinload(Source.artist))
|
||||||
.join(Artist, Source.artist_id == Artist.id)
|
.join(Artist, Source.artist_id == Artist.id)
|
||||||
.where(Source.enabled.is_(True))
|
.where(Source.enabled.is_(True))
|
||||||
.where(Artist.auto_check.is_(True))
|
|
||||||
.order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
|
.order_by(Source.last_checked_at.asc().nulls_first(), Source.id)
|
||||||
)).scalars().all()
|
)).scalars().all()
|
||||||
|
|
||||||
@@ -147,6 +175,11 @@ async def select_due_sources(session: AsyncSession) -> list[Source]:
|
|||||||
for s in rows:
|
for s in rows:
|
||||||
if s.platform in cooldowns:
|
if s.platform in cooldowns:
|
||||||
continue
|
continue
|
||||||
|
if backfill_ready(s):
|
||||||
|
due.append(s)
|
||||||
|
continue
|
||||||
|
if not s.artist.auto_check:
|
||||||
|
continue
|
||||||
interval = compute_effective_interval(s, s.artist, settings)
|
interval = compute_effective_interval(s, s.artist, settings)
|
||||||
if s.last_checked_at is None:
|
if s.last_checked_at is None:
|
||||||
due.append(s)
|
due.append(s)
|
||||||
@@ -163,6 +196,8 @@ def compute_next_check_at(
|
|||||||
"""Return the projected datetime of the next check, or None if never checked."""
|
"""Return the projected datetime of the next check, or None if never checked."""
|
||||||
if source.last_checked_at is None:
|
if source.last_checked_at is None:
|
||||||
return None
|
return None
|
||||||
|
if backfill_ready(source):
|
||||||
|
return datetime.now(UTC)
|
||||||
interval = compute_effective_interval(source, artist, settings)
|
interval = compute_effective_interval(source, artist, settings)
|
||||||
return source.last_checked_at + timedelta(seconds=interval)
|
return source.last_checked_at + timedelta(seconds=interval)
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ from ..models import (
|
|||||||
Source,
|
Source,
|
||||||
)
|
)
|
||||||
from .db_helpers import failing_sources_clause
|
from .db_helpers import failing_sources_clause
|
||||||
|
from .download_backends import uses_native_ingester
|
||||||
from .gallery_dl import ErrorType
|
from .gallery_dl import ErrorType
|
||||||
from .membership_reconcile import KEPT_KEY, STOPPED_KEY
|
from .membership_reconcile import KEPT_KEY, STOPPED_KEY
|
||||||
from .membership_roster import gated_reasons_for_sources
|
from .membership_roster import gated_reasons_for_sources
|
||||||
@@ -67,6 +68,9 @@ class SourceRecord:
|
|||||||
artist_slug: str
|
artist_slug: str
|
||||||
platform: str
|
platform: str
|
||||||
url: str
|
url: str
|
||||||
|
# alembic 0115: the name the platform gives it, where the URL is opaque
|
||||||
|
# (a Discord link is two ids). None until a walk has read it.
|
||||||
|
display_name: str | None
|
||||||
enabled: bool
|
enabled: bool
|
||||||
config_overrides: dict | None
|
config_overrides: dict | None
|
||||||
last_checked_at: str | None
|
last_checked_at: str | None
|
||||||
@@ -109,6 +113,7 @@ class SourceRecord:
|
|||||||
"artist_slug": self.artist_slug,
|
"artist_slug": self.artist_slug,
|
||||||
"platform": self.platform,
|
"platform": self.platform,
|
||||||
"url": self.url,
|
"url": self.url,
|
||||||
|
"display_name": self.display_name,
|
||||||
"enabled": self.enabled,
|
"enabled": self.enabled,
|
||||||
"config_overrides": self.config_overrides,
|
"config_overrides": self.config_overrides,
|
||||||
"last_checked_at": self.last_checked_at,
|
"last_checked_at": self.last_checked_at,
|
||||||
@@ -125,6 +130,11 @@ class SourceRecord:
|
|||||||
"backfill_posts": self.backfill_posts,
|
"backfill_posts": self.backfill_posts,
|
||||||
"tier_gated_count": self.tier_gated_count,
|
"tier_gated_count": self.tier_gated_count,
|
||||||
"gated_reason": self.gated_reason,
|
"gated_reason": self.gated_reason,
|
||||||
|
# Recover / recapture exist only on the native ingester. Sent so the
|
||||||
|
# UI asks the backend's own predicate instead of keeping a copy of
|
||||||
|
# the platform list — the copy said "patreon, subscribestar" for a
|
||||||
|
# day after Discord went native (milestone 428).
|
||||||
|
"native_ingester": uses_native_ingester(self.platform),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -295,6 +305,7 @@ class SourceService:
|
|||||||
artist_slug=artist.slug,
|
artist_slug=artist.slug,
|
||||||
platform=source.platform,
|
platform=source.platform,
|
||||||
url=source.url,
|
url=source.url,
|
||||||
|
display_name=source.display_name,
|
||||||
enabled=source.enabled,
|
enabled=source.enabled,
|
||||||
config_overrides=source.config_overrides,
|
config_overrides=source.config_overrides,
|
||||||
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
last_checked_at=source.last_checked_at.isoformat() if source.last_checked_at else None,
|
||||||
@@ -551,8 +562,8 @@ class SourceService:
|
|||||||
whole source); the two flags are mutually exclusive, so arming recapture
|
whole source); the two flags are mutually exclusive, so arming recapture
|
||||||
clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
|
clears bypass_seen. Clears prior cursor/chunk/stall state so it walks
|
||||||
fresh from the top. The flag is cleared on completion (download_service)
|
fresh from the top. The flag is cleared on completion (download_service)
|
||||||
and on stop. Recapture is Patreon-only (the native ingester's post-record
|
and on stop. Recapture needs the native ingester's post-record capture,
|
||||||
capture); inert elsewhere. The UI gates the action to Patreon sources."""
|
so the UI offers it on native sources only (`native_ingester`)."""
|
||||||
source = (await self.session.execute(
|
source = (await self.session.execute(
|
||||||
select(Source).where(Source.id == source_id)
|
select(Source).where(Source.id == source_id)
|
||||||
)).scalar_one_or_none()
|
)).scalar_one_or_none()
|
||||||
|
|||||||
@@ -27,13 +27,10 @@ from .image_tag_apply import insert_image_tags
|
|||||||
|
|
||||||
# image_tag.source stamped on title-heuristic WIP tags — distinct from the other
|
# image_tag.source stamped on title-heuristic WIP tags — distinct from the other
|
||||||
# apply sources so provenance stays legible and a future undo can target only these.
|
# apply sources so provenance stays legible and a future undo can target only these.
|
||||||
# HARD tier ("WIP"/"work in progress") is high-precision → trains the wip head.
|
# Only the artist's own "WIP"/"work in progress" label counts — high-precision, so it
|
||||||
|
# trains the wip head. A sketch/doodle/scribble tier (#1474) was retired in milestone
|
||||||
|
# 430: a "sketch" is usually finished art, and its 6k tags flooded the review strip.
|
||||||
WIP_TITLE_SOURCE = "wip_title"
|
WIP_TITLE_SOURCE = "wip_title"
|
||||||
# SOFT tier (sketch/doodle/scribble, #1474) is LOWER-precision — a finished "sketch"
|
|
||||||
# is often not WIP. This source is PROVISIONAL (in training_data._AUTO_SOURCES) so it
|
|
||||||
# NEVER trains the wip head; a soft-tagged image that also looks like real content is
|
|
||||||
# surfaced by the ring-loud audit for review.
|
|
||||||
WIP_TITLE_SOFT_SOURCE = "wip_title_soft"
|
|
||||||
|
|
||||||
# A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
|
# A standalone "WIP" / "W.I.P" token, or the phrase "work in progress"
|
||||||
# (space/underscore/hyphen separated). The letter-boundary lookarounds are what
|
# (space/underscore/hyphen separated). The letter-boundary lookarounds are what
|
||||||
@@ -45,20 +42,10 @@ _WIP_RE = re.compile(
|
|||||||
re.IGNORECASE,
|
re.IGNORECASE,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Soft tier: sketch / doodle / scribble (+ plurals), letter-boundary anchored so
|
# Coarse SQL prefilter for the backfill sweep — narrows the post scan to rows that
|
||||||
# "sketchbook" / "kadoodle" don't trip it. Deliberately conservative — recall is
|
|
||||||
# secondary because the soft source doesn't train the head and the ring-loud audit
|
|
||||||
# catches false positives.
|
|
||||||
_SOFT_WIP_RE = re.compile(
|
|
||||||
r"(?<![A-Za-z])(?:sketch|sketches|doodle|doodles|scribble|scribbles)(?![A-Za-z])",
|
|
||||||
re.IGNORECASE,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Coarse SQL prefilters for the backfill sweep — narrow the post scan to rows that
|
|
||||||
# COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
|
# COULD match before the precise regex confirms. Case-insensitive ILIKE patterns.
|
||||||
# Each MUST stay a SUPERSET of its regex or the sweep would silently miss posts.
|
# It MUST stay a SUPERSET of the regex or the sweep would silently miss posts.
|
||||||
WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
|
WIP_TITLE_SQL_PREFILTER = ("%wip%", "%work%progress%")
|
||||||
SOFT_WIP_TITLE_SQL_PREFILTER = ("%sketch%", "%doodle%", "%scribble%")
|
|
||||||
|
|
||||||
# Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
|
# Chunk bulk inserts so a large sweep can't blow past psycopg's 65535-parameter
|
||||||
# ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
|
# ceiling (3 params/row → ~21k rows max; 5k stays comfortably under).
|
||||||
@@ -66,19 +53,12 @@ _INSERT_CHUNK = 5000
|
|||||||
|
|
||||||
|
|
||||||
def matches_wip_title(title: str | None) -> bool:
|
def matches_wip_title(title: str | None) -> bool:
|
||||||
"""True when a post title explicitly marks it work-in-progress (HARD tier)."""
|
"""True when a post title explicitly marks it work-in-progress."""
|
||||||
if not title:
|
if not title:
|
||||||
return False
|
return False
|
||||||
return _WIP_RE.search(title) is not None
|
return _WIP_RE.search(title) is not None
|
||||||
|
|
||||||
|
|
||||||
def matches_soft_wip_title(title: str | None) -> bool:
|
|
||||||
"""True when a title carries a SOFT WIP cue (sketch/doodle/scribble, #1474)."""
|
|
||||||
if not title:
|
|
||||||
return False
|
|
||||||
return _SOFT_WIP_RE.search(title) is not None
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_wip_tag_id(session: Session) -> int | None:
|
def resolve_wip_tag_id(session: Session) -> int | None:
|
||||||
"""The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
|
"""The seeded ``wip`` system tag's id (migration 0075), or None if absent."""
|
||||||
return session.execute(
|
return session.execute(
|
||||||
|
|||||||
@@ -211,11 +211,10 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
|
|||||||
state.reserved += len(reserved.get(hostname, []))
|
state.reserved += len(reserved.get(hostname, []))
|
||||||
state.consuming.update(q["name"] for q in queues)
|
state.consuming.update(q["name"] for q in queues)
|
||||||
|
|
||||||
# `pool.max-concurrency` is the number pool_grow/pool_shrink move and
|
# Absent on a worker whose stats did not answer, which leaves
|
||||||
# the number the UI shows. Absent on a worker whose stats did not
|
# pool=None — unknown, not zero.
|
||||||
# answer, which leaves pool=None — unknown, not zero.
|
pool = pool_size((stats.get(hostname) or {}).get("pool") or {})
|
||||||
pool = (stats.get(hostname) or {}).get("pool", {}).get("max-concurrency")
|
if pool is not None:
|
||||||
if isinstance(pool, int):
|
|
||||||
state.pools[hostname] = pool
|
state.pools[hostname] = pool
|
||||||
|
|
||||||
for state in out.values():
|
for state in out.values():
|
||||||
@@ -223,6 +222,27 @@ def inspect_lanes_sync() -> dict[str, LaneLiveState]:
|
|||||||
return out
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def pool_size(pool_stats: dict) -> int | None:
|
||||||
|
"""How many processes a prefork pool is running NOW, from `inspect stats`.
|
||||||
|
|
||||||
|
The length of `processes`, not `max-concurrency`. Celery reports
|
||||||
|
`max-concurrency` as the pool's `limit`, set once at boot; `pool_grow` and
|
||||||
|
`pool_shrink` hand straight to billiard and never touch it (celery 5.6
|
||||||
|
`concurrency/prefork.py`: `self.grow = P.grow`). So it read 1 forever on a
|
||||||
|
lane booted at 1, and the sizing sweep, computing `target - 1` on every
|
||||||
|
tick, grew a scheduler capped at 2 to six processes while the System tab
|
||||||
|
showed "6 / 1" (2026-09-24) — and could never shrink one, since 1 - 1 is 0.
|
||||||
|
|
||||||
|
`max-concurrency` is the fallback only for a pool that lists no processes
|
||||||
|
(a non-prefork pool), where it is the best number there is.
|
||||||
|
"""
|
||||||
|
procs = pool_stats.get("processes")
|
||||||
|
if isinstance(procs, list):
|
||||||
|
return len(procs)
|
||||||
|
limit = pool_stats.get("max-concurrency")
|
||||||
|
return limit if isinstance(limit, int) else None
|
||||||
|
|
||||||
|
|
||||||
def effective_slots(target: int) -> int:
|
def effective_slots(target: int) -> int:
|
||||||
"""What a pool can actually be set to. Never below one process.
|
"""What a pool can actually be set to. Never below one process.
|
||||||
|
|
||||||
|
|||||||
@@ -137,7 +137,13 @@ IMPORT_BATCH_KEEP_DAYS = 30
|
|||||||
# (the import queue itself stays at the 5-min default for single
|
# (the import queue itself stays at the 5-min default for single
|
||||||
# files); time_limit=2100.
|
# files); time_limit=2100.
|
||||||
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
"ml": 25,
|
# ml: the scheduled auto-apply sweeps and refresh_character_prototypes run
|
||||||
|
# to a 35-min hard limit (2100s); 25 swept them mid-run (#4432). The two
|
||||||
|
# 65-min jobs have their own entries below.
|
||||||
|
"ml": 40,
|
||||||
|
# import: import_media_file's hard limit is 6 min (360s), one past the
|
||||||
|
# 5-min default this queue fell to (#4432).
|
||||||
|
"import": 10,
|
||||||
# download_source legitimately walks 5-25 min (Patreon/gallery-dl
|
# download_source legitimately walks 5-25 min (Patreon/gallery-dl
|
||||||
# deep creators); its hard time_limit is DOWNLOAD_HARD_TIME_LIMIT
|
# deep creators); its hard time_limit is DOWNLOAD_HARD_TIME_LIMIT
|
||||||
# (1500s = 25m). The 5-min default flagged healthy in-flight walks as
|
# (1500s = 25m). The 5-min default flagged healthy in-flight walks as
|
||||||
@@ -154,6 +160,12 @@ QUEUE_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
|||||||
# overrides below cover the outliers (backups, library audit).
|
# overrides below cover the outliers (backups, library audit).
|
||||||
"maintenance": 75,
|
"maintenance": 75,
|
||||||
"scan": 75,
|
"scan": 75,
|
||||||
|
# The long lane (#4432). Until TaskRun.queue asked the router, nothing was
|
||||||
|
# recorded here: these runs read as `maintenance` (75) or, for
|
||||||
|
# translation, `default` (5 — which failed healthy 35-min runs). The
|
||||||
|
# longest task without its own entry below is the admin family at a
|
||||||
|
# 40-min hard limit; 45 = 40 + 5.
|
||||||
|
"maintenance_long": 45,
|
||||||
}
|
}
|
||||||
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
||||||
"backend.app.tasks.import_file.import_archive_file": 40,
|
"backend.app.tasks.import_file.import_archive_file": 40,
|
||||||
@@ -179,6 +191,10 @@ TASK_STUCK_THRESHOLD_MINUTES: dict[str, int] = {
|
|||||||
# external-fetch entry above — without an override a healthy in-flight walk
|
# external-fetch entry above — without an override a healthy in-flight walk
|
||||||
# is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
|
# is swept 'RecoverySweep' at the bare 5-min default. 30 = 25 + 5.
|
||||||
"backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
|
"backend.app.tasks.admin.reclaim_orphaned_attachments_task": 30,
|
||||||
|
# Head training and the manual head apply run to 65 min (3900s) — past the
|
||||||
|
# ml queue's threshold (#4432). 70 = 65 + 5.
|
||||||
|
"backend.app.tasks.ml.train_heads": 70,
|
||||||
|
"backend.app.tasks.ml.apply_head_tags": 70,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -473,6 +489,10 @@ def prune_task_runs() -> dict:
|
|||||||
(recover_stalled_task_runs) is the mechanism that flips them to
|
(recover_stalled_task_runs) is the mechanism that flips them to
|
||||||
terminal state; prune doesn't touch in-flight state.
|
terminal state; prune doesn't touch in-flight state.
|
||||||
- 'retry' rows: treated as failures (>7d).
|
- 'retry' rows: treated as failures (>7d).
|
||||||
|
- The NEWEST row of each task is never deleted, whatever its age: it is
|
||||||
|
what the beat scheduler reads to know when a job last ran (#4408).
|
||||||
|
Without it a weekly job's last run would be pruned after a day, and beat
|
||||||
|
would think it had never run and fire it on every restart.
|
||||||
|
|
||||||
Returns dict of how many rows were deleted in each bucket.
|
Returns dict of how many rows were deleted in each bucket.
|
||||||
"""
|
"""
|
||||||
@@ -480,16 +500,19 @@ def prune_task_runs() -> dict:
|
|||||||
now = datetime.now(UTC)
|
now = datetime.now(UTC)
|
||||||
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
|
ok_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_OK_SECONDS)
|
||||||
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
|
fail_cutoff = now - timedelta(seconds=TASK_RUN_KEEP_FAILURE_SECONDS)
|
||||||
|
newest = select(func.max(TaskRun.id)).group_by(TaskRun.task_name)
|
||||||
with SessionLocal() as session:
|
with SessionLocal() as session:
|
||||||
ok_deleted = session.execute(
|
ok_deleted = session.execute(
|
||||||
delete(TaskRun)
|
delete(TaskRun)
|
||||||
.where(TaskRun.status == "ok")
|
.where(TaskRun.status == "ok")
|
||||||
.where(TaskRun.finished_at < ok_cutoff)
|
.where(TaskRun.finished_at < ok_cutoff)
|
||||||
|
.where(TaskRun.id.not_in(newest))
|
||||||
).rowcount or 0
|
).rowcount or 0
|
||||||
fail_deleted = session.execute(
|
fail_deleted = session.execute(
|
||||||
delete(TaskRun)
|
delete(TaskRun)
|
||||||
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
|
.where(TaskRun.status.in_(["error", "timeout", "retry"]))
|
||||||
.where(TaskRun.finished_at < fail_cutoff)
|
.where(TaskRun.finished_at < fail_cutoff)
|
||||||
|
.where(TaskRun.id.not_in(newest))
|
||||||
).rowcount or 0
|
).rowcount or 0
|
||||||
session.commit()
|
session.commit()
|
||||||
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
return {"ok_deleted": ok_deleted, "failures_deleted": fail_deleted}
|
||||||
@@ -708,6 +731,32 @@ def recover_stalled_download_events() -> int:
|
|||||||
return events_recovered
|
return events_recovered
|
||||||
|
|
||||||
|
|
||||||
|
@celery.task(
|
||||||
|
name="backend.app.tasks.maintenance.date_posts_from_records",
|
||||||
|
soft_time_limit=1500,
|
||||||
|
time_limit=1800,
|
||||||
|
)
|
||||||
|
def date_posts_from_records() -> dict:
|
||||||
|
"""Date undated native posts from the records their walk left on disk
|
||||||
|
(#4436). Hourly and self-limiting: once every post is dated it is one
|
||||||
|
empty query."""
|
||||||
|
from ..services.importer import Importer
|
||||||
|
from ..services.post_record_repair import date_posts_from_records as _repair
|
||||||
|
from ..services.thumbnailer import Thumbnailer
|
||||||
|
|
||||||
|
images_root = IMAGES_ROOT
|
||||||
|
SessionLocal = _sync_session_factory()
|
||||||
|
with SessionLocal() as session:
|
||||||
|
importer = Importer(
|
||||||
|
session=session,
|
||||||
|
images_root=images_root,
|
||||||
|
import_root=images_root,
|
||||||
|
thumbnailer=Thumbnailer(images_root=images_root),
|
||||||
|
settings=ImportSettings.load_sync(session),
|
||||||
|
)
|
||||||
|
return _repair(session, importer, images_root)
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
|
@celery.task(name="backend.app.tasks.maintenance.recover_stalled_backup_runs")
|
||||||
def recover_stalled_backup_runs() -> int:
|
def recover_stalled_backup_runs() -> int:
|
||||||
"""Flip BackupRun rows stuck in running/restoring past the hard limit
|
"""Flip BackupRun rows stuck in running/restoring past the hard limit
|
||||||
@@ -1034,8 +1083,7 @@ def cleanup_old_download_events() -> int:
|
|||||||
|
|
||||||
def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
||||||
"""One keyset-paginated pass over posts whose title matches a WIP tier, applying
|
"""One keyset-paginated pass over posts whose title matches a WIP tier, applying
|
||||||
`tag_id` (stamped `source`) to their images. Shared by the hard + soft tiers
|
`tag_id` (stamped `source`) to their images (#1458). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
||||||
(#1458 / #1474). Coarse `prefilter` (ILIKE superset) narrows the scan; the precise
|
|
||||||
`matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
|
`matcher` confirms. Idempotent-additive (ON CONFLICT DO NOTHING). Returns the row
|
||||||
count newly applied."""
|
count newly applied."""
|
||||||
from ..models import Post
|
from ..models import Post
|
||||||
@@ -1075,26 +1123,17 @@ def _backfill_wip_tier(session, tag_id, prefilter, matcher, source) -> int:
|
|||||||
)
|
)
|
||||||
def backfill_wip_title_tags() -> int:
|
def backfill_wip_title_tags() -> int:
|
||||||
"""Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
|
"""Scan EXISTING posts for WIP titles and apply the `wip` system tag to their
|
||||||
images — the operator-triggered back-catalogue catch-up (task #1458 hard tier +
|
images — the operator-triggered back-catalogue catch-up (task #1458). New
|
||||||
#1474 soft tier). New imports are tagged live by the importer; this covers the
|
imports are tagged live by the importer; this covers the existing library.
|
||||||
existing library.
|
Keyset-paginated, restart-safe.
|
||||||
|
|
||||||
HARD tier ("WIP"/"work in progress") always runs (the operator triggered the
|
|
||||||
scan); the SOFT tier (sketch/doodle, provisional source) runs only when
|
|
||||||
wip_soft_title_tagging_enabled, AFTER hard so a title matching both keeps the
|
|
||||||
trained hard tag (ON CONFLICT DO NOTHING). Keyset-paginated, restart-safe.
|
|
||||||
|
|
||||||
Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
|
Deliberately NOT scheduled as a beat: a periodic re-run would re-apply to matching
|
||||||
posts and silently undo a manual WIP removal, so it stays an explicit operator
|
posts and silently undo a manual WIP removal, so it stays an explicit operator
|
||||||
action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
|
action (Settings → "Scan existing posts for WIP titles"). Returns rows applied.
|
||||||
"""
|
"""
|
||||||
from ..models import ImportSettings
|
|
||||||
from ..services.wip_title import (
|
from ..services.wip_title import (
|
||||||
SOFT_WIP_TITLE_SQL_PREFILTER,
|
|
||||||
WIP_TITLE_SOFT_SOURCE,
|
|
||||||
WIP_TITLE_SOURCE,
|
WIP_TITLE_SOURCE,
|
||||||
WIP_TITLE_SQL_PREFILTER,
|
WIP_TITLE_SQL_PREFILTER,
|
||||||
matches_soft_wip_title,
|
|
||||||
matches_wip_title,
|
matches_wip_title,
|
||||||
resolve_wip_tag_id,
|
resolve_wip_tag_id,
|
||||||
)
|
)
|
||||||
@@ -1107,16 +1146,10 @@ def backfill_wip_title_tags() -> int:
|
|||||||
"backfill_wip_title_tags: no `wip` system tag present; nothing to do"
|
"backfill_wip_title_tags: no `wip` system tag present; nothing to do"
|
||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
settings = ImportSettings.load_sync(session)
|
|
||||||
applied = _backfill_wip_tier(
|
applied = _backfill_wip_tier(
|
||||||
session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
|
session, tag_id, WIP_TITLE_SQL_PREFILTER, matches_wip_title,
|
||||||
WIP_TITLE_SOURCE,
|
WIP_TITLE_SOURCE,
|
||||||
)
|
)
|
||||||
if settings.wip_soft_title_tagging_enabled:
|
|
||||||
applied += _backfill_wip_tier(
|
|
||||||
session, tag_id, SOFT_WIP_TITLE_SQL_PREFILTER, matches_soft_wip_title,
|
|
||||||
WIP_TITLE_SOFT_SOURCE,
|
|
||||||
)
|
|
||||||
if applied:
|
if applied:
|
||||||
log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
|
log.info("backfill_wip_title_tags: applied wip to %d image(s)", applied)
|
||||||
return applied
|
return applied
|
||||||
|
|||||||
@@ -620,24 +620,6 @@ def scheduled_process_auto_apply() -> str:
|
|||||||
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
return f"applied={result['n_applied']} flagged={result['n_flagged']}"
|
||||||
|
|
||||||
|
|
||||||
@celery.task(
|
|
||||||
name="backend.app.tasks.ml.scheduled_soft_wip_conflict_audit",
|
|
||||||
soft_time_limit=1800, time_limit=2100,
|
|
||||||
)
|
|
||||||
def scheduled_soft_wip_conflict_audit() -> str:
|
|
||||||
"""Ring-loud audit over the SOFT WIP-title cohort (#1474) — flag sketch/doodle
|
|
||||||
auto-tags that ALSO look like real content for review. No-op when there are no
|
|
||||||
content heads; idempotent (already-flagged images skipped). Runs regardless of
|
|
||||||
the process-sweep toggle, since soft-title tags come from the importer, not that
|
|
||||||
sweep. Wall-clock bounded by the task time limits."""
|
|
||||||
from ..services.ml.heads import soft_wip_conflict_audit
|
|
||||||
|
|
||||||
SessionLocal = _sync_session_factory()
|
|
||||||
with SessionLocal() as session:
|
|
||||||
result = soft_wip_conflict_audit(session)
|
|
||||||
return f"scanned={result['n_scanned']} flagged={result['n_flagged']}"
|
|
||||||
|
|
||||||
|
|
||||||
@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
|
@celery.task(name="backend.app.tasks.ml.prune_presentation_reviews")
|
||||||
def prune_presentation_reviews() -> str:
|
def prune_presentation_reviews() -> str:
|
||||||
"""Retention (rule 89): drop RESOLVED presentation-review flags older than 30
|
"""Retention (rule 89): drop RESOLVED presentation-review flags older than 30
|
||||||
|
|||||||
+7
-4
@@ -15,7 +15,7 @@ git.fabledsword.com/bvandeusen/ci-python:3.14
|
|||||||
|
|
||||||
## Secondary runtime image
|
## Secondary runtime image
|
||||||
|
|
||||||
node:24-bookworm-slim — `.forgejo/workflows/extension.yml` only.
|
node:24-bookworm-slim — `build.yml`'s `extension-test` job only.
|
||||||
|
|
||||||
`.forgejo/workflows/release.yml` runs on `ci-python:3.14` like everything else
|
`.forgejo/workflows/release.yml` runs on `ci-python:3.14` like everything else
|
||||||
and installs nothing: it needs git and stdlib python, and builds no image.
|
and installs nothing: it needs git and stdlib python, and builds no image.
|
||||||
@@ -29,8 +29,8 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
|||||||
|
|
||||||
- `pip install -r requirements.txt pytest pytest-asyncio` — in `backend-lint-and-test` and `integration` jobs
|
- `pip install -r requirements.txt pytest pytest-asyncio` — in `backend-lint-and-test` and `integration` jobs
|
||||||
- `npm install --no-audit --no-fund` — in `frontend-build` job
|
- `npm install --no-audit --no-fund` — in `frontend-build` job
|
||||||
- `npm install --no-audit --no-fund` — in `extension.yml`'s `lint` job (web-ext + vitest)
|
- `npm install --no-audit --no-fund` — in `build.yml`'s `extension-test` job (web-ext + vitest)
|
||||||
- `unzip` — in `extension.yml`'s "Verify XPI contents" step, installed via apt
|
- `unzip` — in `extension-test`'s "Verify XPI contents" step, installed via apt
|
||||||
only when absent (`node:24-bookworm-slim` may or may not carry it). Debian
|
only when absent (`node:24-bookworm-slim` may or may not carry it). Debian
|
||||||
package, ~2s. Not worth baking into a shared image for a single consumer, per
|
package, ~2s. Not worth baking into a shared image for a single consumer, per
|
||||||
`docs/process.md`'s ">1 project" rule.
|
`docs/process.md`'s ">1 project" rule.
|
||||||
@@ -180,7 +180,10 @@ per `docs/process.md`'s "add deps to the image when used by >1 project".
|
|||||||
BUILD_REF` that every checkout in the file takes, rather than per job —
|
BUILD_REF` that every checkout in the file takes, rather than per job —
|
||||||
otherwise `sign-extension` would derive dev's extension version while
|
otherwise `sign-extension` would derive dev's extension version while
|
||||||
`build-web` bundled main's, and the release download would 404 on a version
|
`build-web` bundled main's, and the release download would 404 on a version
|
||||||
that exists perfectly well. Every job then ASSERTS its checkout is `main`
|
that exists perfectly well. On every other trigger `BUILD_REF` is the
|
||||||
|
triggering COMMIT (`github.sha`), not the branch: a branch is re-resolved
|
||||||
|
per job, so a push landing mid-run used to move the publishing jobs onto a
|
||||||
|
commit the run's lanes never tested (run 7499, #4427). Every job then ASSERTS its checkout is `main`
|
||||||
before doing anything, because `env` inside `with:` is not a context this
|
before doing anything, because `env` inside `with:` is not a context this
|
||||||
runner is known to evaluate — if it silently resolved to empty, checkout
|
runner is known to evaluate — if it silently resolved to empty, checkout
|
||||||
would fall back to the triggering ref and the refresh would publish dev's
|
would fall back to the triggering ref and the refresh would publish dev's
|
||||||
|
|||||||
@@ -29,7 +29,7 @@
|
|||||||
|
|
||||||
services:
|
services:
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:8-alpine
|
||||||
volumes:
|
volumes:
|
||||||
- redis_data:/data
|
- redis_data:/data
|
||||||
healthcheck:
|
healthcheck:
|
||||||
|
|||||||
+1
-1
@@ -48,7 +48,7 @@ x-celery-healthcheck: &celery_healthcheck
|
|||||||
|
|
||||||
services:
|
services:
|
||||||
redis:
|
redis:
|
||||||
image: redis:7-alpine
|
image: redis:8-alpine
|
||||||
volumes:
|
volumes:
|
||||||
- redis_data:/data
|
- redis_data:/data
|
||||||
healthcheck:
|
healthcheck:
|
||||||
|
|||||||
@@ -88,7 +88,12 @@ async function checkForUpdateInfo() {
|
|||||||
currentVersion,
|
currentVersion,
|
||||||
latestVersion,
|
latestVersion,
|
||||||
channel,
|
channel,
|
||||||
xpiUrl: info && info.latest_url ? `${base}${info.latest_url}` : null,
|
// Where the Update button sends the operator: FC's own install card, not
|
||||||
|
// the XPI. Firefox refuses an add-on install whose navigation an extension
|
||||||
|
// started (tabs.create on the .xpi dies with NS_ERROR_FAILURE — operator-
|
||||||
|
// flagged 2026-09-25); it accepts one from a user click on a web page,
|
||||||
|
// which is exactly what the card's Install button is.
|
||||||
|
installPageUrl: base ? `${base}/subscriptions?tab=settings` : null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -128,7 +133,8 @@ browser.webRequest.onBeforeSendHeaders.addListener(
|
|||||||
saveDiscordToken(auth.value);
|
saveDiscordToken(auth.value);
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{ urls: ['https://discord.com/api/*'] },
|
// ptb/canary are Discord's beta clients; their API calls carry the same token.
|
||||||
|
{ urls: ['https://discord.com/api/*', 'https://*.discord.com/api/*'] },
|
||||||
['requestHeaders'],
|
['requestHeaders'],
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -213,7 +219,17 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
if (key === 'discord') {
|
if (key === 'discord') {
|
||||||
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
if (!discordToken) return { error: 'Open discord.com to capture a token first.' };
|
||||||
await api.uploadCredentials('discord', 'token', discordToken);
|
await api.uploadCredentials('discord', 'token', discordToken);
|
||||||
return { success: true };
|
// Then have FC try it against a Discord source, so a token Discord
|
||||||
|
// has already revoked shows up here rather than at the next check.
|
||||||
|
// A failed verify never undoes the upload: valid=null means FC could
|
||||||
|
// not test (no Discord source yet), not that the token is bad.
|
||||||
|
let verify = null;
|
||||||
|
try {
|
||||||
|
verify = await api.verifyCredential('discord');
|
||||||
|
} catch (e) {
|
||||||
|
verify = { valid: null, reason: e.message };
|
||||||
|
}
|
||||||
|
return { success: true, verify };
|
||||||
}
|
}
|
||||||
return { error: 'Unsupported platform.' };
|
return { error: 'Unsupported platform.' };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
@@ -256,14 +272,40 @@ browser.runtime.onMessage.addListener(async (msg) => {
|
|||||||
|
|
||||||
case 'ADD_AS_SOURCE':
|
case 'ADD_AS_SOURCE':
|
||||||
try {
|
try {
|
||||||
return await api.quickAddSource(msg.url);
|
return await api.quickAddSource(msg.url, {
|
||||||
|
artistId: msg.artistId ?? null,
|
||||||
|
artistName: msg.artistName ?? null,
|
||||||
|
usePlatformName: msg.usePlatformName === true,
|
||||||
|
discordAuthors: Array.isArray(msg.discordAuthors) ? msg.discordAuthors : null,
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
return { error: e.message };
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'DISCORD_POSTERS':
|
||||||
|
try {
|
||||||
|
return await api.getDiscordPosters(msg.url);
|
||||||
|
} catch (e) {
|
||||||
|
return { error: e.message };
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'SET_DISCORD_POSTERS':
|
||||||
|
try {
|
||||||
|
return await api.setDiscordPosters(msg.url, msg.authors || []);
|
||||||
|
} catch (e) {
|
||||||
|
return { error: e.message };
|
||||||
|
}
|
||||||
|
|
||||||
|
case 'SEARCH_ARTISTS':
|
||||||
|
try {
|
||||||
|
return { artists: await api.searchArtists(msg.q || '') };
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return { error: e.message };
|
return { error: e.message };
|
||||||
}
|
}
|
||||||
|
|
||||||
case 'PROBE_SOURCE':
|
case 'PROBE_SOURCE':
|
||||||
try {
|
try {
|
||||||
return await api.probeSource(msg.url);
|
return await api.probeSource(msg.url, { names: msg.names === true });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return { error: e.message };
|
return { error: e.message };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,3 +40,67 @@
|
|||||||
from { transform: translateY(20px); opacity: 0; }
|
from { transform: translateY(20px); opacity: 0; }
|
||||||
to { transform: translateY(0); opacity: 1; }
|
to { transform: translateY(0); opacity: 1; }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Discord Add panel — sits above the chip. Same slate/parchment palette. */
|
||||||
|
.fc-panel {
|
||||||
|
all: revert;
|
||||||
|
position: fixed; bottom: 76px; right: 24px; z-index: 2147483647;
|
||||||
|
box-sizing: border-box; width: 320px; max-width: calc(100vw - 48px);
|
||||||
|
padding: 14px 16px; border-radius: 10px;
|
||||||
|
background: rgb(20, 23, 26); color: rgb(232, 228, 216);
|
||||||
|
border: 1px solid rgb(60, 64, 70);
|
||||||
|
font: 14px/1.4 system-ui, sans-serif;
|
||||||
|
box-shadow: 0 8px 28px rgba(0, 0, 0, 0.5);
|
||||||
|
}
|
||||||
|
.fc-panel__title { font-weight: 600; color: rgb(244, 186, 122); }
|
||||||
|
.fc-panel__sub { color: rgb(170, 166, 156); font-size: 12px; margin-bottom: 8px; }
|
||||||
|
.fc-panel__label {
|
||||||
|
margin: 10px 0 4px; font-size: 11px; letter-spacing: 0.06em;
|
||||||
|
text-transform: uppercase; color: rgb(170, 166, 156);
|
||||||
|
}
|
||||||
|
.fc-panel__radio { display: flex; gap: 8px; align-items: center; padding: 2px 0; cursor: pointer; }
|
||||||
|
.fc-panel__radio input { margin: 0; accent-color: rgb(244, 186, 122); }
|
||||||
|
.fc-panel__input {
|
||||||
|
all: revert; box-sizing: border-box; width: 100%;
|
||||||
|
padding: 7px 9px; border-radius: 6px;
|
||||||
|
border: 1px solid rgb(70, 74, 80); background: rgb(12, 14, 16); color: inherit;
|
||||||
|
font: inherit;
|
||||||
|
}
|
||||||
|
.fc-panel__input:focus { outline: 2px solid rgb(244, 186, 122); outline-offset: -1px; }
|
||||||
|
.fc-panel__results { display: flex; flex-direction: column; max-height: 160px; overflow-y: auto; }
|
||||||
|
.fc-panel__result {
|
||||||
|
all: revert; text-align: left; padding: 6px 9px; border: none; border-radius: 4px;
|
||||||
|
background: transparent; color: inherit; font: inherit; cursor: pointer;
|
||||||
|
}
|
||||||
|
.fc-panel__result:hover, .fc-panel__result:focus { background: rgb(36, 40, 46); }
|
||||||
|
.fc-panel__hint { margin-top: 8px; font-size: 12px; color: rgb(170, 166, 156); }
|
||||||
|
.fc-panel__actions { display: flex; justify-content: flex-end; gap: 8px; margin-top: 12px; }
|
||||||
|
.fc-panel__btn {
|
||||||
|
all: revert; padding: 6px 14px; border-radius: 999px; cursor: pointer;
|
||||||
|
border: 1px solid rgb(70, 74, 80); background: transparent; color: inherit;
|
||||||
|
font: 500 13px/1.2 system-ui, sans-serif;
|
||||||
|
}
|
||||||
|
.fc-panel__btn--primary { border-color: rgb(244, 186, 122); background: rgb(244, 186, 122); color: rgb(20, 23, 26); }
|
||||||
|
.fc-panel__btn:disabled { opacity: 0.5; cursor: default; }
|
||||||
|
|
||||||
|
/* Artist autocomplete — the list sits under the field, inside the panel. */
|
||||||
|
.fc-panel__combo { position: relative; }
|
||||||
|
.fc-panel__results {
|
||||||
|
margin-top: 4px; border-radius: 6px;
|
||||||
|
background: rgb(12, 14, 16);
|
||||||
|
}
|
||||||
|
.fc-panel__results:empty { display: none; }
|
||||||
|
.fc-panel__results:not(:empty) { border: 1px solid rgb(70, 74, 80); padding: 3px; }
|
||||||
|
.fc-panel__result { display: flex; justify-content: space-between; align-items: center; width: 100%; box-sizing: border-box; }
|
||||||
|
.fc-panel__result--active { background: rgb(52, 44, 32); outline: 1px solid rgb(244, 186, 122); }
|
||||||
|
.fc-panel__result--new { color: rgb(244, 186, 122); }
|
||||||
|
.fc-panel__result-tag { font-size: 11px; color: rgb(140, 220, 160); }
|
||||||
|
.fc-panel__empty { padding: 6px 9px; font-size: 12px; color: rgb(170, 166, 156); }
|
||||||
|
.fc-panel__hint--match { color: rgb(140, 220, 160); }
|
||||||
|
/* The panel's own [hidden] — its rows are display:flex, which beats the UA's. */
|
||||||
|
.fc-panel [hidden] { display: none !important; }
|
||||||
|
.fc-panel__rename { margin-top: 8px; font-size: 13px; }
|
||||||
|
/* Poster picker (#4491): who posts in the channel, the likely creator ticked. */
|
||||||
|
.fc-panel__posters { max-height: 220px; overflow-y: auto; }
|
||||||
|
.fc-panel__poster { flex-wrap: wrap; }
|
||||||
|
.fc-panel__poster-meta { flex-basis: 100%; padding-left: 21px; font-size: 11px; color: rgb(170, 166, 156); }
|
||||||
|
|||||||
@@ -5,42 +5,61 @@
|
|||||||
// Cached probe result for the current URL so click-handlers know which
|
// Cached probe result for the current URL so click-handlers know which
|
||||||
// action to dispatch without round-tripping again.
|
// action to dispatch without round-tripping again.
|
||||||
let currentProbe = null;
|
let currentProbe = null;
|
||||||
|
// Bumped on every evaluate(): a probe that answers after the operator has
|
||||||
|
// navigated on is for a page they've left, and must not repaint the chip.
|
||||||
|
let generation = 0;
|
||||||
|
let lastUrl = window.location.href;
|
||||||
|
|
||||||
evaluate();
|
evaluate();
|
||||||
|
|
||||||
const reEval = () => evaluate();
|
// SPA navigation. Patreon, SubscribeStar and above all Discord change
|
||||||
window.addEventListener('popstate', reEval);
|
// channel/page without a reload. Patching history.pushState from here never
|
||||||
const origPush = history.pushState;
|
// worked: a content script runs in an isolated world, so the page's own
|
||||||
history.pushState = function () { origPush.apply(this, arguments); reEval(); };
|
// pushState is not the function we'd replace. Polling the URL is the one
|
||||||
|
// signal that sees every navigation, and costs a string compare.
|
||||||
|
window.addEventListener('popstate', () => onUrlMaybeChanged());
|
||||||
|
setInterval(onUrlMaybeChanged, 500);
|
||||||
|
|
||||||
|
function onUrlMaybeChanged() {
|
||||||
|
if (window.location.href === lastUrl) return;
|
||||||
|
lastUrl = window.location.href;
|
||||||
|
closePanel();
|
||||||
|
evaluate();
|
||||||
|
}
|
||||||
|
|
||||||
async function evaluate() {
|
async function evaluate() {
|
||||||
|
const mine = ++generation;
|
||||||
const url = window.location.href;
|
const url = window.location.href;
|
||||||
const platform = getPlatformFromUrl(url);
|
const platform = getPlatformFromUrl(url);
|
||||||
const onArtist = platform && isArtistPage(url, platform);
|
const onArtist = platform && isArtistPage(url, platform);
|
||||||
const btn = document.getElementById('fc-add-source-btn');
|
|
||||||
if (!onArtist) {
|
if (!onArtist) {
|
||||||
if (btn) btn.remove();
|
removeButton();
|
||||||
currentProbe = null;
|
currentProbe = null;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// On artist pages, ask the backend what state the URL is in BEFORE
|
// Ask the backend what state the URL is in BEFORE drawing the button, so
|
||||||
// injecting the button — so the chip can render the right state on
|
// the chip renders the right state on first paint instead of flashing the
|
||||||
// first paint instead of flashing the generic "Add" copy and
|
// generic "Add" copy and updating afterwards.
|
||||||
// updating afterwards.
|
|
||||||
let probe;
|
let probe;
|
||||||
try {
|
try {
|
||||||
probe = await browser.runtime.sendMessage({ type: 'PROBE_SOURCE', url });
|
probe = await browser.runtime.sendMessage({ type: 'PROBE_SOURCE', url });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
probe = { error: e?.message || 'probe failed' };
|
probe = { error: e?.message || 'probe failed' };
|
||||||
}
|
}
|
||||||
|
if (mine !== generation) return;
|
||||||
currentProbe = probe;
|
currentProbe = probe;
|
||||||
if (probe?.state === 'unknown_platform') {
|
if (probe?.state === 'unknown_platform') {
|
||||||
if (btn) btn.remove();
|
removeButton();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
renderButton(probe);
|
renderButton(probe);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function removeButton() {
|
||||||
|
document.getElementById('fc-add-source-btn')?.remove();
|
||||||
|
closePanel();
|
||||||
|
}
|
||||||
|
|
||||||
function renderButton(probe) {
|
function renderButton(probe) {
|
||||||
let btn = document.getElementById('fc-add-source-btn');
|
let btn = document.getElementById('fc-add-source-btn');
|
||||||
if (!btn) {
|
if (!btn) {
|
||||||
@@ -51,79 +70,44 @@
|
|||||||
}
|
}
|
||||||
// Reset state classes so re-renders (SPA navigation) don't stack.
|
// Reset state classes so re-renders (SPA navigation) don't stack.
|
||||||
btn.className = 'fc-add-source-btn';
|
btn.className = 'fc-add-source-btn';
|
||||||
btn.classList.add(`fc-add-source-btn--${stateModifier(probe)}`);
|
btn.classList.add(`fc-add-source-btn--${chipState(probe)}`);
|
||||||
btn.textContent = labelFor(probe);
|
btn.textContent = chipLabel(probe, PLATFORMS[probe?.platform]?.name || probe?.platform || '');
|
||||||
btn.disabled = false;
|
btn.disabled = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
function stateModifier(probe) {
|
|
||||||
if (!probe || probe.error) return 'new';
|
|
||||||
return ({
|
|
||||||
source_match: 'source-match',
|
|
||||||
artist_match: 'artist-match',
|
|
||||||
new: 'new',
|
|
||||||
})[probe.state] || 'new';
|
|
||||||
}
|
|
||||||
|
|
||||||
function labelFor(probe) {
|
|
||||||
if (!probe || probe.error) return '+ Add to FabledCurator';
|
|
||||||
const platformName = platformDisplayName(probe.platform);
|
|
||||||
const artistName = probe.artist?.name;
|
|
||||||
switch (probe.state) {
|
|
||||||
case 'source_match':
|
|
||||||
return `✓ In FabledCurator · ${platformName}`;
|
|
||||||
case 'artist_match':
|
|
||||||
return `+ Add ${platformName} source to ${artistName || 'artist'}`;
|
|
||||||
case 'new':
|
|
||||||
default:
|
|
||||||
return '+ Add to FabledCurator';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function platformDisplayName(key) {
|
|
||||||
return PLATFORMS[key]?.name || key || '';
|
|
||||||
}
|
|
||||||
|
|
||||||
async function onClick() {
|
async function onClick() {
|
||||||
const btn = document.getElementById('fc-add-source-btn');
|
const btn = document.getElementById('fc-add-source-btn');
|
||||||
if (!btn) return;
|
if (!btn) return;
|
||||||
btn.disabled = true;
|
|
||||||
const original = btn.textContent;
|
|
||||||
const probe = currentProbe;
|
const probe = currentProbe;
|
||||||
|
|
||||||
if (probe?.state === 'source_match') {
|
if (probe?.state === 'source_match') {
|
||||||
btn.textContent = 'Opening…';
|
// A followed Discord channel opens its poster list (#4491); the artist
|
||||||
try {
|
// page is one click further, in that panel.
|
||||||
const r = await browser.runtime.sendMessage({
|
if (probe.platform === 'discord' && probe.discord?.channel_id) {
|
||||||
type: 'OPEN_ARTIST_PAGE',
|
if (document.getElementById('fc-add-panel')) closePanel();
|
||||||
slug: probe.artist?.slug,
|
else openPostersPanel(btn, probe);
|
||||||
});
|
|
||||||
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
|
||||||
} catch (e) {
|
|
||||||
showToast(`Error: ${e.message}`, 'error');
|
|
||||||
} finally {
|
|
||||||
btn.disabled = false;
|
|
||||||
btn.textContent = original;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
btn.textContent = 'Adding…';
|
|
||||||
try {
|
|
||||||
const r = await browser.runtime.sendMessage({
|
|
||||||
type: 'ADD_AS_SOURCE',
|
|
||||||
url: window.location.href,
|
|
||||||
});
|
|
||||||
if (r?.error) {
|
|
||||||
showToast(`Error: ${r.error}`, 'error');
|
|
||||||
} else {
|
|
||||||
const verb = r.created_source ? 'Added' : 'Already a source for';
|
|
||||||
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})`, 'success');
|
|
||||||
// Re-probe so the chip flips green without waiting for the next
|
|
||||||
// navigation.
|
|
||||||
evaluate();
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
await openArtist(btn, probe.artist?.slug);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Every add goes through the panel, so the operator can match the page to
|
||||||
|
// an artist FabledCurator already has (the same creator is often spelled
|
||||||
|
// differently per platform) instead of minting a duplicate.
|
||||||
|
if (document.getElementById('fc-add-panel')) {
|
||||||
|
closePanel();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await openAddPanel(btn, probe);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openArtist(btn, slug) {
|
||||||
|
btn.disabled = true;
|
||||||
|
const original = btn.textContent;
|
||||||
|
btn.textContent = 'Opening…';
|
||||||
|
try {
|
||||||
|
const r = await browser.runtime.sendMessage({ type: 'OPEN_ARTIST_PAGE', slug });
|
||||||
|
if (r?.error) showToast(`Error: ${r.error}`, 'error');
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
showToast(`Error: ${e.message}`, 'error');
|
showToast(`Error: ${e.message}`, 'error');
|
||||||
} finally {
|
} finally {
|
||||||
@@ -132,6 +116,425 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One add, shared by the one-click chip and the Discord panel. Resolves
|
||||||
|
// true on success.
|
||||||
|
async function add(btn, request) {
|
||||||
|
btn.disabled = true;
|
||||||
|
const original = btn.textContent;
|
||||||
|
btn.textContent = 'Adding…';
|
||||||
|
try {
|
||||||
|
const r = await browser.runtime.sendMessage({ type: 'ADD_AS_SOURCE', ...request });
|
||||||
|
if (r?.error) {
|
||||||
|
showToast(`Error: ${r.error}`, 'error');
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const verb = r.created_source ? 'Added to' : 'Already a source for';
|
||||||
|
const renamed = r.renamed_from ? ` — renamed from “${r.renamed_from}”` : '';
|
||||||
|
showToast(`${verb} ${r.artist?.name || 'artist'} (${r.source?.platform || ''})${renamed}`, 'success');
|
||||||
|
// Re-probe so the chip flips green without waiting for a navigation.
|
||||||
|
evaluate();
|
||||||
|
return true;
|
||||||
|
} catch (e) {
|
||||||
|
showToast(`Error: ${e.message}`, 'error');
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
btn.disabled = false;
|
||||||
|
btn.textContent = original;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Add panel ----
|
||||||
|
// Who: the suggested artist, one found by search, or a new one by name —
|
||||||
|
// on every platform. Where (Discord only): this channel or the whole
|
||||||
|
// server. On Patreon, joining an artist known by another name offers the
|
||||||
|
// Patreon name, which the operator treats as canon. Built with
|
||||||
|
// createElement only — server, channel and artist names are other people's
|
||||||
|
// text.
|
||||||
|
|
||||||
|
function el(tag, props = {}, children = []) {
|
||||||
|
const node = document.createElement(tag);
|
||||||
|
const { class: className, text, ...rest } = props;
|
||||||
|
if (className) node.className = className;
|
||||||
|
if (text != null) node.textContent = text;
|
||||||
|
Object.assign(node, rest);
|
||||||
|
for (const c of children) node.appendChild(c);
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
|
||||||
|
function closePanel() {
|
||||||
|
document.getElementById('fc-add-panel')?.remove();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Poster picker (#4491) ----
|
||||||
|
// Who posted in this channel lately, read by FabledCurator with its Discord
|
||||||
|
// token, the creator the server is named for (or owned by) ticked. The
|
||||||
|
// source keeps their ids: a username can change on a whim, an id can't, so
|
||||||
|
// the list never quietly stops matching. Names are other people's text —
|
||||||
|
// createElement only.
|
||||||
|
|
||||||
|
function posterSection() {
|
||||||
|
const list = el('div', { class: 'fc-panel__posters' }, [
|
||||||
|
el('div', { class: 'fc-panel__hint', text: 'Reading who posts here…' }),
|
||||||
|
]);
|
||||||
|
const node = el('div', {}, [
|
||||||
|
el('div', { class: 'fc-panel__label', text: 'Only posts by' }),
|
||||||
|
list,
|
||||||
|
]);
|
||||||
|
const state = { data: null, picked: new Set() };
|
||||||
|
|
||||||
|
function render() {
|
||||||
|
const posters = withSelected(state.data);
|
||||||
|
if (!posters.length) {
|
||||||
|
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: 'No one has posted here lately.' }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const rows = posters.map((p) => {
|
||||||
|
const id = String(p.id);
|
||||||
|
const box = el('input', { type: 'checkbox', checked: state.picked.has(id) });
|
||||||
|
box.addEventListener('change', () => {
|
||||||
|
if (box.checked) state.picked.add(id);
|
||||||
|
else state.picked.delete(id);
|
||||||
|
hint.textContent = pickHint();
|
||||||
|
});
|
||||||
|
const bits = [`${p.images} image${p.images === 1 ? '' : 's'}`, `${p.messages} message${p.messages === 1 ? '' : 's'}`];
|
||||||
|
if (p.reasons?.length) bits.unshift(p.reasons.join(', '));
|
||||||
|
return el('label', { class: 'fc-panel__radio fc-panel__poster' }, [
|
||||||
|
box,
|
||||||
|
el('span', { text: posterLabel(p) }),
|
||||||
|
el('span', { class: 'fc-panel__poster-meta', text: bits.join(' · ') }),
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
const hint = el('div', { class: 'fc-panel__hint', text: pickHint() });
|
||||||
|
list.replaceChildren(...rows, hint);
|
||||||
|
}
|
||||||
|
|
||||||
|
function pickHint() {
|
||||||
|
return state.picked.size
|
||||||
|
? 'Only the ticked people’s posts are taken.'
|
||||||
|
: 'Nothing ticked: everyone’s posts are taken.';
|
||||||
|
}
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
let r;
|
||||||
|
try {
|
||||||
|
r = await browser.runtime.sendMessage({ type: 'DISCORD_POSTERS', url: window.location.href });
|
||||||
|
} catch (e) {
|
||||||
|
r = { error: e.message };
|
||||||
|
}
|
||||||
|
if (r?.error) {
|
||||||
|
list.replaceChildren(el('div', { class: 'fc-panel__hint', text: `Couldn’t list posters — ${r.error}` }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
state.data = r;
|
||||||
|
state.picked = initialPosterPick(r);
|
||||||
|
render();
|
||||||
|
}
|
||||||
|
|
||||||
|
// null until the list has loaded: an add made before then leaves the
|
||||||
|
// source's list as it is instead of clearing it.
|
||||||
|
const request = () => (state.data ? postersRequest(state.data.posters, state.picked) : null);
|
||||||
|
return { node, load, request };
|
||||||
|
}
|
||||||
|
|
||||||
|
function openPostersPanel(btn, probe) {
|
||||||
|
closePanel();
|
||||||
|
const d = probe.discord || {};
|
||||||
|
const posters = posterSection();
|
||||||
|
const saveBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Save' });
|
||||||
|
const artistBtn = el('button', { class: 'fc-panel__btn', text: `Open ${probe.artist?.name || 'artist'}` });
|
||||||
|
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Close' });
|
||||||
|
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||||
|
el('div', { class: 'fc-panel__title', text: 'Discord source' }),
|
||||||
|
el('div', { class: 'fc-panel__sub', text: d.channel_id ? channelLabel(d) : serverLabel(d) }),
|
||||||
|
posters.node,
|
||||||
|
el('div', { class: 'fc-panel__actions' }, [artistBtn, cancelBtn, saveBtn]),
|
||||||
|
]);
|
||||||
|
panel.addEventListener('keydown', (e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
if (e.key === 'Escape') closePanel();
|
||||||
|
});
|
||||||
|
cancelBtn.addEventListener('click', closePanel);
|
||||||
|
artistBtn.addEventListener('click', () => { closePanel(); openArtist(btn, probe.artist?.slug); });
|
||||||
|
saveBtn.addEventListener('click', async () => {
|
||||||
|
const authors = posters.request();
|
||||||
|
if (!authors) return;
|
||||||
|
saveBtn.disabled = true;
|
||||||
|
try {
|
||||||
|
const r = await browser.runtime.sendMessage({
|
||||||
|
type: 'SET_DISCORD_POSTERS', url: window.location.href, authors,
|
||||||
|
});
|
||||||
|
if (r?.error) {
|
||||||
|
showToast(`Error: ${r.error}`, 'error');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
showToast(authors.length ? `Only posts by ${authors.map((a) => a.name || a.id).join(', ')}` : 'Taking everyone’s posts', 'success');
|
||||||
|
closePanel();
|
||||||
|
} catch (e) {
|
||||||
|
showToast(`Error: ${e.message}`, 'error');
|
||||||
|
} finally {
|
||||||
|
saveBtn.disabled = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
document.body.appendChild(panel);
|
||||||
|
posters.load();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openAddPanel(btn, probe) {
|
||||||
|
closePanel();
|
||||||
|
const platformName = PLATFORMS[probe.platform]?.name || probe.platform;
|
||||||
|
// Discord's probe already carries its names. Patreon/SubscribeStar read the
|
||||||
|
// creator's display name only now, when the panel needs it — a request to
|
||||||
|
// the platform the chip's own probe deliberately doesn't make.
|
||||||
|
if (probe.platform !== 'discord') {
|
||||||
|
const original = btn.textContent;
|
||||||
|
btn.disabled = true;
|
||||||
|
btn.textContent = `Reading the ${platformName} name…`;
|
||||||
|
try {
|
||||||
|
const named = await browser.runtime.sendMessage({
|
||||||
|
type: 'PROBE_SOURCE', url: window.location.href, names: true,
|
||||||
|
});
|
||||||
|
if (named && !named.error) probe = named;
|
||||||
|
} catch { /* fall back to the chip's probe: the URL handle */ }
|
||||||
|
btn.disabled = false;
|
||||||
|
btn.textContent = original;
|
||||||
|
if (probe.state === 'source_match') {
|
||||||
|
renderButton(probe);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const d = probe.discord || {};
|
||||||
|
const discord = probe.platform === 'discord';
|
||||||
|
const choice = panelDefaults(probe, window.location.href);
|
||||||
|
// #4491: on a channel page, pick whose posts the new source takes.
|
||||||
|
const posters = discord && d.channel_id ? posterSection() : null;
|
||||||
|
|
||||||
|
const scopeRow = (value, label, disabled) => {
|
||||||
|
const input = el('input', {
|
||||||
|
type: 'radio', name: 'fc-discord-scope', value,
|
||||||
|
checked: choice.scope === value, disabled,
|
||||||
|
});
|
||||||
|
input.addEventListener('change', () => { choice.scope = value; refresh(); });
|
||||||
|
return el('label', { class: 'fc-panel__radio' }, [input, el('span', { text: label })]);
|
||||||
|
};
|
||||||
|
|
||||||
|
// The artist field is an autocomplete over FC's artists: it searches as
|
||||||
|
// soon as the panel opens (with the prefilled name) and on every keystroke,
|
||||||
|
// lists the matches under the field, fills in the rest of the top match as
|
||||||
|
// you type (Tab or Enter accepts it, typing on replaces it), and picks an
|
||||||
|
// artist whose name IS the text, spacing and case aside, without being
|
||||||
|
// asked. ↑/↓ walk the list; the last row creates a new artist instead.
|
||||||
|
const nameInput = el('input', {
|
||||||
|
type: 'text', class: 'fc-panel__input', value: choice.artistName,
|
||||||
|
placeholder: 'Search artists or type a new name',
|
||||||
|
autocomplete: 'off', spellcheck: false,
|
||||||
|
role: 'combobox',
|
||||||
|
});
|
||||||
|
nameInput.setAttribute('aria-autocomplete', 'both');
|
||||||
|
nameInput.setAttribute('aria-expanded', 'false');
|
||||||
|
const results = el('div', { class: 'fc-panel__results', role: 'listbox' });
|
||||||
|
const hint = el('div', { class: 'fc-panel__hint' });
|
||||||
|
const addBtn = el('button', { class: 'fc-panel__btn fc-panel__btn--primary', text: 'Add' });
|
||||||
|
const cancelBtn = el('button', { class: 'fc-panel__btn', text: 'Cancel' });
|
||||||
|
|
||||||
|
// Patreon is canon: joining an artist known by another name takes the
|
||||||
|
// Patreon name unless this is unticked. Shown only when it would rename.
|
||||||
|
const renameBox = el('input', { type: 'checkbox', checked: choice.adoptPlatformName });
|
||||||
|
const renameText = el('span');
|
||||||
|
const renameRow = el('label', { class: 'fc-panel__radio fc-panel__rename' }, [renameBox, renameText]);
|
||||||
|
renameBox.addEventListener('change', () => { choice.adoptPlatformName = renameBox.checked; refresh(); });
|
||||||
|
|
||||||
|
const sub = discord
|
||||||
|
? serverLabel(d)
|
||||||
|
: [probe.display_name, probe.slug].filter(Boolean).filter((v, i, a) => a.indexOf(v) === i).join(' · ');
|
||||||
|
const panel = el('div', { id: 'fc-add-panel', class: 'fc-panel' }, [
|
||||||
|
el('div', { class: 'fc-panel__title', text: `Add ${platformName} source` }),
|
||||||
|
el('div', { class: 'fc-panel__sub', text: sub }),
|
||||||
|
...(discord ? [
|
||||||
|
el('div', { class: 'fc-panel__label', text: 'Follow' }),
|
||||||
|
scopeRow('channel', d.channel_id ? channelLabel(d) : 'this channel', !d.channel_id),
|
||||||
|
scopeRow('server', `Every channel in ${serverLabel(d)}`, false),
|
||||||
|
] : []),
|
||||||
|
el('div', { class: 'fc-panel__label', text: 'Artist' }),
|
||||||
|
el('div', { class: 'fc-panel__combo' }, [nameInput, results]),
|
||||||
|
renameRow,
|
||||||
|
...(posters ? [posters.node] : []),
|
||||||
|
hint,
|
||||||
|
el('div', { class: 'fc-panel__actions' }, [cancelBtn, addBtn]),
|
||||||
|
]);
|
||||||
|
|
||||||
|
// Search state: the rows on screen, which one ↑/↓ has highlighted (-1 =
|
||||||
|
// none), and whether the list is open.
|
||||||
|
let rows = [];
|
||||||
|
let active = -1;
|
||||||
|
let listOpen = false;
|
||||||
|
|
||||||
|
function refresh() {
|
||||||
|
const req = addRequest(choice);
|
||||||
|
addBtn.disabled = !req;
|
||||||
|
if (!req) hint.textContent = 'Pick an artist or type a name.';
|
||||||
|
else if (req.artistId != null) hint.textContent = `✓ Connects to ${choice.artist.name}, already in FabledCurator.`;
|
||||||
|
else if (req.artistName) hint.textContent = `Creates a new artist “${req.artistName}”.`;
|
||||||
|
else hint.textContent = `Creates a new artist, named from the ${platformName} page.`;
|
||||||
|
hint.classList.toggle('fc-panel__hint--match', !!req && req.artistId != null);
|
||||||
|
const offer = renameOffer(choice);
|
||||||
|
renameRow.hidden = !offer;
|
||||||
|
if (offer) renameText.textContent = `Rename “${offer.from}” to the Patreon name “${offer.to}”`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function pick(artist) {
|
||||||
|
choice.artist = artist ? { id: artist.id, name: artist.name } : null;
|
||||||
|
if (artist) {
|
||||||
|
choice.artistName = artist.name;
|
||||||
|
nameInput.value = artist.name;
|
||||||
|
}
|
||||||
|
closeList();
|
||||||
|
refresh();
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeList() {
|
||||||
|
listOpen = false;
|
||||||
|
active = -1;
|
||||||
|
results.replaceChildren();
|
||||||
|
nameInput.setAttribute('aria-expanded', 'false');
|
||||||
|
}
|
||||||
|
|
||||||
|
function renderList() {
|
||||||
|
const typed = nameInput.value.trim();
|
||||||
|
const exact = exactArtistMatch(typed, rows);
|
||||||
|
const items = rows.map((a, i) => {
|
||||||
|
const row = el('button', { type: 'button', class: 'fc-panel__result', role: 'option' }, [
|
||||||
|
el('span', { text: a.name }),
|
||||||
|
]);
|
||||||
|
if (choice.artist && choice.artist.id === a.id) {
|
||||||
|
row.appendChild(el('span', { class: 'fc-panel__result-tag', text: 'selected' }));
|
||||||
|
}
|
||||||
|
row.classList.toggle('fc-panel__result--active', i === active);
|
||||||
|
// mousedown, not click: it fires before the input's blur closes the list.
|
||||||
|
row.addEventListener('mousedown', (e) => { e.preventDefault(); pick(a); });
|
||||||
|
return row;
|
||||||
|
});
|
||||||
|
if (typed && !exact) {
|
||||||
|
const i = rows.length;
|
||||||
|
const create = el('button', { type: 'button', class: 'fc-panel__result fc-panel__result--new', role: 'option',
|
||||||
|
text: `+ New artist “${typed}”` });
|
||||||
|
create.classList.toggle('fc-panel__result--active', i === active);
|
||||||
|
create.addEventListener('mousedown', (e) => { e.preventDefault(); pick(null); choice.artistName = typed; refresh(); });
|
||||||
|
items.push(create);
|
||||||
|
}
|
||||||
|
if (typed && !rows.length) {
|
||||||
|
items.unshift(el('div', { class: 'fc-panel__empty', text: 'No FabledCurator artist matches.' }));
|
||||||
|
}
|
||||||
|
results.replaceChildren(...items);
|
||||||
|
listOpen = items.length > 0;
|
||||||
|
nameInput.setAttribute('aria-expanded', String(listOpen));
|
||||||
|
results.querySelector('.fc-panel__result--active')?.scrollIntoView({ block: 'nearest' });
|
||||||
|
}
|
||||||
|
|
||||||
|
let debounce = null;
|
||||||
|
let searchSeq = 0;
|
||||||
|
// `autofill` is false for deletions: filling the name back in as you
|
||||||
|
// backspace would make it impossible to delete.
|
||||||
|
function search(autofill) {
|
||||||
|
clearTimeout(debounce);
|
||||||
|
const q = nameInput.value.trim();
|
||||||
|
if (!q) { rows = []; closeList(); return; }
|
||||||
|
debounce = setTimeout(async () => {
|
||||||
|
const mine = ++searchSeq;
|
||||||
|
let r;
|
||||||
|
try {
|
||||||
|
r = await browser.runtime.sendMessage({ type: 'SEARCH_ARTISTS', q });
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Stale: a later keystroke has its own search coming.
|
||||||
|
if (mine !== searchSeq || r?.error || nameInput.value.trim() !== q) return;
|
||||||
|
rows = r.artists || [];
|
||||||
|
active = -1;
|
||||||
|
const exact = exactArtistMatch(q, rows);
|
||||||
|
if (exact && !choice.artist) {
|
||||||
|
choice.artist = { id: exact.id, name: exact.name };
|
||||||
|
} else if (autofill && document.activeElement === nameInput) {
|
||||||
|
const hit = inlineCompletion(nameInput.value, rows);
|
||||||
|
const caret = nameInput.value.length;
|
||||||
|
if (hit && nameInput.selectionStart === caret) {
|
||||||
|
nameInput.value = nameInput.value + hit.name.slice(caret);
|
||||||
|
nameInput.setSelectionRange(caret, hit.name.length);
|
||||||
|
choice.artist = { id: hit.id, name: hit.name };
|
||||||
|
choice.artistName = hit.name;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
renderList();
|
||||||
|
refresh();
|
||||||
|
}, 150);
|
||||||
|
}
|
||||||
|
|
||||||
|
nameInput.addEventListener('input', (e) => {
|
||||||
|
choice.artistName = nameInput.value;
|
||||||
|
// Any edit un-picks: the artist is whatever the field now says.
|
||||||
|
choice.artist = null;
|
||||||
|
refresh();
|
||||||
|
search(!String(e.inputType || '').startsWith('delete'));
|
||||||
|
});
|
||||||
|
nameInput.addEventListener('focus', () => { if (rows.length) renderList(); });
|
||||||
|
nameInput.addEventListener('blur', () => closeList());
|
||||||
|
|
||||||
|
// Keep Discord's global shortcuts from eating keystrokes meant for us.
|
||||||
|
panel.addEventListener('keydown', (e) => {
|
||||||
|
e.stopPropagation();
|
||||||
|
const count = results.querySelectorAll('.fc-panel__result').length;
|
||||||
|
if (e.target === nameInput && listOpen && count) {
|
||||||
|
if (e.key === 'ArrowDown' || e.key === 'ArrowUp') {
|
||||||
|
e.preventDefault();
|
||||||
|
active = e.key === 'ArrowDown' ? (active + 1) % count : (active <= 0 ? count - 1 : active - 1);
|
||||||
|
renderList();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ((e.key === 'Enter' || e.key === 'Tab') && active >= 0) {
|
||||||
|
e.preventDefault();
|
||||||
|
results.querySelectorAll('.fc-panel__result')[active]
|
||||||
|
.dispatchEvent(new MouseEvent('mousedown', { cancelable: true }));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if ((e.key === 'Tab' || e.key === 'Enter') && choice.artist
|
||||||
|
&& nameInput.selectionEnd > nameInput.selectionStart) {
|
||||||
|
// Accept the inline autofill — Enter too, but only to accept: the
|
||||||
|
// add itself takes a second Enter, once the hint names the artist.
|
||||||
|
e.preventDefault();
|
||||||
|
pick(choice.artist);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (e.key === 'Escape') {
|
||||||
|
if (listOpen) closeList();
|
||||||
|
else closePanel();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (e.key === 'Enter' && e.target === nameInput && !addBtn.disabled) addBtn.click();
|
||||||
|
});
|
||||||
|
|
||||||
|
cancelBtn.addEventListener('click', closePanel);
|
||||||
|
addBtn.addEventListener('click', async () => {
|
||||||
|
const req = addRequest(choice);
|
||||||
|
if (!req) return;
|
||||||
|
const authors = posters?.request();
|
||||||
|
if (authors) req.discordAuthors = authors;
|
||||||
|
const btn = document.getElementById('fc-add-source-btn');
|
||||||
|
addBtn.disabled = true;
|
||||||
|
const ok = await add(btn || addBtn, req);
|
||||||
|
if (ok) closePanel();
|
||||||
|
else refresh();
|
||||||
|
});
|
||||||
|
|
||||||
|
document.body.appendChild(panel);
|
||||||
|
refresh();
|
||||||
|
nameInput.focus();
|
||||||
|
posters?.load();
|
||||||
|
nameInput.select();
|
||||||
|
// Search what the field opens with — the server's name, usually — so an
|
||||||
|
// artist it already matches is picked before the operator types anything.
|
||||||
|
if (!choice.artist && nameInput.value.trim()) search(false);
|
||||||
|
}
|
||||||
|
|
||||||
function showToast(text, kind) {
|
function showToast(text, kind) {
|
||||||
const t = document.createElement('div');
|
const t = document.createElement('div');
|
||||||
t.className = `fc-toast fc-toast--${kind}`;
|
t.className = `fc-toast fc-toast--${kind}`;
|
||||||
|
|||||||
+41
-5
@@ -80,6 +80,17 @@ class FabledCuratorAPI {
|
|||||||
getCredentials() {
|
getCredentials() {
|
||||||
return this.request('GET', '/credentials');
|
return this.request('GET', '/credentials');
|
||||||
}
|
}
|
||||||
|
// Test the STORED credential against one of the platform's sources — the
|
||||||
|
// same check the web UI's Verify button runs. {valid: true|false|null, reason}.
|
||||||
|
verifyCredential(platform) {
|
||||||
|
return this.request('POST', `/credentials/${encodeURIComponent(platform)}/verify`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Artist search for the Discord Add panel — the web UI's autocomplete.
|
||||||
|
searchArtists(q, limit = 8) {
|
||||||
|
const qs = new URLSearchParams({ q, limit: String(limit) }).toString();
|
||||||
|
return this.request('GET', `/artists/autocomplete?${qs}`);
|
||||||
|
}
|
||||||
|
|
||||||
// FC-3a — sources.
|
// FC-3a — sources.
|
||||||
listSources() {
|
listSources() {
|
||||||
@@ -90,15 +101,40 @@ class FabledCuratorAPI {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// FC-3g — extension-specific.
|
// FC-3g — extension-specific.
|
||||||
quickAddSource(url) {
|
// artistId connects the source to an existing artist, artistName to the
|
||||||
return this.request('POST', '/extension/quick-add-source', { url });
|
// artist of that name (created if new); with neither the server derives the
|
||||||
|
// artist from the URL. A Discord channel always sends one.
|
||||||
|
// usePlatformName: a Patreon source joining an existing artist renames it
|
||||||
|
// to the Patreon display name (Patreon is canon; name only, never the slug).
|
||||||
|
// discordAuthors ([{id, name}], #4491): a Discord source's poster list, set
|
||||||
|
// in the same step.
|
||||||
|
quickAddSource(url, { artistId = null, artistName = null, usePlatformName = false, discordAuthors = null } = {}) {
|
||||||
|
const body = { url };
|
||||||
|
if (discordAuthors) body.discord_authors = discordAuthors;
|
||||||
|
if (artistId != null) body.artist_id = artistId;
|
||||||
|
else if (artistName) body.artist_name = artistName;
|
||||||
|
if (usePlatformName) body.use_platform_name = true;
|
||||||
|
return this.request('POST', '/extension/quick-add-source', body);
|
||||||
}
|
}
|
||||||
probeSource(url) {
|
probeSource(url, { names = false } = {}) {
|
||||||
// Read-only existence check. Drives the content-script chip's
|
// Read-only existence check. Drives the content-script chip's
|
||||||
// color/copy BEFORE the operator clicks Add.
|
// color/copy BEFORE the operator clicks Add. `names` also reads the
|
||||||
const qs = new URLSearchParams({ url }).toString();
|
// creator's display name from the platform — the Add panel asks for it,
|
||||||
|
// the chip doesn't, so a plain page view never costs a platform request.
|
||||||
|
const params = { url };
|
||||||
|
if (names) params.names = '1';
|
||||||
|
const qs = new URLSearchParams(params).toString();
|
||||||
return this.request('GET', `/extension/probe?${qs}`);
|
return this.request('GET', `/extension/probe?${qs}`);
|
||||||
}
|
}
|
||||||
|
// #4491: who posts in the Discord channel `url` shows, lately, with the
|
||||||
|
// likely creator marked; and setting the list on the source that follows it.
|
||||||
|
getDiscordPosters(url) {
|
||||||
|
const qs = new URLSearchParams({ url }).toString();
|
||||||
|
return this.request('GET', `/extension/discord/posters?${qs}`);
|
||||||
|
}
|
||||||
|
setDiscordPosters(url, authors) {
|
||||||
|
return this.request('POST', '/extension/discord/posters', { url, authors });
|
||||||
|
}
|
||||||
// Latest published extension version on this instance — drives the in-app
|
// Latest published extension version on this instance — drives the in-app
|
||||||
// update prompt. Public endpoint (no key needed, but request() sends it
|
// update prompt. Public endpoint (no key needed, but request() sends it
|
||||||
// harmlessly). Returns {version, xpi_url, latest_url, sha256}.
|
// harmlessly). Returns {version, xpi_url, latest_url, sha256}.
|
||||||
|
|||||||
@@ -0,0 +1,192 @@
|
|||||||
|
/**
|
||||||
|
* The content script's decisions, kept apart from its DOM so the specs can
|
||||||
|
* load them (test/chip.spec.js): which state the chip shows, what it says,
|
||||||
|
* and what the Add panel starts out proposing and finally sends.
|
||||||
|
*
|
||||||
|
* `probe` is /api/extension/probe's answer; `platformName` is the display
|
||||||
|
* name (PLATFORMS[key].name), passed in so this file needs no other lib.
|
||||||
|
*/
|
||||||
|
|
||||||
|
function chipState(probe) {
|
||||||
|
if (!probe || probe.error) return 'new';
|
||||||
|
return ({ source_match: 'source-match', artist_match: 'artist-match', new: 'new' })[probe.state] || 'new';
|
||||||
|
}
|
||||||
|
|
||||||
|
function chipLabel(probe, platformName) {
|
||||||
|
if (!probe || probe.error) return '+ Add to FabledCurator';
|
||||||
|
const artist = probe.artist?.name || 'artist';
|
||||||
|
if (probe.platform === 'discord') {
|
||||||
|
const d = probe.discord || {};
|
||||||
|
if (probe.state === 'source_match') {
|
||||||
|
return probe.covered_by_server
|
||||||
|
? `✓ Whole server in FabledCurator · ${artist}`
|
||||||
|
: `✓ In FabledCurator · ${artist}`;
|
||||||
|
}
|
||||||
|
// Every other Discord state opens the panel: the artist is always chosen.
|
||||||
|
return d.channel_id ? `+ Add ${channelLabel(d)} to FabledCurator` : '+ Add server to FabledCurator';
|
||||||
|
}
|
||||||
|
switch (probe.state) {
|
||||||
|
case 'source_match':
|
||||||
|
return `✓ In FabledCurator · ${platformName}`;
|
||||||
|
case 'artist_match':
|
||||||
|
return `+ Add ${platformName} source to ${artist}`;
|
||||||
|
default:
|
||||||
|
return '+ Add to FabledCurator';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `#name` when the probe could read it, else a neutral "this channel". */
|
||||||
|
function channelLabel(d) {
|
||||||
|
return d.channel_name ? `#${d.channel_name}` : 'this channel';
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `name` when the probe could read it, else "this server". */
|
||||||
|
function serverLabel(d) {
|
||||||
|
return d.server_name || 'this server';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What the Add panel opens with, on any platform.
|
||||||
|
*
|
||||||
|
* Discord: the channel is the default scope when there is one — a server
|
||||||
|
* source walks every channel the token can read, which is rarely what a
|
||||||
|
* single art channel wants. The artist is the probe's suggestion (the owner
|
||||||
|
* of another source on this server), else a new one named after the server.
|
||||||
|
*
|
||||||
|
* Patreon / SubscribeStar: the page URL is the source. The artist is the one
|
||||||
|
* whose slug the URL already names (artist_match), else a new one under the
|
||||||
|
* creator's display name — `probe.display_name`, from the probe the panel
|
||||||
|
* makes with names=1 — falling back to the URL handle, which the server
|
||||||
|
* resolves on its own when it is left untouched (`nameIsHandle`).
|
||||||
|
*/
|
||||||
|
function panelDefaults(probe, pageUrl) {
|
||||||
|
const d = probe?.discord || {};
|
||||||
|
const suggested = probe?.state === 'artist_match' && probe.artist ? probe.artist : null;
|
||||||
|
const discord = probe?.platform === 'discord';
|
||||||
|
const shown = probe?.display_name || null;
|
||||||
|
let artistName = '';
|
||||||
|
if (suggested) artistName = suggested.name;
|
||||||
|
else if (discord) artistName = d.server_name || '';
|
||||||
|
else artistName = shown || probe?.slug || '';
|
||||||
|
return {
|
||||||
|
platform: probe?.platform || null,
|
||||||
|
scope: discord ? (d.channel_id ? 'channel' : 'server') : 'page',
|
||||||
|
pageUrl: pageUrl || null,
|
||||||
|
channelUrl: d.channel_url || null,
|
||||||
|
serverUrl: d.server_url || null,
|
||||||
|
artist: suggested ? { id: suggested.id, name: suggested.name } : null,
|
||||||
|
artistName,
|
||||||
|
nameIsHandle: !discord && !suggested && !shown,
|
||||||
|
handle: probe?.slug || '',
|
||||||
|
displayName: shown,
|
||||||
|
// Patreon is canon: joining an artist known by another name takes the
|
||||||
|
// Patreon name, unless the operator unticks it.
|
||||||
|
adoptPlatformName: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The rename the panel offers, or null: only on Patreon (the canon name),
|
||||||
|
* only when joining an existing artist, only with a name actually read from
|
||||||
|
* Patreon, and only when it differs from what the artist is called now.
|
||||||
|
*/
|
||||||
|
function renameOffer(choice) {
|
||||||
|
if (choice.platform !== 'patreon' || !choice.artist || !choice.displayName) return null;
|
||||||
|
if (choice.artist.name === choice.displayName) return null;
|
||||||
|
return { from: choice.artist.name, to: choice.displayName };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The quick-add body for the panel's current choice. A picked artist goes by
|
||||||
|
* id — names can collide once slugified — and a typed name creates (or
|
||||||
|
* finds) that artist. The URL handle left untouched sends no name, so the
|
||||||
|
* server resolves the display name itself. null when there is nothing valid.
|
||||||
|
*/
|
||||||
|
function addRequest(choice) {
|
||||||
|
let url = choice.pageUrl;
|
||||||
|
if (choice.scope === 'server') url = choice.serverUrl;
|
||||||
|
else if (choice.scope === 'channel') url = choice.channelUrl;
|
||||||
|
if (!url) return null;
|
||||||
|
if (choice.artist && choice.artist.id != null && choice.artist.name === choice.artistName) {
|
||||||
|
const req = { url, artistId: choice.artist.id };
|
||||||
|
if (choice.adoptPlatformName && renameOffer(choice)) req.usePlatformName = true;
|
||||||
|
return req;
|
||||||
|
}
|
||||||
|
const name = (choice.artistName || '').trim();
|
||||||
|
if (!name) return null;
|
||||||
|
if (choice.nameIsHandle && name === choice.handle) return { url };
|
||||||
|
return { url, artistName: name };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An artist name reduced to what identifies it: lowercase letters and digits
|
||||||
|
* of any script, nothing else — so "Tamada Heijun", "tamada_heijun" and
|
||||||
|
* "TamadaHeijun" are one name. Mirrors the server's autocomplete (#429).
|
||||||
|
*/
|
||||||
|
function squashName(name) {
|
||||||
|
return String(name || '').toLowerCase().replace(/[^\p{L}\p{N}]/gu, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The search result that IS the query, spacing aside, else null. */
|
||||||
|
function exactArtistMatch(query, results) {
|
||||||
|
const q = squashName(query);
|
||||||
|
if (!q) return null;
|
||||||
|
return (results || []).find((a) => squashName(a.name) === q) || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Inline autofill: the first result whose name extends what was typed
|
||||||
|
* (case-insensitive), so the panel can fill in the rest and select it —
|
||||||
|
* typing on overwrites it, Tab or Enter accepts it. null when none does.
|
||||||
|
*/
|
||||||
|
function inlineCompletion(typed, results) {
|
||||||
|
const t = String(typed || '').toLowerCase();
|
||||||
|
if (!t) return null;
|
||||||
|
return (results || []).find((a) => a.name.length > t.length && a.name.toLowerCase().startsWith(t)) || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---- Discord poster picker (#4491) ----
|
||||||
|
// /api/extension/discord/posters lists who posted in the channel lately, with
|
||||||
|
// the creator the server is named for (or owned by) marked `suggested`. The
|
||||||
|
// picker ticks posters; the source keeps their ids, since a username can
|
||||||
|
// change on a whim and an id can't.
|
||||||
|
|
||||||
|
/** "Todding (@todding)" — the display name, then the handle when it differs. */
|
||||||
|
function posterLabel(p) {
|
||||||
|
const shown = p?.global_name || p?.username || p?.id || '';
|
||||||
|
const handle = p?.username && p.username !== shown ? ` (@${p.username})` : '';
|
||||||
|
return `${shown}${handle}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What the picker starts with: the source's current list when it has one,
|
||||||
|
* else the suggested creator(s). A set of ids. */
|
||||||
|
function initialPosterPick(data) {
|
||||||
|
const current = (data?.selected || []).map(String);
|
||||||
|
if (current.length) return new Set(current);
|
||||||
|
return new Set((data?.posters || []).filter((p) => p.suggested).map((p) => String(p.id)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The checklist's rows: who posted lately, then anyone already on the
|
||||||
|
* source's list who hasn't — by the name they were picked under — so they
|
||||||
|
* can still be unticked. */
|
||||||
|
function withSelected(data) {
|
||||||
|
const posters = [...(data?.posters || [])];
|
||||||
|
const shown = new Set(posters.map((p) => String(p.id)));
|
||||||
|
const labels = data?.selected_labels || {};
|
||||||
|
for (const id of (data?.selected || []).map(String)) {
|
||||||
|
if (shown.has(id)) continue;
|
||||||
|
posters.push({ id, global_name: labels[id] || null, username: null,
|
||||||
|
messages: 0, images: 0, reasons: ['on the list, not posting lately'] });
|
||||||
|
}
|
||||||
|
return posters;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The picked posters as the API takes them: [{id, name}], in list order.
|
||||||
|
* An id already on the source but no longer posting lately is kept. */
|
||||||
|
function postersRequest(posters, picked) {
|
||||||
|
const byId = new Map((posters || []).map((p) => [String(p.id), p]));
|
||||||
|
return [...picked].map((id) => ({
|
||||||
|
id,
|
||||||
|
name: byId.has(id) ? (byId.get(id).global_name || byId.get(id).username || id) : null,
|
||||||
|
}));
|
||||||
|
}
|
||||||
@@ -57,7 +57,8 @@ const PLATFORMS = {
|
|||||||
domains: ['.discord.com', 'discord.com'],
|
domains: ['.discord.com', 'discord.com'],
|
||||||
authType: 'token',
|
authType: 'token',
|
||||||
color: '#5865F2',
|
color: '#5865F2',
|
||||||
urlPattern: /^https?:\/\/(www\.)?discord\.com/,
|
// ptb/canary are Discord's beta clients — same channels, same token.
|
||||||
|
urlPattern: /^https?:\/\/((www|ptb|canary)\.)?discord\.com/,
|
||||||
note: 'Open Discord in browser to capture token',
|
note: 'Open Discord in browser to capture token',
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
@@ -80,8 +81,23 @@ const PLATFORM_ARTIST_PATTERNS = {
|
|||||||
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i,
|
patreon: /^https?:\/\/(www\.)?patreon\.com\/(?:cw\/|c\/)?(?!(?:home|search|messages|notifications|library|settings|posts)(?:[\/?#]|$))[^/?#]+/i,
|
||||||
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
subscribestar: /^https?:\/\/(www\.)?subscribestar\.(com|adult)\/(?!feed$|messages$|library$)[^/?#]+\/?$/i,
|
||||||
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
hentaifoundry: /^https?:\/\/(www\.)?hentai-foundry\.com\/user\/[^/?#]+/i,
|
||||||
|
// A Discord URL names a server or channel, not a creator: the backend's slug
|
||||||
|
// is `<server>[/<channel>]` and the Add panel asks which artist it belongs
|
||||||
|
// to. A message jump link still names its channel; DMs (@me) and thread
|
||||||
|
// links don't match. Mirrors extension_service._PLATFORM_PATTERNS.
|
||||||
|
discord: /^https?:\/\/(?:www\.|ptb\.|canary\.)?discord\.com\/channels\/\d+(?:\/\d+)?(?:\/\d+)?\/?(?:[?#].*)?$/i,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* `{serverId, channelId}` from a Discord channel/server URL the artist
|
||||||
|
* pattern accepts, else null. channelId is null for a whole-server URL.
|
||||||
|
*/
|
||||||
|
function parseDiscordUrl(url) {
|
||||||
|
if (!PLATFORM_ARTIST_PATTERNS.discord.test(url || '')) return null;
|
||||||
|
const m = /\/channels\/(\d+)(?:\/(\d+))?/.exec(url);
|
||||||
|
return m ? { serverId: m[1], channelId: m[2] || null } : null;
|
||||||
|
}
|
||||||
|
|
||||||
function getPlatformFromUrl(url) {
|
function getPlatformFromUrl(url) {
|
||||||
for (const [key, platform] of Object.entries(PLATFORMS)) {
|
for (const [key, platform] of Object.entries(PLATFORMS)) {
|
||||||
if (platform.urlPattern.test(url)) return key;
|
if (platform.urlPattern.test(url)) return key;
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
/**
|
||||||
|
* The popup's wording, kept apart from its DOM so the specs can load it
|
||||||
|
* (test/popup-format.spec.js). Classic script, like the rest of lib/.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One line of state for a source row, from /api/sources' fields, with the
|
||||||
|
* status class the popup colours it by ('ready' | 'error' | 'no-cookies' for
|
||||||
|
* a warning | '' for plain). The most actionable fact wins: an error before
|
||||||
|
* a running backfill, a backfill before the last-checked time.
|
||||||
|
*/
|
||||||
|
function sourceStatus(src, now = Date.now()) {
|
||||||
|
if (!src.enabled) return { text: 'Disabled', kind: '' };
|
||||||
|
if (src.last_error) {
|
||||||
|
const first = String(src.last_error).split('\n')[0].trim();
|
||||||
|
const short = first.length > 90 ? `${first.slice(0, 89)}…` : first;
|
||||||
|
return { text: `Error — ${short}`, kind: 'error' };
|
||||||
|
}
|
||||||
|
if (src.backfill_state === 'running') {
|
||||||
|
const n = src.backfill_chunks || 0;
|
||||||
|
return { text: n ? `Backfilling — ${n} chunk${n === 1 ? '' : 's'} done` : 'Backfill queued', kind: 'ready' };
|
||||||
|
}
|
||||||
|
if (src.backfill_state === 'stalled') return { text: 'Backfill stalled', kind: 'no-cookies' };
|
||||||
|
if (!src.last_checked_at) return { text: 'Not checked yet', kind: '' };
|
||||||
|
return { text: `Checked ${relativeTime(src.last_checked_at, now)}`, kind: '' };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Same buckets and wording as the web UI's canonical formatRelative
|
||||||
|
// (frontend/src/utils/date.js, snippet #3959) — the extension can't import it (classic
|
||||||
|
// scripts, separate package), so it mirrors it: "42s ago", "5m ago", "3h ago",
|
||||||
|
// "2d ago", and "Never" for a missing or unreadable time.
|
||||||
|
function relativeTime(iso, now = Date.now()) {
|
||||||
|
const t = iso ? Date.parse(iso) : NaN;
|
||||||
|
if (Number.isNaN(t)) return 'Never';
|
||||||
|
const abs = Math.abs(now - t) / 1000;
|
||||||
|
let body;
|
||||||
|
if (abs < 60) body = `${Math.floor(abs)}s`;
|
||||||
|
else if (abs < 3600) body = `${Math.floor(abs / 60)}m`;
|
||||||
|
else if (abs < 86400) body = `${Math.floor(abs / 3600)}h`;
|
||||||
|
else body = `${Math.floor(abs / 86400)}d`;
|
||||||
|
return `${body} ago`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The popup message after a Discord token export, from FC's verify of the
|
||||||
|
* stored token: {text, kind} with kind 'success' | 'warning' | 'error'.
|
||||||
|
* valid=null is "FC couldn't test it" (no Discord source yet, or a network
|
||||||
|
* hiccup) — a warning with FC's reason, never a failure.
|
||||||
|
*/
|
||||||
|
function tokenExportMessage(verify) {
|
||||||
|
if (!verify) return { text: 'Discord: token exported', kind: 'success' };
|
||||||
|
if (verify.valid === true) return { text: `Discord: token exported and verified ✓ — ${verify.reason}`, kind: 'success' };
|
||||||
|
if (verify.valid === false) return { text: `Discord: token exported, but FC's check failed — ${verify.reason}`, kind: 'error' };
|
||||||
|
return { text: `Discord: token exported (not verified — ${verify.reason})`, kind: 'warning' };
|
||||||
|
}
|
||||||
@@ -56,9 +56,10 @@
|
|||||||
"*://*.patreon.com/*",
|
"*://*.patreon.com/*",
|
||||||
"*://*.subscribestar.com/*",
|
"*://*.subscribestar.com/*",
|
||||||
"*://*.subscribestar.adult/*",
|
"*://*.subscribestar.adult/*",
|
||||||
"*://*.hentai-foundry.com/*"
|
"*://*.hentai-foundry.com/*",
|
||||||
|
"*://*.discord.com/*"
|
||||||
],
|
],
|
||||||
"js": ["lib/platforms.js", "content/content-script.js"],
|
"js": ["lib/platforms.js", "lib/chip.js", "content/content-script.js"],
|
||||||
"css": ["content/content-script.css"],
|
"css": ["content/content-script.css"],
|
||||||
"run_at": "document_idle"
|
"run_at": "document_idle"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -47,6 +47,7 @@
|
|||||||
</section>
|
</section>
|
||||||
|
|
||||||
<script src="../lib/platforms.js"></script>
|
<script src="../lib/platforms.js"></script>
|
||||||
|
<script src="../lib/popup-format.js"></script>
|
||||||
<script src="popup.js"></script>
|
<script src="popup.js"></script>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ function updateConnectionDot(connected) {
|
|||||||
async function checkForUpdate() {
|
async function checkForUpdate() {
|
||||||
try {
|
try {
|
||||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_UPDATE' });
|
const r = await browser.runtime.sendMessage({ type: 'CHECK_UPDATE' });
|
||||||
if (r && r.updateAvailable && r.xpiUrl) showUpdateBanner(r);
|
if (r && r.updateAvailable && r.installPageUrl) showUpdateBanner(r);
|
||||||
} catch { /* non-fatal */ }
|
} catch { /* non-fatal */ }
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -86,10 +86,14 @@ function showUpdateBanner(r) {
|
|||||||
// exactly as it did before the field existed.
|
// exactly as it did before the field existed.
|
||||||
const channel = r.channel ? ` (${r.channel})` : '';
|
const channel = r.channel ? ` (${r.channel})` : '';
|
||||||
document.getElementById('update-text').textContent =
|
document.getElementById('update-text').textContent =
|
||||||
`Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion})`;
|
`Update available${channel} — v${r.latestVersion} (installed v${r.currentVersion}). ` +
|
||||||
// Opening the signed XPI triggers Firefox's native install prompt.
|
'Opens FabledCurator — click “Install Firefox extension” there.';
|
||||||
|
// Opens FC's install card rather than the XPI: Firefox only installs an
|
||||||
|
// add-on from a user click on a web page, never from a tab an extension
|
||||||
|
// opened on the .xpi itself.
|
||||||
document.getElementById('update-btn').addEventListener('click', () => {
|
document.getElementById('update-btn').addEventListener('click', () => {
|
||||||
browser.tabs.create({ url: r.xpiUrl });
|
browser.tabs.create({ url: r.installPageUrl });
|
||||||
|
window.close();
|
||||||
});
|
});
|
||||||
document.getElementById('update-banner').classList.remove('hidden');
|
document.getElementById('update-banner').classList.remove('hidden');
|
||||||
}
|
}
|
||||||
@@ -159,7 +163,11 @@ async function exportPlatformCookies(key, card) {
|
|||||||
try {
|
try {
|
||||||
const r = await browser.runtime.sendMessage({ type: 'EXPORT_COOKIES', platform: key });
|
const r = await browser.runtime.sendMessage({ type: 'EXPORT_COOKIES', platform: key });
|
||||||
if (r.error) showError(r.error);
|
if (r.error) showError(r.error);
|
||||||
else {
|
else if (key === 'discord') {
|
||||||
|
const m = tokenExportMessage(r.verify);
|
||||||
|
showStatusMessage(m.text, m.kind);
|
||||||
|
await loadPlatformStatus();
|
||||||
|
} else {
|
||||||
const n = r.cookieCount ?? null;
|
const n = r.cookieCount ?? null;
|
||||||
const verifiedSuffix = r.verified ? ' (verified ✓)' : '';
|
const verifiedSuffix = r.verified ? ' (verified ✓)' : '';
|
||||||
const msg = n !== null
|
const msg = n !== null
|
||||||
@@ -205,7 +213,10 @@ async function loadSources() {
|
|||||||
c.appendChild(mutedNote('No sources yet.'));
|
c.appendChild(mutedNote('No sources yet.'));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
for (const src of r.sources) c.appendChild(createSourceRow(src));
|
// Grouped by artist so a creator's Patreon and Discord sit together.
|
||||||
|
const sorted = [...r.sources].sort((a, b) =>
|
||||||
|
(a.artist_name || '').localeCompare(b.artist_name || '') || a.id - b.id);
|
||||||
|
for (const src of sorted) c.appendChild(createSourceRow(src));
|
||||||
}
|
}
|
||||||
|
|
||||||
function createSourceRow(src) {
|
function createSourceRow(src) {
|
||||||
@@ -215,11 +226,16 @@ function createSourceRow(src) {
|
|||||||
info.className = 'info';
|
info.className = 'info';
|
||||||
const name = document.createElement('div');
|
const name = document.createElement('div');
|
||||||
name.className = 'name';
|
name.className = 'name';
|
||||||
name.textContent = `${src.platform} · #${src.id}`;
|
const platformName = PLATFORMS[src.platform]?.name || src.platform;
|
||||||
|
name.textContent = `${src.artist_name || `Source #${src.id}`} · ${platformName}`;
|
||||||
|
const state = sourceStatus(src);
|
||||||
|
const st = document.createElement('div');
|
||||||
|
st.className = `status ${state.kind}`;
|
||||||
|
st.textContent = state.text;
|
||||||
const url = document.createElement('div');
|
const url = document.createElement('div');
|
||||||
url.className = 'url';
|
url.className = 'url';
|
||||||
url.textContent = src.url;
|
url.textContent = src.url;
|
||||||
info.appendChild(name); info.appendChild(url);
|
info.appendChild(name); info.appendChild(st); info.appendChild(url);
|
||||||
const play = document.createElement('button');
|
const play = document.createElement('button');
|
||||||
play.className = 'play';
|
play.className = 'play';
|
||||||
play.textContent = '▶';
|
play.textContent = '▶';
|
||||||
@@ -229,7 +245,7 @@ function createSourceRow(src) {
|
|||||||
const r = await browser.runtime.sendMessage({ type: 'CHECK_SOURCE', sourceId: src.id });
|
const r = await browser.runtime.sendMessage({ type: 'CHECK_SOURCE', sourceId: src.id });
|
||||||
play.disabled = false;
|
play.disabled = false;
|
||||||
if (r.error) showError(r.error);
|
if (r.error) showError(r.error);
|
||||||
else showSuccess(`Triggered check for source #${src.id}`);
|
else showSuccess(`Check queued for ${src.artist_name || `source #${src.id}`} (${platformName})`);
|
||||||
});
|
});
|
||||||
row.appendChild(info); row.appendChild(play);
|
row.appendChild(info); row.appendChild(play);
|
||||||
return row;
|
return row;
|
||||||
|
|||||||
@@ -134,5 +134,42 @@
|
|||||||
{ "url": "https://www.hentai-foundry.com/pictures/popular", "why": "gallery listing, not a user" },
|
{ "url": "https://www.hentai-foundry.com/pictures/popular", "why": "gallery listing, not a user" },
|
||||||
{ "url": "https://www.hentai-foundry.com/", "why": "site root" }
|
{ "url": "https://www.hentai-foundry.com/", "why": "site root" }
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
|
||||||
|
"discord": {
|
||||||
|
"match": [
|
||||||
|
{
|
||||||
|
"url": "https://discord.com/channels/111111111111111111/222222222222222222",
|
||||||
|
"slug": "111111111111111111/222222222222222222",
|
||||||
|
"why": "a channel: the slug is server/channel -- a Discord URL names a place, not a creator, so the artist is chosen in the Add panel"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://discord.com/channels/111111111111111111",
|
||||||
|
"slug": "111111111111111111",
|
||||||
|
"why": "a whole server"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://discord.com/channels/111111111111111111/222222222222222222/333333333333333333",
|
||||||
|
"slug": "111111111111111111/222222222222222222",
|
||||||
|
"why": "a message jump link still names its channel"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://ptb.discord.com/channels/111111111111111111/222222222222222222",
|
||||||
|
"slug": "111111111111111111/222222222222222222",
|
||||||
|
"why": "the ptb and canary clients serve the same channels"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"url": "https://discord.com/channels/111111111111111111/222222222222222222/",
|
||||||
|
"slug": "111111111111111111/222222222222222222",
|
||||||
|
"why": "trailing slash is tolerated"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"no_match": [
|
||||||
|
{ "url": "https://discord.com/channels/@me", "why": "the DM list is not a source" },
|
||||||
|
{ "url": "https://discord.com/channels/@me/222222222222222222", "why": "a DM is not a source" },
|
||||||
|
{ "url": "https://discord.com/app", "why": "the app shell, no server open" },
|
||||||
|
{ "url": "https://discord.com/channels/111111111111111111/222222222222222222/threads/444444444444444444", "why": "thread links are left to the manual Add form" },
|
||||||
|
{ "url": "https://discord.com/servers/111111111111111111", "why": "a server-discovery page, not a channel" }
|
||||||
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,243 @@
|
|||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
import { loadLib } from './helpers/loadLib.js'
|
||||||
|
|
||||||
|
const { chipState, chipLabel, panelDefaults, addRequest, renameOffer } = loadLib('chip.js', [
|
||||||
|
'chipState',
|
||||||
|
'chipLabel',
|
||||||
|
'panelDefaults',
|
||||||
|
'addRequest',
|
||||||
|
'renameOffer'
|
||||||
|
])
|
||||||
|
|
||||||
|
const discord = (extra = {}) => ({
|
||||||
|
platform: 'discord',
|
||||||
|
slug: '111/222',
|
||||||
|
discord: {
|
||||||
|
server_id: '111',
|
||||||
|
channel_id: '222',
|
||||||
|
server_name: 'Studio',
|
||||||
|
channel_name: 'drops',
|
||||||
|
server_url: 'https://discord.com/channels/111',
|
||||||
|
channel_url: 'https://discord.com/channels/111/222'
|
||||||
|
},
|
||||||
|
...extra
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('chip state and label', () => {
|
||||||
|
it('keeps the one-click wording on creator platforms', () => {
|
||||||
|
expect(chipLabel({ state: 'new', platform: 'patreon' }, 'Patreon')).toBe('+ Add to FabledCurator')
|
||||||
|
expect(
|
||||||
|
chipLabel({ state: 'artist_match', platform: 'patreon', artist: { name: 'Atole' } }, 'Patreon')
|
||||||
|
).toBe('+ Add Patreon source to Atole')
|
||||||
|
expect(chipLabel({ state: 'source_match', platform: 'patreon' }, 'Patreon')).toBe(
|
||||||
|
'✓ In FabledCurator · Patreon'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('offers the channel by name on Discord, whatever the suggestion', () => {
|
||||||
|
expect(chipLabel(discord({ state: 'new' }), 'Discord')).toBe('+ Add #drops to FabledCurator')
|
||||||
|
expect(chipLabel(discord({ state: 'artist_match', artist: { name: 'A' } }), 'Discord')).toBe(
|
||||||
|
'+ Add #drops to FabledCurator'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('says "this channel" when the token could not read the name', () => {
|
||||||
|
const p = discord({ state: 'new' })
|
||||||
|
p.discord.channel_name = null
|
||||||
|
expect(chipLabel(p, 'Discord')).toBe('+ Add this channel to FabledCurator')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('says whose source a Discord channel already is, and when the server covers it', () => {
|
||||||
|
const artist = { name: 'Tamada', slug: 'tamada' }
|
||||||
|
expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: false }), 'Discord')).toBe(
|
||||||
|
'✓ In FabledCurator · Tamada'
|
||||||
|
)
|
||||||
|
expect(chipLabel(discord({ state: 'source_match', artist, covered_by_server: true }), 'Discord')).toBe(
|
||||||
|
'✓ Whole server in FabledCurator · Tamada'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('falls back to the generic add when the probe failed', () => {
|
||||||
|
expect(chipState({ error: 'x' })).toBe('new')
|
||||||
|
expect(chipLabel({ error: 'x' }, '')).toBe('+ Add to FabledCurator')
|
||||||
|
expect(chipState({ state: 'source_match' })).toBe('source-match')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('Add panel on Discord', () => {
|
||||||
|
it('opens on the channel with the suggested artist preselected', () => {
|
||||||
|
const d = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||||
|
expect(d.scope).toBe('channel')
|
||||||
|
expect(d.artist).toEqual({ id: 7, name: 'Tamada' })
|
||||||
|
expect(d.artistName).toBe('Tamada')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('proposes a new artist named after the server when nothing is suggested', () => {
|
||||||
|
const d = panelDefaults(discord({ state: 'new' }))
|
||||||
|
expect(d.artist).toBe(null)
|
||||||
|
expect(d.artistName).toBe('Studio')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('sends a picked artist by id', () => {
|
||||||
|
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||||
|
expect(addRequest(choice)).toEqual({
|
||||||
|
url: 'https://discord.com/channels/111/222',
|
||||||
|
artistId: 7
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('sends a typed name once the picked artist has been edited away', () => {
|
||||||
|
const choice = panelDefaults(discord({ state: 'artist_match', artist: { id: 7, name: 'Tamada' } }))
|
||||||
|
choice.artistName = 'Tamada Alt'
|
||||||
|
expect(addRequest(choice)).toEqual({
|
||||||
|
url: 'https://discord.com/channels/111/222',
|
||||||
|
artistName: 'Tamada Alt'
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('adds the whole server when the operator picks it', () => {
|
||||||
|
const choice = panelDefaults(discord({ state: 'new' }))
|
||||||
|
choice.scope = 'server'
|
||||||
|
expect(addRequest(choice).url).toBe('https://discord.com/channels/111')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('has nothing to send without an artist', () => {
|
||||||
|
const choice = panelDefaults(discord({ state: 'new' }))
|
||||||
|
choice.artistName = ' '
|
||||||
|
expect(addRequest(choice)).toBe(null)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
const { squashName, exactArtistMatch, inlineCompletion } = loadLib('chip.js', [
|
||||||
|
'squashName',
|
||||||
|
'exactArtistMatch',
|
||||||
|
'inlineCompletion'
|
||||||
|
])
|
||||||
|
|
||||||
|
describe('artist matching for the Add panel', () => {
|
||||||
|
const results = [
|
||||||
|
{ id: 1, name: 'TamadaHeijun' },
|
||||||
|
{ id: 2, name: 'Tamago' },
|
||||||
|
{ id: 3, name: 'Sabu Art' }
|
||||||
|
]
|
||||||
|
|
||||||
|
it('treats spacing, case and punctuation as the same name', () => {
|
||||||
|
expect(squashName('Tamada Heijun')).toBe('tamadaheijun')
|
||||||
|
expect(squashName('sabu_art!')).toBe('sabuart')
|
||||||
|
expect(squashName('玉田 平順')).toBe('玉田平順')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('finds the result that is the query, spacing aside', () => {
|
||||||
|
expect(exactArtistMatch('tamada heijun', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||||
|
expect(exactArtistMatch('Tama', results)).toBe(null)
|
||||||
|
expect(exactArtistMatch(' ', results)).toBe(null)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('autofills the first name that extends what was typed', () => {
|
||||||
|
expect(inlineCompletion('tamad', results)).toEqual({ id: 1, name: 'TamadaHeijun' })
|
||||||
|
expect(inlineCompletion('Sab', results).name).toBe('Sabu Art')
|
||||||
|
// Nothing to add once the name is complete, or when nothing extends it.
|
||||||
|
expect(inlineCompletion('Sabu Art', results)).toBe(null)
|
||||||
|
expect(inlineCompletion('heijun', results)).toBe(null)
|
||||||
|
expect(inlineCompletion('', results)).toBe(null)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('Add panel on Patreon and SubscribeStar', () => {
|
||||||
|
const PAGE = 'https://www.patreon.com/cw/tamadaheijun'
|
||||||
|
const patreon = (extra = {}) => ({ platform: 'patreon', slug: 'tamadaheijun', ...extra })
|
||||||
|
|
||||||
|
it('opens on the Patreon display name as a new artist', () => {
|
||||||
|
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||||
|
expect(c.scope).toBe('page')
|
||||||
|
expect(c.artistName).toBe('Tamada Heijun')
|
||||||
|
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Tamada Heijun' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('leaves an untouched URL handle to the server to resolve', () => {
|
||||||
|
const c = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||||
|
expect(c.artistName).toBe('tamadaheijun')
|
||||||
|
expect(addRequest(c)).toEqual({ url: PAGE })
|
||||||
|
c.artistName = 'Someone Else'
|
||||||
|
expect(addRequest(c)).toEqual({ url: PAGE, artistName: 'Someone Else' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('offers the Patreon name when joining an artist known by another name', () => {
|
||||||
|
const c = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||||
|
c.artist = { id: 4, name: 'tamada' }
|
||||||
|
c.artistName = 'tamada'
|
||||||
|
expect(renameOffer(c)).toEqual({ from: 'tamada', to: 'Tamada Heijun' })
|
||||||
|
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4, usePlatformName: true })
|
||||||
|
c.adoptPlatformName = false
|
||||||
|
expect(addRequest(c)).toEqual({ url: PAGE, artistId: 4 })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('offers no rename when the names agree or the name was not read', () => {
|
||||||
|
const same = panelDefaults(patreon({ state: 'new', display_name: 'Tamada Heijun' }), PAGE)
|
||||||
|
same.artist = { id: 4, name: 'Tamada Heijun' }
|
||||||
|
expect(renameOffer(same)).toBe(null)
|
||||||
|
const unread = panelDefaults(patreon({ state: 'new' }), PAGE)
|
||||||
|
unread.artist = { id: 4, name: 'tamada' }
|
||||||
|
expect(renameOffer(unread)).toBe(null)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('never renames from SubscribeStar or Discord — Patreon is the canon', () => {
|
||||||
|
const ss = panelDefaults(
|
||||||
|
{ platform: 'subscribestar', slug: 'tamada', state: 'new', display_name: 'SS Tamada' },
|
||||||
|
'https://subscribestar.adult/tamada'
|
||||||
|
)
|
||||||
|
ss.artist = { id: 4, name: 'Tamada Heijun' }
|
||||||
|
ss.artistName = 'Tamada Heijun'
|
||||||
|
expect(renameOffer(ss)).toBe(null)
|
||||||
|
expect(addRequest(ss)).toEqual({ url: 'https://subscribestar.adult/tamada', artistId: 4 })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('preselects the artist the URL already names', () => {
|
||||||
|
const c = panelDefaults(patreon({ state: 'artist_match', artist: { id: 9, name: 'Tamada' } }), PAGE)
|
||||||
|
expect(c.artist).toEqual({ id: 9, name: 'Tamada' })
|
||||||
|
expect(c.artistName).toBe('Tamada')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
const { posterLabel, initialPosterPick, postersRequest, withSelected } = loadLib('chip.js', [
|
||||||
|
'posterLabel', 'initialPosterPick', 'postersRequest', 'withSelected',
|
||||||
|
])
|
||||||
|
|
||||||
|
describe('the poster checklist rows', () => {
|
||||||
|
it('adds someone on the list who has not posted lately, by their saved name', () => {
|
||||||
|
const rows = withSelected({
|
||||||
|
posters: [{ id: '7', username: 'todding', global_name: 'Todding' }],
|
||||||
|
selected: ['7', '42'],
|
||||||
|
selected_labels: { 42: 'Old Name' },
|
||||||
|
})
|
||||||
|
expect(rows.map((r) => r.id)).toEqual(['7', '42'])
|
||||||
|
expect(posterLabel(rows[1])).toBe('Old Name')
|
||||||
|
expect(rows[1].reasons).toEqual(['on the list, not posting lately'])
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('the Discord poster picker (#4491)', () => {
|
||||||
|
const posters = [
|
||||||
|
{ id: '7', username: 'todding', global_name: 'Todding', suggested: true },
|
||||||
|
{ id: '8', username: 'jakeboii', global_name: 'Jake Boii', suggested: false },
|
||||||
|
{ id: '9', username: 'plain', global_name: null, suggested: false },
|
||||||
|
]
|
||||||
|
|
||||||
|
it('labels a poster by display name, then handle', () => {
|
||||||
|
expect(posterLabel(posters[0])).toBe('Todding (@todding)')
|
||||||
|
expect(posterLabel(posters[2])).toBe('plain')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('starts from the source list when it has one, else the suggestion', () => {
|
||||||
|
expect([...initialPosterPick({ posters, selected: [] })]).toEqual(['7'])
|
||||||
|
expect([...initialPosterPick({ posters, selected: ['8'] })]).toEqual(['8'])
|
||||||
|
expect([...initialPosterPick({ posters: [] })]).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
|
it('sends ids with the names they were picked under, keeping unknown ids', () => {
|
||||||
|
expect(postersRequest(posters, new Set(['7', '42']))).toEqual([
|
||||||
|
{ id: '7', name: 'Todding' },
|
||||||
|
{ id: '42', name: null },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -7,10 +7,14 @@ import { loadLib } from './helpers/loadLib.js'
|
|||||||
const EXT_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..')
|
const EXT_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), '..')
|
||||||
const manifest = JSON.parse(readFileSync(path.join(EXT_DIR, 'manifest.json'), 'utf8'))
|
const manifest = JSON.parse(readFileSync(path.join(EXT_DIR, 'manifest.json'), 'utf8'))
|
||||||
|
|
||||||
const { getPlatformFromUrl, isArtistPage, PLATFORMS, PLATFORM_ARTIST_PATTERNS } = loadLib(
|
const { getPlatformFromUrl, isArtistPage, parseDiscordUrl, PLATFORMS, PLATFORM_ARTIST_PATTERNS } =
|
||||||
'platforms.js',
|
loadLib('platforms.js', [
|
||||||
['getPlatformFromUrl', 'isArtistPage', 'PLATFORMS', 'PLATFORM_ARTIST_PATTERNS']
|
'getPlatformFromUrl',
|
||||||
)
|
'isArtistPage',
|
||||||
|
'parseDiscordUrl',
|
||||||
|
'PLATFORMS',
|
||||||
|
'PLATFORM_ARTIST_PATTERNS'
|
||||||
|
])
|
||||||
|
|
||||||
describe('getPlatformFromUrl', () => {
|
describe('getPlatformFromUrl', () => {
|
||||||
it('identifies each platform from a domain URL', () => {
|
it('identifies each platform from a domain URL', () => {
|
||||||
@@ -88,8 +92,11 @@ describe('isArtistPage', () => {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('returns false for a platform with no artist pattern (discord)', () => {
|
it('matches Discord server and channel pages, not DMs (milestone 429)', () => {
|
||||||
|
expect(isArtistPage('https://discord.com/channels/111/222', 'discord')).toBe(true)
|
||||||
|
expect(isArtistPage('https://discord.com/channels/111', 'discord')).toBe(true)
|
||||||
expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false)
|
expect(isArtistPage('https://discord.com/channels/@me', 'discord')).toBe(false)
|
||||||
|
expect(isArtistPage('https://discord.com/channels/@me/222', 'discord')).toBe(false)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('returns false for an unknown platform key', () => {
|
it('returns false for an unknown platform key', () => {
|
||||||
@@ -100,8 +107,8 @@ describe('isArtistPage', () => {
|
|||||||
describe('platform table integrity', () => {
|
describe('platform table integrity', () => {
|
||||||
it('gives every artist pattern a corresponding platform entry', () => {
|
it('gives every artist pattern a corresponding platform entry', () => {
|
||||||
// A pattern keyed to a platform that no longer exists is dead code that
|
// A pattern keyed to a platform that no longer exists is dead code that
|
||||||
// silently never fires; the reverse (a platform with no pattern) is the
|
// silently never fires; the reverse (a platform with no pattern) would be
|
||||||
// legitimate discord case, so only this direction is an error.
|
// a platform the button never offers, a product choice rather than an error.
|
||||||
for (const key of Object.keys(PLATFORM_ARTIST_PATTERNS)) {
|
for (const key of Object.keys(PLATFORM_ARTIST_PATTERNS)) {
|
||||||
expect(Object.keys(PLATFORMS)).toContain(key)
|
expect(Object.keys(PLATFORMS)).toContain(key)
|
||||||
}
|
}
|
||||||
@@ -125,7 +132,8 @@ describe('platform table integrity', () => {
|
|||||||
const samples = {
|
const samples = {
|
||||||
patreon: 'https://www.patreon.com/cw/Atole',
|
patreon: 'https://www.patreon.com/cw/Atole',
|
||||||
subscribestar: 'https://subscribestar.adult/someone',
|
subscribestar: 'https://subscribestar.adult/someone',
|
||||||
hentaifoundry: 'https://www.hentai-foundry.com/user/someone'
|
hentaifoundry: 'https://www.hentai-foundry.com/user/someone',
|
||||||
|
discord: 'https://ptb.discord.com/channels/111/222'
|
||||||
}
|
}
|
||||||
for (const [key, url] of Object.entries(samples)) {
|
for (const [key, url] of Object.entries(samples)) {
|
||||||
expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true)
|
expect(isArtistPage(url, key), `${key} artist pattern`).toBe(true)
|
||||||
@@ -152,7 +160,7 @@ describe('manifest.json agrees with the platform table', () => {
|
|||||||
)
|
)
|
||||||
expect(owner, `no platform claims content-script match "${m}"`).toBeTruthy()
|
expect(owner, `no platform claims content-script match "${m}"`).toBeTruthy()
|
||||||
// The content script exists to draw the Add-as-source button, so a
|
// The content script exists to draw the Add-as-source button, so a
|
||||||
// platform with no artist pattern (discord) has no business here.
|
// platform with no artist pattern has no business here.
|
||||||
expect(
|
expect(
|
||||||
PLATFORM_ARTIST_PATTERNS[owner[0]],
|
PLATFORM_ARTIST_PATTERNS[owner[0]],
|
||||||
`"${m}" injects for ${owner[0]}, which has no artist pattern`
|
`"${m}" injects for ${owner[0]}, which has no artist pattern`
|
||||||
@@ -232,9 +240,8 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => {
|
|||||||
|
|
||||||
it('has samples for every platform that has an artist pattern', () => {
|
it('has samples for every platform that has an artist pattern', () => {
|
||||||
// The guard's own coverage check: without it, deleting a platform's
|
// The guard's own coverage check: without it, deleting a platform's
|
||||||
// samples would make this block pass by testing less. Discord is
|
// samples would make this block pass by testing less. Discord joined at
|
||||||
// deliberately in neither — it is channel-based, with no creator page to
|
// milestone 429 — its slug is server/channel and the artist is chosen.
|
||||||
// put a button on, so it has no artist pattern on either side.
|
|
||||||
expect(Object.keys(samples).sort()).toEqual(Object.keys(PLATFORM_ARTIST_PATTERNS).sort())
|
expect(Object.keys(samples).sort()).toEqual(Object.keys(PLATFORM_ARTIST_PATTERNS).sort())
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -247,3 +254,26 @@ describe('the JS<->Py artist-pattern mirror (#3093)', () => {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('parseDiscordUrl', () => {
|
||||||
|
it('reads the server and channel ids the Add panel offers', () => {
|
||||||
|
expect(parseDiscordUrl('https://discord.com/channels/111/222')).toEqual({
|
||||||
|
serverId: '111',
|
||||||
|
channelId: '222'
|
||||||
|
})
|
||||||
|
expect(parseDiscordUrl('https://discord.com/channels/111/222/333')).toEqual({
|
||||||
|
serverId: '111',
|
||||||
|
channelId: '222'
|
||||||
|
})
|
||||||
|
expect(parseDiscordUrl('https://discord.com/channels/111')).toEqual({
|
||||||
|
serverId: '111',
|
||||||
|
channelId: null
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns null for anything the artist pattern rejects', () => {
|
||||||
|
expect(parseDiscordUrl('https://discord.com/channels/@me/222')).toBe(null)
|
||||||
|
expect(parseDiscordUrl('https://discord.com/app')).toBe(null)
|
||||||
|
expect(parseDiscordUrl('')).toBe(null)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
import { describe, it, expect } from 'vitest'
|
||||||
|
import { loadLib } from './helpers/loadLib.js'
|
||||||
|
|
||||||
|
const { sourceStatus, relativeTime, tokenExportMessage } = loadLib('popup-format.js', [
|
||||||
|
'sourceStatus',
|
||||||
|
'relativeTime',
|
||||||
|
'tokenExportMessage'
|
||||||
|
])
|
||||||
|
|
||||||
|
const NOW = Date.parse('2026-09-25T12:00:00Z')
|
||||||
|
const src = (extra = {}) => ({ enabled: true, last_error: null, backfill_state: null, ...extra })
|
||||||
|
|
||||||
|
describe('sourceStatus', () => {
|
||||||
|
it('puts an error first, trimmed to its first line', () => {
|
||||||
|
const s = sourceStatus(src({ last_error: 'Discord rejected the token\nstack…', backfill_state: 'running' }), NOW)
|
||||||
|
expect(s).toEqual({ text: 'Error — Discord rejected the token', kind: 'error' })
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows a running backfill and its progress', () => {
|
||||||
|
expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 0 }), NOW).text).toBe('Backfill queued')
|
||||||
|
expect(sourceStatus(src({ backfill_state: 'running', backfill_chunks: 3 }), NOW).text).toBe(
|
||||||
|
'Backfilling — 3 chunks done'
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
it('says when a source was last checked, or that it never was', () => {
|
||||||
|
expect(sourceStatus(src({ last_checked_at: '2026-09-25T11:55:00Z' }), NOW).text).toBe('Checked 5m ago')
|
||||||
|
expect(sourceStatus(src({ last_checked_at: null }), NOW).text).toBe('Not checked yet')
|
||||||
|
})
|
||||||
|
|
||||||
|
it('shows a disabled source as disabled, whatever else it carries', () => {
|
||||||
|
expect(sourceStatus(src({ enabled: false, last_error: 'x' }), NOW).text).toBe('Disabled')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('relativeTime', () => {
|
||||||
|
it('uses the web UI formatRelative buckets', () => {
|
||||||
|
expect(relativeTime('2026-09-25T11:59:18Z', NOW)).toBe('42s ago')
|
||||||
|
expect(relativeTime('2026-09-25T09:00:00Z', NOW)).toBe('3h ago')
|
||||||
|
expect(relativeTime('2026-09-23T12:00:00Z', NOW)).toBe('2d ago')
|
||||||
|
expect(relativeTime('garbage', NOW)).toBe('Never')
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
describe('tokenExportMessage', () => {
|
||||||
|
it('distinguishes verified, rejected and untested tokens', () => {
|
||||||
|
expect(tokenExportMessage({ valid: true, reason: 'Token valid (me)' }).kind).toBe('success')
|
||||||
|
expect(tokenExportMessage({ valid: false, reason: 'Discord rejected the token' }).kind).toBe('error')
|
||||||
|
const untested = tokenExportMessage({ valid: null, reason: 'No enabled source' })
|
||||||
|
expect(untested.kind).toBe('warning')
|
||||||
|
expect(untested.text).toContain('No enabled source')
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -153,6 +153,14 @@ const health = computed(() => {
|
|||||||
label: (worst?.detail || 'A part has stopped') + suffix,
|
label: (worst?.detail || 'A part has stopped') + suffix,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Running but slow — a GPU agent that fell back to the CPU (#4410). Worth
|
||||||
|
// the amber dot: nothing else anywhere says so.
|
||||||
|
if (overall === 'degraded') {
|
||||||
|
return {
|
||||||
|
icon: 'mdi-speedometer-slow', color: 'warning',
|
||||||
|
label: (worst?.detail || 'A part is running degraded') + suffix,
|
||||||
|
}
|
||||||
|
}
|
||||||
if (overall === 'stale') {
|
if (overall === 'stale') {
|
||||||
return {
|
return {
|
||||||
icon: 'mdi-alert', color: 'warning',
|
icon: 'mdi-alert', color: 'warning',
|
||||||
|
|||||||
@@ -103,9 +103,9 @@ const stats = computed(() => {
|
|||||||
padding: 0.5rem 1rem;
|
padding: 0.5rem 1rem;
|
||||||
background: linear-gradient(
|
background: linear-gradient(
|
||||||
to bottom,
|
to bottom,
|
||||||
rgba(20, 23, 26, 0.92) 0%,
|
rgba(var(--fc-chrome-rgb), 0.92) 0%,
|
||||||
rgba(20, 23, 26, 0.65) 60%,
|
rgba(var(--fc-chrome-rgb), 0.65) 60%,
|
||||||
rgba(20, 23, 26, 0) 100%
|
rgba(var(--fc-chrome-rgb), 0) 100%
|
||||||
);
|
);
|
||||||
backdrop-filter: blur(2px);
|
backdrop-filter: blur(2px);
|
||||||
-webkit-backdrop-filter: blur(2px);
|
-webkit-backdrop-filter: blur(2px);
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ import MaintenanceTile from '../common/MaintenanceTile.vue'
|
|||||||
import { useCleanupStore } from '../../stores/cleanup.js'
|
import { useCleanupStore } from '../../stores/cleanup.js'
|
||||||
|
|
||||||
const store = useCleanupStore()
|
const store = useCleanupStore()
|
||||||
const threshold = ref(0.95)
|
const threshold = ref(0.995)
|
||||||
const tolerance = ref(30)
|
const tolerance = ref(30)
|
||||||
const audit = ref(null)
|
const audit = ref(null)
|
||||||
const busy = ref(false)
|
const busy = ref(false)
|
||||||
|
|||||||
@@ -329,7 +329,7 @@ function pushFilter(mutate) {
|
|||||||
opaque than the bar/nav so the controls stay legible. */
|
opaque than the bar/nav so the controls stay legible. */
|
||||||
.fc-filterbar-wrap :deep(.v-field),
|
.fc-filterbar-wrap :deep(.v-field),
|
||||||
.fc-filterbar-wrap :deep(.v-btn-group) {
|
.fc-filterbar-wrap :deep(.v-btn-group) {
|
||||||
background-color: rgba(20, 23, 26, 0.72);
|
background-color: rgba(var(--v-theme-background), 0.72);
|
||||||
}
|
}
|
||||||
/* Media toggle (All / Images / Videos) as ONE cohesive segmented control.
|
/* Media toggle (All / Images / Videos) as ONE cohesive segmented control.
|
||||||
FC's global VBtn { rounded: 'pill' } default made Vuetify 4 pill-round each
|
FC's global VBtn { rounded: 'pill' } default made Vuetify 4 pill-round each
|
||||||
|
|||||||
@@ -2,14 +2,14 @@
|
|||||||
<!-- System-tag auto-applies (chrome hides / process WIP tags) that ALSO looked
|
<!-- System-tag auto-applies (chrome hides / process WIP tags) that ALSO looked
|
||||||
like real content — surfaced PROACTIVELY atop the gallery whenever there's
|
like real content — surfaced PROACTIVELY atop the gallery whenever there's
|
||||||
something to review (NOT gated on the Show-hidden toggle, so misfires can't
|
something to review (NOT gated on the Show-hidden toggle, so misfires can't
|
||||||
go unnoticed), most-concerning first, with keep / remove (#141, #1464).
|
go unnoticed), most-concerning first (#141, #1464). Each card asks one
|
||||||
|
question — is this image a <tag>? — and its buttons answer it (#4424).
|
||||||
Renders nothing when there's nothing to review. -->
|
Renders nothing when there's nothing to review. -->
|
||||||
<section v-if="items.length" class="fc-review" aria-label="Auto-tagged images to review">
|
<section v-if="items.length" class="fc-review" aria-label="Auto-tagged images to review">
|
||||||
<div class="fc-review__head">
|
<div class="fc-review__head">
|
||||||
<v-icon size="18" color="warning">mdi-alert-outline</v-icon>
|
<v-icon size="18" color="warning">mdi-alert-outline</v-icon>
|
||||||
<span class="fc-review__title">
|
<span class="fc-review__title">
|
||||||
{{ items.length }} auto-tagged {{ items.length === 1 ? 'image' : 'images' }}
|
{{ items.length }} {{ items.length === 1 ? 'auto-tag' : 'auto-tags' }} to check
|
||||||
may be real content — review
|
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="fc-review__cards">
|
<div class="fc-review__cards">
|
||||||
@@ -21,22 +21,23 @@
|
|||||||
class="fc-review-card__thumb" loading="lazy"
|
class="fc-review-card__thumb" loading="lazy"
|
||||||
>
|
>
|
||||||
<div class="fc-review-card__body">
|
<div class="fc-review-card__body">
|
||||||
|
<div class="fc-review-card__question">{{ question(it) }}</div>
|
||||||
<div
|
<div
|
||||||
class="fc-review-card__conflict"
|
class="fc-review-card__reason"
|
||||||
:title="`Scored ${Math.round(it.conflict_score * 100)}% on “${it.conflict_name || 'a content tag'}”`"
|
:title="reasonTitle(it)"
|
||||||
>
|
>
|
||||||
also looks like <strong>{{ it.conflict_name || 'content' }}</strong>
|
also {{ Math.round(it.conflict_score * 100) }}%
|
||||||
|
<strong>{{ it.conflict_name || 'content' }}</strong>
|
||||||
</div>
|
</div>
|
||||||
<div class="fc-review-card__tag">{{ tagLine(it) }}</div>
|
|
||||||
<div class="fc-review-card__acts">
|
<div class="fc-review-card__acts">
|
||||||
<button
|
<button
|
||||||
type="button" class="fc-review-btn fc-review-btn--keep"
|
type="button" class="fc-review-btn fc-review-btn--keep"
|
||||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')"
|
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'keep')"
|
||||||
>{{ keepLabel(it) }}</button>
|
>Is {{ withArticle(it) }}</button>
|
||||||
<button
|
<button
|
||||||
type="button" class="fc-review-btn fc-review-btn--unhide"
|
type="button" class="fc-review-btn fc-review-btn--unhide"
|
||||||
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')"
|
:disabled="busy.includes(keyOf(it))" @click="resolve(it, 'unhide')"
|
||||||
>{{ removeLabel(it) }}</button>
|
>Is not {{ withArticle(it) }}</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -55,11 +56,18 @@ const items = ref([])
|
|||||||
const busy = ref([])
|
const busy = ref([])
|
||||||
|
|
||||||
function keyOf(it) { return `${it.image_id}:${it.tag_id}` }
|
function keyOf(it) { return `${it.image_id}:${it.tag_id}` }
|
||||||
// Chrome flags hide the image (keep-hidden / un-hide); process flags leave it
|
// The card asks whether the image IS the auto-applied system tag, and the buttons
|
||||||
// visible and just tagged (keep-tag / remove-tag). Same endpoints, different words.
|
// answer that (operator, #4424: "is a <tag>" / "is not a <tag>"). "Is" keeps the
|
||||||
function tagLine(it) { return (it.mode === 'process' ? 'auto-tagged ' : 'hidden as ') + it.tag_name }
|
// tag ('keep'); "Is not" removes it, un-hiding a chrome image ('unhide'). The
|
||||||
function keepLabel(it) { return it.mode === 'process' ? 'Keep tag' : 'Keep hidden' }
|
// content tag it also scored on is the reason it was flagged, not the question.
|
||||||
function removeLabel(it) { return it.mode === 'process' ? 'Remove tag' : 'Un-hide' }
|
function noun(it) { return it.tag_name === 'wip' ? 'WIP' : it.tag_name }
|
||||||
|
function withArticle(it) { return (/^[aeiou]/i.test(noun(it)) ? 'an ' : 'a ') + noun(it) }
|
||||||
|
function question(it) { return `Is this ${withArticle(it)}?` }
|
||||||
|
function reasonTitle(it) {
|
||||||
|
const hidden = it.mode === 'process' ? '' : ' It is hidden from the gallery until you answer.'
|
||||||
|
return `Auto-tagged “${it.tag_name}”, but it also scored ${Math.round(it.conflict_score * 100)}% `
|
||||||
|
+ `on “${it.conflict_name || 'a content tag'}”, so it may be finished art.${hidden}`
|
||||||
|
}
|
||||||
|
|
||||||
async function load() {
|
async function load() {
|
||||||
// Fetched unconditionally on mount — the strip prompts for pending misfires
|
// Fetched unconditionally on mount — the strip prompts for pending misfires
|
||||||
@@ -77,14 +85,11 @@ async function resolve(it, action) {
|
|||||||
await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`)
|
await api.post(`/api/gallery/hidden-review/${it.image_id}/${it.tag_id}/${action}`)
|
||||||
items.value = items.value.filter((x) => keyOf(x) !== k)
|
items.value = items.value.filter((x) => keyOf(x) !== k)
|
||||||
if (action === 'unhide') {
|
if (action === 'unhide') {
|
||||||
const verb = it.mode === 'process' ? 'Removed' : 'Un-hidden'
|
const shown = it.mode === 'process' ? '' : ', back in the gallery'
|
||||||
toast({ text: `${verb} — “${it.tag_name}” removed; it'll train the head`, type: 'success' })
|
toast({ text: `Not ${withArticle(it)} — “${it.tag_name}” removed${shown}; the tagger learns from it`, type: 'success' })
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
toast({
|
toast({ text: `Could not save your answer: ${e.message}`, type: 'error' })
|
||||||
text: `Could not ${action === 'keep' ? 'keep hidden' : 'un-hide'}: ${e.message}`,
|
|
||||||
type: 'error',
|
|
||||||
})
|
|
||||||
} finally {
|
} finally {
|
||||||
busy.value = busy.value.filter((x) => x !== k)
|
busy.value = busy.value.filter((x) => x !== k)
|
||||||
}
|
}
|
||||||
@@ -112,7 +117,7 @@ onMounted(load)
|
|||||||
display: flex; gap: 10px; overflow-x: auto; padding-bottom: 4px;
|
display: flex; gap: 10px; overflow-x: auto; padding-bottom: 4px;
|
||||||
}
|
}
|
||||||
.fc-review-card {
|
.fc-review-card {
|
||||||
flex: 0 0 auto; width: 150px;
|
flex: 0 0 auto; width: 170px;
|
||||||
display: flex; flex-direction: column;
|
display: flex; flex-direction: column;
|
||||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||||
border-radius: 6px; overflow: hidden;
|
border-radius: 6px; overflow: hidden;
|
||||||
@@ -123,16 +128,16 @@ onMounted(load)
|
|||||||
background: rgb(var(--v-theme-surface-light));
|
background: rgb(var(--v-theme-surface-light));
|
||||||
}
|
}
|
||||||
.fc-review-card__body { padding: 6px 8px; }
|
.fc-review-card__body { padding: 6px 8px; }
|
||||||
.fc-review-card__conflict {
|
.fc-review-card__question {
|
||||||
font-size: 11px; color: rgb(var(--v-theme-on-surface));
|
font-size: 12px; font-weight: 600; color: rgb(var(--v-theme-on-surface));
|
||||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
}
|
}
|
||||||
.fc-review-card__conflict strong { color: rgb(var(--v-theme-warning)); }
|
.fc-review-card__reason {
|
||||||
.fc-review-card__tag {
|
|
||||||
font-size: 10px; color: rgb(var(--v-theme-on-surface-variant));
|
font-size: 10px; color: rgb(var(--v-theme-on-surface-variant));
|
||||||
margin: 1px 0 6px;
|
margin: 1px 0 6px;
|
||||||
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
overflow: hidden; text-overflow: ellipsis; white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
.fc-review-card__reason strong { color: rgb(var(--v-theme-warning)); font-weight: 600; }
|
||||||
.fc-review-card__acts { display: flex; gap: 4px; }
|
.fc-review-card__acts { display: flex; gap: 4px; }
|
||||||
.fc-review-btn {
|
.fc-review-btn {
|
||||||
flex: 1; font-size: 11px; padding: 3px 4px; border-radius: 4px;
|
flex: 1; font-size: 11px; padding: 3px 4px; border-radius: 4px;
|
||||||
|
|||||||
@@ -213,7 +213,7 @@ function nextFrame() {
|
|||||||
--fc-side-w: 320px;
|
--fc-side-w: 320px;
|
||||||
/* Obsidian haze (#14171A = 20,23,26) — same palette as TopNav,
|
/* Obsidian haze (#14171A = 20,23,26) — same palette as TopNav,
|
||||||
mid-opacity + blur so the page behind shows through faintly. */
|
mid-opacity + blur so the page behind shows through faintly. */
|
||||||
background: rgba(20, 23, 26, 0.65);
|
background: rgba(var(--v-theme-background), 0.65);
|
||||||
backdrop-filter: blur(8px);
|
backdrop-filter: blur(8px);
|
||||||
-webkit-backdrop-filter: blur(8px);
|
-webkit-backdrop-filter: blur(8px);
|
||||||
outline: none;
|
outline: none;
|
||||||
@@ -221,7 +221,7 @@ function nextFrame() {
|
|||||||
}
|
}
|
||||||
.fc-viewer__close, .fc-viewer__nav {
|
.fc-viewer__close, .fc-viewer__nav {
|
||||||
position: absolute; top: 50%;
|
position: absolute; top: 50%;
|
||||||
background: rgba(20, 23, 26, 0.7);
|
background: rgba(var(--v-theme-background), 0.7);
|
||||||
color: rgb(var(--v-theme-parchment, 232 228 216));
|
color: rgb(var(--v-theme-parchment, 232 228 216));
|
||||||
border: 1px solid rgb(var(--v-theme-surface-light));
|
border: 1px solid rgb(var(--v-theme-surface-light));
|
||||||
border-radius: 50%;
|
border-radius: 50%;
|
||||||
@@ -359,7 +359,7 @@ function nextFrame() {
|
|||||||
z-index: 1;
|
z-index: 1;
|
||||||
/* Opaque obsidian so the scrolling panel never bleeds through the
|
/* Opaque obsidian so the scrolling panel never bleeds through the
|
||||||
haze behind the pinned image. */
|
haze behind the pinned image. */
|
||||||
background: rgb(20, 23, 26);
|
background: rgb(var(--v-theme-background));
|
||||||
}
|
}
|
||||||
.fc-viewer__side {
|
.fc-viewer__side {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
|
|||||||
@@ -20,12 +20,13 @@
|
|||||||
class="fc-post-card__synthetic" :title="synthesisTitle"
|
class="fc-post-card__synthetic" :title="synthesisTitle"
|
||||||
>
|
>
|
||||||
<v-icon icon="mdi-auto-fix" size="x-small" start />
|
<v-icon icon="mdi-auto-fix" size="x-small" start />
|
||||||
grouped by FabledCurator
|
{{ synthesisChip }}
|
||||||
</v-chip>
|
</v-chip>
|
||||||
<RouterLink
|
<RouterLink
|
||||||
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
:to="{ name: 'artist', params: { slug: post.artist.slug } }"
|
||||||
class="fc-post-card__artist"
|
class="fc-post-card__artist"
|
||||||
>{{ post.artist.name }}</RouterLink>
|
>{{ post.artist.name }}</RouterLink>
|
||||||
|
<span v-if="post.channel" class="fc-post-card__meta">#{{ post.channel }}</span>
|
||||||
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
<span class="fc-post-card__date" :title="absoluteDate">{{ relativeDate }}</span>
|
||||||
<span v-if="totalImages" class="fc-post-card__meta">
|
<span v-if="totalImages" class="fc-post-card__meta">
|
||||||
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
· {{ totalImages }} image{{ totalImages === 1 ? '' : 's' }}
|
||||||
@@ -270,11 +271,18 @@ const plainTitle = computed(() => toPlainText(props.post.post_title))
|
|||||||
// post dict by hand, must degrade to "not synthetic" rather than throw.
|
// post dict by hand, must degrade to "not synthetic" rather than throw.
|
||||||
const synthesized = computed(() => Boolean(props.post.synthesized_by))
|
const synthesized = computed(() => Boolean(props.post.synthesized_by))
|
||||||
const messageCount = computed(() => props.post.synthesis?.message_count ?? 0)
|
const messageCount = computed(() => props.post.synthesis?.message_count ?? 0)
|
||||||
|
// A drop of one message stays a synthetic post (teaser matching only looks at
|
||||||
|
// drops), but there is nothing grouped in it — #4390: "Grouped from 1 Discord
|
||||||
|
// message" described a wrapper, not a grouping. Say what it is instead.
|
||||||
const synthesisTitle = computed(() => {
|
const synthesisTitle = computed(() => {
|
||||||
const n = messageCount.value
|
const n = messageCount.value
|
||||||
if (!n) return 'Grouped from Discord'
|
if (!n) return 'Grouped from Discord'
|
||||||
return `Grouped from ${n} Discord message${n === 1 ? '' : 's'}`
|
if (n === 1) return 'Discord message'
|
||||||
|
return `Grouped from ${n} Discord messages`
|
||||||
})
|
})
|
||||||
|
const synthesisChip = computed(() =>
|
||||||
|
messageCount.value === 1 ? 'from Discord' : 'grouped by FabledCurator'
|
||||||
|
)
|
||||||
|
|
||||||
const hero = computed(() => images.value[0])
|
const hero = computed(() => images.value[0])
|
||||||
|
|
||||||
|
|||||||
@@ -17,7 +17,7 @@
|
|||||||
<v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now
|
<v-icon start>mdi-folder-zip-outline</v-icon> Re-extract archives now
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
||||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
<QueueStatusBar queue="maintenance_long" queue-label="Long maintenance" />
|
||||||
</MaintenanceTile>
|
</MaintenanceTile>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
|
|||||||
@@ -49,16 +49,20 @@
|
|||||||
sometimes triggered nothing instead of the install dialog
|
sometimes triggered nothing instead of the install dialog
|
||||||
(operator-flagged 2026-05-26). No `download` attribute —
|
(operator-flagged 2026-05-26). No `download` attribute —
|
||||||
that would force a save dialog instead of install. -->
|
that would force a save dialog instead of install. -->
|
||||||
|
<!-- The VERSIONED file, not the `latest` alias: a versioned URL can
|
||||||
|
only ever be this build's bytes, so a browser cache can't hand
|
||||||
|
back the previous build (operator-flagged 2026-09-25: a cached
|
||||||
|
alias reinstalled the old version and the update never took). -->
|
||||||
<v-btn
|
<v-btn
|
||||||
v-if="isFirefox"
|
v-if="isFirefox"
|
||||||
color="accent" variant="flat" rounded="pill"
|
color="accent" variant="flat" rounded="pill"
|
||||||
prepend-icon="mdi-firefox"
|
prepend-icon="mdi-firefox"
|
||||||
:href="manifest.latest_url"
|
:href="manifest.xpi_url"
|
||||||
>Install Firefox extension</v-btn>
|
>Install Firefox extension</v-btn>
|
||||||
|
|
||||||
<v-btn
|
<v-btn
|
||||||
variant="outlined" rounded="pill"
|
variant="outlined" rounded="pill"
|
||||||
:href="manifest.latest_url" download
|
:href="manifest.xpi_url" download
|
||||||
prepend-icon="mdi-download"
|
prepend-icon="mdi-download"
|
||||||
>Download XPI</v-btn>
|
>Download XPI</v-btn>
|
||||||
|
|
||||||
|
|||||||
@@ -77,7 +77,7 @@
|
|||||||
<v-col cols="12" sm="6">
|
<v-col cols="12" sm="6">
|
||||||
<v-slider
|
<v-slider
|
||||||
v-model="local.single_color_threshold" label="Single-color threshold"
|
v-model="local.single_color_threshold" label="Single-color threshold"
|
||||||
min="0.5" max="1" step="0.05" thumb-label hide-details color="accent"
|
min="0.9" max="1" step="0.005" thumb-label hide-details color="accent"
|
||||||
:disabled="!local.skip_single_color" @end="save"
|
:disabled="!local.skip_single_color" @end="save"
|
||||||
/>
|
/>
|
||||||
</v-col>
|
</v-col>
|
||||||
@@ -103,17 +103,6 @@
|
|||||||
the Explore browse. Applies to new imports; run the scan below to catch
|
the Explore browse. Applies to new imports; run the scan below to catch
|
||||||
posts already in your library.
|
posts already in your library.
|
||||||
</div>
|
</div>
|
||||||
<v-switch
|
|
||||||
v-model="local.wip_soft_title_tagging_enabled"
|
|
||||||
label="Also tag “sketch” / “doodle” titles (lower precision)"
|
|
||||||
density="compact" hide-details color="primary" @change="save"
|
|
||||||
/>
|
|
||||||
<div class="fc-help mb-3">
|
|
||||||
Extends the above to softer cues (<code>sketch</code>, <code>doodle</code>,
|
|
||||||
<code>scribble</code>). These stay <strong>visible</strong> and never train
|
|
||||||
the tagging model — a daily audit flags any that actually look like finished
|
|
||||||
art for review. Off by default.
|
|
||||||
</div>
|
|
||||||
<v-btn
|
<v-btn
|
||||||
variant="tonal" color="primary" size="small"
|
variant="tonal" color="primary" size="small"
|
||||||
:loading="store.wipScanBusy" prepend-icon="mdi-magnify"
|
:loading="store.wipScanBusy" prepend-icon="mdi-magnify"
|
||||||
@@ -153,10 +142,9 @@ const PHASH_TICKS = { 0: 'Exact', 12: 'Strict', 24: 'Default', 48: 'Loose' }
|
|||||||
const local = reactive({
|
const local = reactive({
|
||||||
min_width: 0, min_height: 0,
|
min_width: 0, min_height: 0,
|
||||||
skip_transparent: false, transparency_threshold: 0.9,
|
skip_transparent: false, transparency_threshold: 0.9,
|
||||||
skip_single_color: false, single_color_threshold: 0.95,
|
skip_single_color: false, single_color_threshold: 0.995,
|
||||||
phash_threshold: 24,
|
phash_threshold: 24,
|
||||||
wip_title_tagging_enabled: true,
|
wip_title_tagging_enabled: true,
|
||||||
wip_soft_title_tagging_enabled: false,
|
|
||||||
})
|
})
|
||||||
|
|
||||||
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
watch(() => store.settings, (s) => { if (s) Object.assign(local, s) }, { immediate: true })
|
||||||
|
|||||||
@@ -18,7 +18,7 @@
|
|||||||
<v-icon start>mdi-file-remove-outline</v-icon> Repair missing-file records
|
<v-icon start>mdi-file-remove-outline</v-icon> Repair missing-file records
|
||||||
</v-btn>
|
</v-btn>
|
||||||
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
<span v-if="queued" class="ml-3 text-caption text-success">Queued ✓</span>
|
||||||
<QueueStatusBar queue="maintenance" queue-label="Maintenance" />
|
<QueueStatusBar queue="maintenance_long" queue-label="Long maintenance" />
|
||||||
</MaintenanceTile>
|
</MaintenanceTile>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ const props = defineProps({
|
|||||||
|
|
||||||
const QUEUE_NAMES = [
|
const QUEUE_NAMES = [
|
||||||
'default', 'import', 'thumbnail', 'ml',
|
'default', 'import', 'thumbnail', 'ml',
|
||||||
'download', 'scan', 'maintenance',
|
'download', 'scan', 'maintenance', 'maintenance_long',
|
||||||
]
|
]
|
||||||
|
|
||||||
function formatDepth(name) {
|
function formatDepth(name) {
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ function step(lane, delta) {
|
|||||||
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
|
.fc-sys__dot { width: 9px; height: 9px; border-radius: 50%; flex: 0 0 auto; }
|
||||||
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
|
.fc-sys__dot--ok { background: rgb(var(--v-theme-success)); }
|
||||||
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
|
.fc-sys__dot--stale { background: rgb(var(--v-theme-warning)); }
|
||||||
|
.fc-sys__dot--degraded { background: rgb(var(--v-theme-warning)); }
|
||||||
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
|
.fc-sys__dot--down { background: rgb(var(--v-theme-error)); }
|
||||||
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
|
.fc-sys__dot--unknown { background: rgb(var(--v-theme-on-surface) / 0.35); }
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,135 @@
|
|||||||
|
<template>
|
||||||
|
<!-- #4486. A Discord source's posts by people outside its "Only posts by"
|
||||||
|
list, taken before the list existed. Preview first, always: nothing is
|
||||||
|
deleted until the operator has seen the per-poster breakdown and typed
|
||||||
|
the token that preview returned. -->
|
||||||
|
<v-dialog :model-value="modelValue" max-width="560"
|
||||||
|
@update:model-value="$emit('update:modelValue', $event)">
|
||||||
|
<v-card>
|
||||||
|
<v-card-title>Remove posts from other posters</v-card-title>
|
||||||
|
<v-card-text>
|
||||||
|
<div class="text-caption mb-2">{{ source.display_name || source.url }}</div>
|
||||||
|
<v-progress-linear v-if="loading" indeterminate color="accent" class="mb-3" />
|
||||||
|
<v-alert v-else-if="error" type="warning" variant="tonal" density="compact">
|
||||||
|
{{ error }}
|
||||||
|
</v-alert>
|
||||||
|
<template v-else-if="result">
|
||||||
|
<v-alert type="success" variant="tonal" density="compact">
|
||||||
|
Removed {{ result.posts_deleted }} post(s), {{ result.images_deleted }} image(s),
|
||||||
|
{{ result.attachments_deleted }} attachment(s) and {{ result.drops_deleted }}
|
||||||
|
grouped post(s). Grouping rebuilds what remains on its next sweep.
|
||||||
|
</v-alert>
|
||||||
|
</template>
|
||||||
|
<template v-else-if="preview">
|
||||||
|
<p class="mb-2">
|
||||||
|
Keeping posts by <strong>{{ preview.authors.join(', ') }}</strong>.
|
||||||
|
</p>
|
||||||
|
<p v-if="!preview.posts" class="fc-dim">
|
||||||
|
Nothing to remove: every post with a known poster is by someone on the list.
|
||||||
|
</p>
|
||||||
|
<template v-else>
|
||||||
|
<v-table density="compact" class="mb-3">
|
||||||
|
<thead><tr><th>Poster</th><th class="text-right">Posts</th></tr></thead>
|
||||||
|
<tbody>
|
||||||
|
<tr v-for="p in preview.posters" :key="p.poster">
|
||||||
|
<td>{{ p.poster }}</td>
|
||||||
|
<td class="text-right">{{ p.posts }}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</v-table>
|
||||||
|
<p class="mb-1">
|
||||||
|
{{ preview.posts }} post(s), {{ preview.images }} image(s) found only on them,
|
||||||
|
{{ preview.attachments }} attachment(s).
|
||||||
|
</p>
|
||||||
|
<p v-if="preview.drops" class="mb-1">
|
||||||
|
{{ preview.drops }} grouped post(s) that include them are ungrouped; the
|
||||||
|
rest regroup on the next sweep.
|
||||||
|
</p>
|
||||||
|
<p class="fc-dim">
|
||||||
|
Images that also belong to a kept post stay. If a name you want to keep
|
||||||
|
appears above, add it to the source's "Only posts by" list first.
|
||||||
|
</p>
|
||||||
|
</template>
|
||||||
|
<p v-if="preview.unknown_posts" class="fc-dim mt-2">
|
||||||
|
{{ preview.unknown_posts }} older post(s) don't record who posted them and
|
||||||
|
are left alone.
|
||||||
|
</p>
|
||||||
|
</template>
|
||||||
|
</v-card-text>
|
||||||
|
<v-card-actions>
|
||||||
|
<v-spacer />
|
||||||
|
<v-btn variant="text" @click="$emit('update:modelValue', false)">
|
||||||
|
{{ result ? 'Close' : 'Cancel' }}
|
||||||
|
</v-btn>
|
||||||
|
<v-btn
|
||||||
|
v-if="preview && preview.posts && !result"
|
||||||
|
color="error" variant="flat" :loading="busy"
|
||||||
|
@click="confirmOpen = true"
|
||||||
|
>Remove {{ preview.posts }} post(s)</v-btn>
|
||||||
|
</v-card-actions>
|
||||||
|
</v-card>
|
||||||
|
<DestructiveConfirmModal
|
||||||
|
v-if="preview"
|
||||||
|
v-model="confirmOpen"
|
||||||
|
action="delete" kind="other posters' posts" tier="C"
|
||||||
|
:expected-token-override="preview.confirm_token"
|
||||||
|
:projected-counts="{ posts: preview.posts, images: preview.images,
|
||||||
|
attachments: preview.attachments, grouped: preview.drops }"
|
||||||
|
@confirm="onConfirm"
|
||||||
|
/>
|
||||||
|
</v-dialog>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<script setup>
|
||||||
|
import { ref, watch } from 'vue'
|
||||||
|
import { useSourcesStore } from '../../stores/sources.js'
|
||||||
|
import DestructiveConfirmModal from '../modal/DestructiveConfirmModal.vue'
|
||||||
|
|
||||||
|
const props = defineProps({
|
||||||
|
modelValue: { type: Boolean, default: false },
|
||||||
|
source: { type: Object, required: true },
|
||||||
|
})
|
||||||
|
defineEmits(['update:modelValue'])
|
||||||
|
const store = useSourcesStore()
|
||||||
|
|
||||||
|
const loading = ref(false)
|
||||||
|
const busy = ref(false)
|
||||||
|
const error = ref('')
|
||||||
|
const preview = ref(null)
|
||||||
|
const result = ref(null)
|
||||||
|
const confirmOpen = ref(false)
|
||||||
|
|
||||||
|
function message(e) {
|
||||||
|
return e?.body?.detail || e?.body?.error || e?.message || String(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
watch(() => props.modelValue, async (open) => {
|
||||||
|
if (!open) return
|
||||||
|
preview.value = null
|
||||||
|
result.value = null
|
||||||
|
error.value = ''
|
||||||
|
loading.value = true
|
||||||
|
try {
|
||||||
|
preview.value = await store.previewOtherPosters(props.source.id)
|
||||||
|
} catch (e) {
|
||||||
|
error.value = message(e)
|
||||||
|
} finally {
|
||||||
|
loading.value = false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
async function onConfirm(token) {
|
||||||
|
busy.value = true
|
||||||
|
try {
|
||||||
|
result.value = await store.removeOtherPosters(props.source.id, token)
|
||||||
|
} catch (e) {
|
||||||
|
error.value = message(e)
|
||||||
|
} finally {
|
||||||
|
busy.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<style scoped>
|
||||||
|
.fc-dim { color: rgb(var(--v-theme-on-surface-variant)); }
|
||||||
|
</style>
|
||||||
@@ -45,8 +45,20 @@
|
|||||||
>
|
>
|
||||||
<v-list-item-title>Recapture post text & links</v-list-item-title>
|
<v-list-item-title>Recapture post text & links</v-list-item-title>
|
||||||
<v-list-item-subtitle>
|
<v-list-item-subtitle>
|
||||||
Re-grab every post's body + external links and localize inline images
|
Re-grab every post's body + external links, localize inline images
|
||||||
already on disk — without re-downloading media
|
already on disk, and import any downloaded file that never reached the
|
||||||
|
library — without re-downloading media
|
||||||
|
</v-list-item-subtitle>
|
||||||
|
</v-list-item>
|
||||||
|
<v-list-item
|
||||||
|
v-if="hasPosterList && !running"
|
||||||
|
prepend-icon="mdi-account-remove-outline"
|
||||||
|
@click="cleanupOpen = true"
|
||||||
|
>
|
||||||
|
<v-list-item-title>Remove posts from other posters</v-list-item-title>
|
||||||
|
<v-list-item-subtitle>
|
||||||
|
Preview, then delete the posts taken from people outside this
|
||||||
|
source's "Only posts by" list
|
||||||
</v-list-item-subtitle>
|
</v-list-item-subtitle>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
<v-divider v-if="isNative && !running" />
|
<v-divider v-if="isNative && !running" />
|
||||||
@@ -58,12 +70,14 @@
|
|||||||
<v-list-item-title>Remove source</v-list-item-title>
|
<v-list-item-title>Remove source</v-list-item-title>
|
||||||
</v-list-item>
|
</v-list-item>
|
||||||
</KebabMenu>
|
</KebabMenu>
|
||||||
|
<PosterCleanupDialog v-if="hasPosterList" v-model="cleanupOpen" :source="source" />
|
||||||
</div>
|
</div>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<script setup>
|
<script setup>
|
||||||
import { computed } from 'vue'
|
import { computed, ref } from 'vue'
|
||||||
import KebabMenu from '../common/KebabMenu.vue'
|
import KebabMenu from '../common/KebabMenu.vue'
|
||||||
|
import PosterCleanupDialog from './PosterCleanupDialog.vue'
|
||||||
|
|
||||||
const props = defineProps({
|
const props = defineProps({
|
||||||
source: { type: Object, required: true },
|
source: { type: Object, required: true },
|
||||||
@@ -75,10 +89,14 @@ const running = computed(() => props.source.backfill_state === 'running')
|
|||||||
const recovering = computed(() => !!props.source.backfill_bypass_seen)
|
const recovering = computed(() => !!props.source.backfill_bypass_seen)
|
||||||
const recapturing = computed(() => !!props.source.backfill_recapture)
|
const recapturing = computed(() => !!props.source.backfill_recapture)
|
||||||
// Recover / recapture are native-ingester features (ledger-bypass re-walk and
|
// Recover / recapture are native-ingester features (ledger-bypass re-walk and
|
||||||
// post-text re-grab), available to every native platform — not just Patreon.
|
// post-text re-grab), available on every native platform. The backend says
|
||||||
// Mirrors backend download_backends.NATIVE_INGESTER_PLATFORMS.
|
// which those are (`native_ingester`); a copied list here went stale when
|
||||||
const NATIVE_PLATFORMS = ['patreon', 'subscribestar']
|
// Discord moved over.
|
||||||
const isNative = computed(() => NATIVE_PLATFORMS.includes(props.source.platform))
|
const isNative = computed(() => !!props.source.native_ingester)
|
||||||
|
// #4486: only a Discord source with an "Only posts by" list has other posters.
|
||||||
|
const hasPosterList = computed(() => props.source.platform === 'discord'
|
||||||
|
&& (props.source.config_overrides?.discord_authors || []).length > 0)
|
||||||
|
const cleanupOpen = ref(false)
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<style scoped>
|
<style scoped>
|
||||||
|
|||||||
@@ -16,7 +16,7 @@
|
|||||||
<a
|
<a
|
||||||
:href="source.url" target="_blank" rel="noopener"
|
:href="source.url" target="_blank" rel="noopener"
|
||||||
class="fc-source-card__url" @click.stop
|
class="fc-source-card__url" @click.stop
|
||||||
>{{ source.url }}</a>
|
>{{ source.display_name || source.url }}</a>
|
||||||
<v-btn
|
<v-btn
|
||||||
icon="mdi-pencil" size="x-small" variant="text"
|
icon="mdi-pencil" size="x-small" variant="text"
|
||||||
@click.stop="$emit('edit', source)"
|
@click.stop="$emit('edit', source)"
|
||||||
@@ -39,18 +39,18 @@
|
|||||||
</v-chip>
|
</v-chip>
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'running'"
|
v-else-if="source.backfill_state === 'running'"
|
||||||
size="x-small" color="info" variant="tonal" label
|
size="x-small" color="info" variant="flat" label
|
||||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||||
}}{{ source.backfill_posts
|
}}{{ source.backfill_posts
|
||||||
? ` · ${source.backfill_posts} posts`
|
? ` · ${source.backfill_posts} posts`
|
||||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'complete'"
|
v-else-if="source.backfill_state === 'complete'"
|
||||||
size="x-small" color="success" variant="tonal" label
|
size="x-small" color="success" variant="flat" label
|
||||||
>Backfilled</v-chip>
|
>Backfilled</v-chip>
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'stalled'"
|
v-else-if="source.backfill_state === 'stalled'"
|
||||||
size="x-small" color="warning" variant="tonal" label
|
size="x-small" color="warning" variant="flat" label
|
||||||
>Stalled</v-chip>
|
>Stalled</v-chip>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,17 @@
|
|||||||
v-model="structuredSince" label="Skip posts older than (YYYY-MM-DD)"
|
v-model="structuredSince" label="Skip posts older than (YYYY-MM-DD)"
|
||||||
placeholder="2024-01-01" hide-details class="mt-2"
|
placeholder="2024-01-01" hide-details class="mt-2"
|
||||||
/>
|
/>
|
||||||
|
<!-- A creator's server is full of other members posting their own
|
||||||
|
pictures; the source is subscribed to the creator (#4481). -->
|
||||||
|
<v-text-field
|
||||||
|
v-if="platform === 'discord'"
|
||||||
|
v-model="structuredAuthors" label="Only posts by (Discord names or ids)"
|
||||||
|
placeholder="Todding" hint="Comma-separated. Empty takes everyone's."
|
||||||
|
persistent-hint class="mt-2"
|
||||||
|
/>
|
||||||
|
<div v-if="platform === 'discord' && authorNames.length" class="text-caption mt-2">
|
||||||
|
{{ authorNames.join(', ') }}
|
||||||
|
</div>
|
||||||
<p class="text-caption mt-2" style="opacity: 0.75">
|
<p class="text-caption mt-2" style="opacity: 0.75">
|
||||||
More per-platform fields land here over time. Use Advanced JSON for everything else.
|
More per-platform fields land here over time. Use Advanced JSON for everything else.
|
||||||
</p>
|
</p>
|
||||||
@@ -98,6 +109,33 @@ const urlError = ref('')
|
|||||||
const configTab = ref('structured')
|
const configTab = ref('structured')
|
||||||
const structuredVideos = ref(true)
|
const structuredVideos = ref(true)
|
||||||
const structuredSince = ref('')
|
const structuredSince = ref('')
|
||||||
|
const structuredAuthors = ref('')
|
||||||
|
// Keys the structured view has no field for, carried through its saves so
|
||||||
|
// switching tabs never drops what the JSON view set.
|
||||||
|
const otherConfig = ref({})
|
||||||
|
const STRUCTURED_KEYS = ['videos', 'since', 'discord_authors']
|
||||||
|
|
||||||
|
// The extension's poster picker stores ids (#4491) and the name each was
|
||||||
|
// picked under, so a list of numbers still says who it means.
|
||||||
|
const authorNames = computed(() => {
|
||||||
|
const labels = otherConfig.value.discord_author_labels || {}
|
||||||
|
return splitAuthors(structuredAuthors.value)
|
||||||
|
.map(a => (labels[a] ? `${labels[a]} (${a})` : null))
|
||||||
|
.filter(Boolean)
|
||||||
|
})
|
||||||
|
|
||||||
|
function splitAuthors(txt) {
|
||||||
|
return (txt || '').split(',').map(s => s.trim()).filter(Boolean)
|
||||||
|
}
|
||||||
|
|
||||||
|
function takeConfig(co) {
|
||||||
|
structuredVideos.value = co.videos !== false
|
||||||
|
structuredSince.value = co.since ?? ''
|
||||||
|
structuredAuthors.value = Array.isArray(co.discord_authors) ? co.discord_authors.join(', ') : ''
|
||||||
|
otherConfig.value = Object.fromEntries(
|
||||||
|
Object.entries(co).filter(([k]) => !STRUCTURED_KEYS.includes(k)),
|
||||||
|
)
|
||||||
|
}
|
||||||
const jsonText = ref('{}')
|
const jsonText = ref('{}')
|
||||||
const jsonError = ref('')
|
const jsonError = ref('')
|
||||||
|
|
||||||
@@ -105,13 +143,15 @@ const busy = ref(false)
|
|||||||
|
|
||||||
// Sync config_overrides between the two views.
|
// Sync config_overrides between the two views.
|
||||||
const config = computed(() => {
|
const config = computed(() => {
|
||||||
const out = {}
|
const out = { ...otherConfig.value }
|
||||||
if (!structuredVideos.value) out.videos = false
|
if (!structuredVideos.value) out.videos = false
|
||||||
if (structuredSince.value) out.since = structuredSince.value
|
if (structuredSince.value) out.since = structuredSince.value
|
||||||
|
const authors = splitAuthors(structuredAuthors.value)
|
||||||
|
if (authors.length) out.discord_authors = authors
|
||||||
return out
|
return out
|
||||||
})
|
})
|
||||||
|
|
||||||
watch([structuredVideos, structuredSince], () => {
|
watch([structuredVideos, structuredSince, structuredAuthors], () => {
|
||||||
if (configTab.value === 'structured') {
|
if (configTab.value === 'structured') {
|
||||||
jsonText.value = JSON.stringify(config.value, null, 2)
|
jsonText.value = JSON.stringify(config.value, null, 2)
|
||||||
jsonError.value = ''
|
jsonError.value = ''
|
||||||
@@ -128,8 +168,7 @@ watch(jsonText, (txt) => {
|
|||||||
}
|
}
|
||||||
jsonError.value = ''
|
jsonError.value = ''
|
||||||
// Reflect recognized keys into the structured view.
|
// Reflect recognized keys into the structured view.
|
||||||
structuredVideos.value = parsed.videos !== false
|
takeConfig(parsed)
|
||||||
structuredSince.value = parsed.since ?? ''
|
|
||||||
} catch {
|
} catch {
|
||||||
jsonError.value = 'Invalid JSON'
|
jsonError.value = 'Invalid JSON'
|
||||||
}
|
}
|
||||||
@@ -146,14 +185,13 @@ watch(() => props.modelValue, async (open) => {
|
|||||||
url.value = props.source.url
|
url.value = props.source.url
|
||||||
enabled.value = props.source.enabled
|
enabled.value = props.source.enabled
|
||||||
const co = props.source.config_overrides || {}
|
const co = props.source.config_overrides || {}
|
||||||
structuredVideos.value = co.videos !== false
|
takeConfig(co)
|
||||||
structuredSince.value = co.since ?? ''
|
|
||||||
jsonText.value = JSON.stringify(co, null, 2)
|
jsonText.value = JSON.stringify(co, null, 2)
|
||||||
artistChoice.value = { id: props.source.artist_id, name: props.source.artist_name }
|
artistChoice.value = { id: props.source.artist_id, name: props.source.artist_name }
|
||||||
} else {
|
} else {
|
||||||
platform.value = platformsStore.list[0]?.key || 'patreon'
|
platform.value = platformsStore.list[0]?.key || 'patreon'
|
||||||
url.value = ''; enabled.value = true
|
url.value = ''; enabled.value = true
|
||||||
structuredVideos.value = true; structuredSince.value = ''
|
takeConfig({})
|
||||||
jsonText.value = '{}'
|
jsonText.value = '{}'
|
||||||
artistChoice.value = props.initialArtist
|
artistChoice.value = props.initialArtist
|
||||||
? { id: props.initialArtist.id, name: props.initialArtist.name }
|
? { id: props.initialArtist.id, name: props.initialArtist.name }
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
<div class="fc-source-row__url-wrap">
|
<div class="fc-source-row__url-wrap">
|
||||||
<a :href="source.url" target="_blank" rel="noopener" class="fc-source-row__url"
|
<a :href="source.url" target="_blank" rel="noopener" class="fc-source-row__url"
|
||||||
@click.stop>
|
@click.stop>
|
||||||
{{ source.url }}
|
{{ source.display_name || source.url }}
|
||||||
</a>
|
</a>
|
||||||
<!-- Edit sits next to the source identity (operator-requested), not in
|
<!-- Edit sits next to the source identity (operator-requested), not in
|
||||||
the action cluster where it was easy to fat-finger Remove. -->
|
the action cluster where it was easy to fat-finger Remove. -->
|
||||||
@@ -72,7 +72,7 @@
|
|||||||
worked, the content just isn't ours. -->
|
worked, the content just isn't ours. -->
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.error_type === 'tier_limited'"
|
v-else-if="source.error_type === 'tier_limited'"
|
||||||
size="x-small" color="info" variant="tonal" label
|
size="x-small" color="info" variant="flat" label
|
||||||
prepend-icon="mdi-lock-outline"
|
prepend-icon="mdi-lock-outline"
|
||||||
>{{ source.tier_gated_count ? `${source.tier_gated_count} gated` : 'No access' }}
|
>{{ source.tier_gated_count ? `${source.tier_gated_count} gated` : 'No access' }}
|
||||||
<v-tooltip activator="parent" location="top" max-width="480">
|
<v-tooltip activator="parent" location="top" max-width="480">
|
||||||
@@ -81,18 +81,18 @@
|
|||||||
</v-chip>
|
</v-chip>
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'running'"
|
v-else-if="source.backfill_state === 'running'"
|
||||||
size="x-small" color="info" variant="tonal" label
|
size="x-small" color="info" variant="flat" label
|
||||||
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
>{{ source.backfill_bypass_seen ? 'Recovering' : (source.backfill_recapture ? 'Recapturing' : 'Backfilling')
|
||||||
}}{{ source.backfill_posts
|
}}{{ source.backfill_posts
|
||||||
? ` · ${source.backfill_posts} posts`
|
? ` · ${source.backfill_posts} posts`
|
||||||
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
: (source.backfill_chunks ? ` (${source.backfill_chunks})` : '') }}</v-chip>
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'complete'"
|
v-else-if="source.backfill_state === 'complete'"
|
||||||
size="x-small" color="success" variant="tonal" label
|
size="x-small" color="success" variant="flat" label
|
||||||
>Backfilled</v-chip>
|
>Backfilled</v-chip>
|
||||||
<v-chip
|
<v-chip
|
||||||
v-else-if="source.backfill_state === 'stalled'"
|
v-else-if="source.backfill_state === 'stalled'"
|
||||||
size="x-small" color="warning" variant="tonal" label
|
size="x-small" color="warning" variant="flat" label
|
||||||
>Stalled</v-chip>
|
>Stalled</v-chip>
|
||||||
<span v-else class="fc-source-row__zero">0</span>
|
<span v-else class="fc-source-row__zero">0</span>
|
||||||
</td>
|
</td>
|
||||||
|
|||||||
@@ -108,7 +108,7 @@
|
|||||||
v-if="item.singleSource"
|
v-if="item.singleSource"
|
||||||
:href="item.singleSource.url" target="_blank" rel="noopener"
|
:href="item.singleSource.url" target="_blank" rel="noopener"
|
||||||
class="fc-subs__sub-url" @click.stop
|
class="fc-subs__sub-url" @click.stop
|
||||||
>{{ item.singleSource.url }}</a>
|
>{{ item.singleSource.display_name || item.singleSource.url }}</a>
|
||||||
</template>
|
</template>
|
||||||
|
|
||||||
<template #item.platforms="{ item }">
|
<template #item.platforms="{ item }">
|
||||||
@@ -517,6 +517,7 @@ const filteredGroups = computed(() => {
|
|||||||
g.sources.some(
|
g.sources.some(
|
||||||
(s) =>
|
(s) =>
|
||||||
(s.url || '').toLowerCase().includes(q) ||
|
(s.url || '').toLowerCase().includes(q) ||
|
||||||
|
(s.display_name || '').toLowerCase().includes(q) ||
|
||||||
(s.platform || '').toLowerCase().includes(q),
|
(s.platform || '').toLowerCase().includes(q),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
@@ -679,7 +680,7 @@ async function onRecover(source) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
// #830: arm a recapture walk (native platforms) — re-grab every post's body +
|
||||||
// external links and localize on-disk inline images, without re-downloading
|
// external links and localize on-disk inline images, without re-downloading
|
||||||
// media. Reuses the backfill lifecycle/badge; stop via the same Stop control.
|
// media. Reuses the backfill lifecycle/badge; stop via the same Stop control.
|
||||||
async function onRecapture(source) {
|
async function onRecapture(source) {
|
||||||
|
|||||||
@@ -10,7 +10,22 @@ export class ApiError extends Error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function request(method, url, { body, params, signal } = {}) {
|
const OPTIONS = new Set(['body', 'params', 'signal'])
|
||||||
|
|
||||||
|
async function request(method, url, opts = {}) {
|
||||||
|
// Refuse an option this wrapper does not know. `api.post(url, { id: 1 })`
|
||||||
|
// reads naturally and sends NO body — the payload lands in the options bag
|
||||||
|
// and is dropped — so the server answers `invalid_body` and the caller
|
||||||
|
// looks broken server-side. It shipped exactly that way in the Subscriptions
|
||||||
|
// "Add subscription" button (2026-09-24). Failing here names the mistake.
|
||||||
|
const unknown = Object.keys(opts).filter((k) => !OPTIONS.has(k))
|
||||||
|
if (unknown.length) {
|
||||||
|
throw new TypeError(
|
||||||
|
`useApi ${method} ${url}: unknown option(s) ${unknown.join(', ')} — ` +
|
||||||
|
'a request payload goes under `body`, a query under `params`'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
const { body, params, signal } = opts
|
||||||
let fullUrl = url
|
let fullUrl = url
|
||||||
if (params) {
|
if (params) {
|
||||||
const search = new URLSearchParams()
|
const search = new URLSearchParams()
|
||||||
|
|||||||
+17
-1
@@ -94,9 +94,25 @@ const routes = [
|
|||||||
const history =
|
const history =
|
||||||
typeof window !== 'undefined' ? createWebHistory() : createMemoryHistory()
|
typeof window !== 'undefined' ? createWebHistory() : createMemoryHistory()
|
||||||
|
|
||||||
|
// Where the window lands after a navigation. Without this the router keeps the
|
||||||
|
// window's scroll offset, so a view entered from halfway down another one opened
|
||||||
|
// halfway down itself, its heading and tab strip under the nav (operator-flagged
|
||||||
|
// 2026-09-24, Settings entered from a scrolled feed).
|
||||||
|
// - back/forward returns to where that entry was left;
|
||||||
|
// - the SAME view with a new query (a gallery filter, a Browse or
|
||||||
|
// Subscriptions tab) keeps its place — that is not a new page;
|
||||||
|
// - a new view opens at its top.
|
||||||
|
export function scrollBehavior(to, from, savedPosition) {
|
||||||
|
if (savedPosition) return savedPosition
|
||||||
|
if (from.matched.length && to.path === from.path) return false
|
||||||
|
if (to.hash) return { el: to.hash }
|
||||||
|
return { top: 0 }
|
||||||
|
}
|
||||||
|
|
||||||
const router = createRouter({
|
const router = createRouter({
|
||||||
history,
|
history,
|
||||||
routes
|
routes,
|
||||||
|
scrollBehavior
|
||||||
})
|
})
|
||||||
|
|
||||||
const DEFAULT_TITLE = 'FabledCurator'
|
const DEFAULT_TITLE = 'FabledCurator'
|
||||||
|
|||||||
@@ -119,7 +119,7 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
// --- Task progress polling (taps FC-3i activity dashboard) --------
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Polls /api/system/activity/runs?queue=maintenance every 3s,
|
* Polls /api/system/activity/runs?celery_task_id=<id> every 3s,
|
||||||
* resolves when a task_run row with the given celery task_id
|
* resolves when a task_run row with the given celery task_id
|
||||||
* reaches a terminal status (ok / error / timeout). Returns the
|
* reaches a terminal status (ok / error / timeout). Returns the
|
||||||
* row. Times out after 30 min by default.
|
* row. Times out after 30 min by default.
|
||||||
@@ -129,7 +129,9 @@ export const useAdminStore = defineStore('admin', () => {
|
|||||||
while (Date.now() < deadline) {
|
while (Date.now() < deadline) {
|
||||||
const body = await api.get(
|
const body = await api.get(
|
||||||
'/api/system/activity/runs',
|
'/api/system/activity/runs',
|
||||||
{ params: { queue: 'maintenance', limit: 20 } },
|
// By id, not by lane: these jobs run on `maintenance_long`, and a
|
||||||
|
// lane filter here is one more copy of the routing table (#4432).
|
||||||
|
{ params: { celery_task_id: taskId, limit: 1 } },
|
||||||
)
|
)
|
||||||
const row = (body.runs || []).find(r => r.celery_task_id === taskId)
|
const row = (body.runs || []).find(r => r.celery_task_id === taskId)
|
||||||
if (row && ['ok', 'error', 'timeout'].includes(row.status)) {
|
if (row && ['ok', 'error', 'timeout'].includes(row.status)) {
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
|||||||
min_width: 0,
|
min_width: 0,
|
||||||
min_height: 0,
|
min_height: 0,
|
||||||
transparency_threshold: 0.9,
|
transparency_threshold: 0.9,
|
||||||
single_color_threshold: 0.95,
|
single_color_threshold: 0.995,
|
||||||
single_color_tolerance: 30,
|
single_color_tolerance: 30,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -24,7 +24,7 @@ export const useCleanupStore = defineStore('cleanup', () => {
|
|||||||
min_width: s.min_width ?? 0,
|
min_width: s.min_width ?? 0,
|
||||||
min_height: s.min_height ?? 0,
|
min_height: s.min_height ?? 0,
|
||||||
transparency_threshold: s.transparency_threshold ?? 0.9,
|
transparency_threshold: s.transparency_threshold ?? 0.9,
|
||||||
single_color_threshold: s.single_color_threshold ?? 0.95,
|
single_color_threshold: s.single_color_threshold ?? 0.995,
|
||||||
single_color_tolerance: s.single_color_tolerance ?? 30,
|
single_color_tolerance: s.single_color_tolerance ?? 30,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ export const useMembershipReconcileStore = defineStore('membershipReconcile', ()
|
|||||||
async function adopt (membershipId) {
|
async function adopt (membershipId) {
|
||||||
try {
|
try {
|
||||||
const res = await api.post('/api/sources/reconciliation/adopt', {
|
const res = await api.post('/api/sources/reconciliation/adopt', {
|
||||||
membership_id: membershipId
|
body: { membership_id: membershipId }
|
||||||
})
|
})
|
||||||
toast({
|
toast({
|
||||||
text: res.already_tracked
|
text: res.already_tracked
|
||||||
|
|||||||
@@ -140,7 +140,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
_patchSource(body)
|
_patchSource(body)
|
||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
// #830: arm a recapture walk (Patreon-only) — re-grab every post's body +
|
// #830: arm a recapture walk (native platforms) — re-grab every post's body +
|
||||||
// external links and localize on-disk inline images, WITHOUT re-downloading
|
// external links and localize on-disk inline images, WITHOUT re-downloading
|
||||||
// media. Shares the backfill lifecycle/badge; stop via stopBackfill.
|
// media. Shares the backfill lifecycle/badge; stop via stopBackfill.
|
||||||
async function recaptureSource(id, artistIdHint = null) {
|
async function recaptureSource(id, artistIdHint = null) {
|
||||||
@@ -149,6 +149,15 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
return body
|
return body
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// #4486: a Discord source's posts by people outside its poster list.
|
||||||
|
// The preview touches nothing and hands back the token the remove must echo.
|
||||||
|
async function previewOtherPosters(id) {
|
||||||
|
return api.get(`/api/sources/${id}/discord/other-posters`)
|
||||||
|
}
|
||||||
|
async function removeOtherPosters(id, confirm) {
|
||||||
|
return api.post(`/api/sources/${id}/discord/other-posters/remove`, { body: { confirm } })
|
||||||
|
}
|
||||||
|
|
||||||
function sourcesByArtistGrouped() {
|
function sourcesByArtistGrouped() {
|
||||||
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
// returns [{artist: {id,name,slug}, sources: [...]}, ...]
|
||||||
const arr = byArtist.value.get(null) ?? []
|
const arr = byArtist.value.get(null) ?? []
|
||||||
@@ -179,6 +188,7 @@ export const useSourcesStore = defineStore('sources', () => {
|
|||||||
stopBackfill,
|
stopBackfill,
|
||||||
recoverSource,
|
recoverSource,
|
||||||
recaptureSource,
|
recaptureSource,
|
||||||
|
previewOtherPosters, removeOtherPosters,
|
||||||
findOrCreateArtist, autocompleteArtist, reassign,
|
findOrCreateArtist, autocompleteArtist, reassign,
|
||||||
loadScheduleStatus,
|
loadScheduleStatus,
|
||||||
sourcesByArtistGrouped,
|
sourcesByArtistGrouped,
|
||||||
|
|||||||
@@ -24,7 +24,8 @@ export function queueKey(queues) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Worst first. A stopped datastore is why someone opened this tab.
|
// Worst first. A stopped datastore is why someone opened this tab.
|
||||||
export const SEVERITY = { down: 3, stale: 2, unknown: 1, ok: 0 }
|
// `degraded`: checking in, but slow — a GPU agent on the CPU (#4410).
|
||||||
|
export const SEVERITY = { down: 4, stale: 3, degraded: 2, unknown: 1, ok: 0 }
|
||||||
|
|
||||||
export function kindLabel(kind) {
|
export function kindLabel(kind) {
|
||||||
if (kind === 'celery') return 'worker lane'
|
if (kind === 'celery') return 'worker lane'
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user